From eba3599a127c9c5d5b9491dcbadba262d763afed Mon Sep 17 00:00:00 2001 From: Jie Cai Date: Wed, 29 Nov 2023 06:37:35 +0800 Subject: [PATCH 001/176] add start/end moving step (#4437) * add start/end moving step * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * set the default parameters to be None in start/end_moving_window_step * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix the indent --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: root --- Python/pywarpx/picmi.py | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 1b0358e2e94..6b27069ea6b 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -581,6 +581,13 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): warpx_reflect_all_velocities: bool default=False Whether the sign of all of the particle velocities are changed upon reflection on a boundary, or only the velocity normal to the surface + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -598,6 +605,9 @@ def init(self, kw): self.potential_zmax = kw.pop('warpx_potential_hi_z', None) self.reflect_all_velocities = kw.pop('warpx_reflect_all_velocities', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -638,6 +648,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[1]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') @@ -672,6 +685,13 @@ class Cartesian1DGrid(picmistandard.PICMI_Cartesian1DGrid): warpx_potential_hi_z: float, default=0. Electrostatic potential on the upper longitudinal boundary + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -686,6 +706,9 @@ def init(self, kw): self.potential_zmin = kw.pop('warpx_potential_lo_z', None) self.potential_zmax = kw.pop('warpx_potential_hi_z', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -715,6 +738,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[0]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') @@ -760,6 +786,13 @@ class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): warpx_potential_hi_z: float, default=0. Electrostatic potential on the upper z boundary + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -776,6 +809,9 @@ def init(self, kw): self.potential_zmin = kw.pop('warpx_potential_lo_z', None) self.potential_zmax = kw.pop('warpx_potential_hi_z', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -810,6 +846,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[1]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') @@ -868,6 +907,13 @@ class Cartesian3DGrid(picmistandard.PICMI_Cartesian3DGrid): warpx_potential_hi_z: float, default=0. Electrostatic potential on the upper z boundary + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -886,6 +932,9 @@ def init(self, kw): self.potential_zmin = kw.pop('warpx_potential_lo_z', None) self.potential_zmax = kw.pop('warpx_potential_hi_z', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -925,6 +974,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[2]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') From 39e19a3721b8f4a0e3345d27ddd7e780140089be Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 28 Nov 2023 19:22:26 -0800 Subject: [PATCH 002/176] Frontier (OLCF): cupy (#4247) Documenting cupy installation for Frontier. --- .../machines/frontier-olcf/install_dependencies.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index 896cd9edbbc..9460f9a5175 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -90,7 +90,10 @@ python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools -python3 -m pip install --upgrade cython +# cupy and h5py need an older Cython +# https://github.com/cupy/cupy/issues/4610 +# https://github.com/h5py/h5py/issues/2268 +python3 -m pip install --upgrade "cython<3.0" python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas python3 -m pip install --upgrade scipy @@ -100,6 +103,14 @@ python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +# cupy for ROCm +# https://docs.cupy.dev/en/stable/install.html#building-cupy-for-rocm-from-source +# https://github.com/cupy/cupy/issues/7830 +CC=cc CXX=CC \ +CUPY_INSTALL_USE_HIP=1 \ +ROCM_HOME=${ROCM_PATH} \ +HCC_AMDGPU_TARGET=${AMREX_AMD_ARCH} \ + python3 -m pip install -v cupy # optional: for libEnsemble python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) From 7e54efe256876452a79274d4a108c188cf3c8215 Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 29 Nov 2023 12:08:40 -0800 Subject: [PATCH 003/176] Allow multiple injection sources per species (#4196) * Extensive clean up of PlasmaInjection * Small cleanup of parseMomentum * Fix string argument reference * Remove unneeded surface_flux * Allow multiple injection sources per species * Add documentation * Some clean up * Bug fix * Add ParmParseWithOptionalGroup * Used langmuir/inputs_2d as a CI test case * Update documentation * Each source must have injection_style specified * Fix in documentation * Small clean up in Examples/Tests/langmuir/inputs_2d * Add documentation to ParmParseWithOptionalGroup * Update nuclear_fusion tests to use multiple injection_source * Clean up "const int" * Make explicit double and float in ParmParseWithOptionalGroup * Make charge and mass ParticleReal * Update Deuterium_Tritium_Fusion_3D.json * Move ParmParseWithOptionalGroup to Utils and update method names * Allow the injection_style to be the same for all sources * Revert the nuclear_fusion/inputs_deuterium_tritium_rz test * Revert the changes to Examples/Tests/langmuir/inputs_2d * Small cleanup to use references insteads of pointers * Update Source/Particles/PhysicalParticleContainer.H Co-authored-by: Remi Lehe * Rework the ParmParse get routines * Update doc regarding moving windows * Further rework, now cleaning up the query routines * Rework the handling of the optional group name in input parameter parsing * Fix consts * Add multiple source capability to the PICMI interface * Small clean up in picmi --------- Co-authored-by: Remi Lehe --- Docs/source/usage/parameters.rst | 13 + .../inputs_deuterium_tritium_3d | 24 +- Examples/Tests/plasma_lens/PICMI_inputs_3d.py | 24 +- Examples/Tests/plasma_lens/analysis.py | 22 +- Python/pywarpx/Bucket.py | 11 +- Python/pywarpx/picmi.py | 194 +++++----- .../Deuterium_Tritium_Fusion_3D.json | 86 ++--- Source/Fluids/WarpXFluidContainer.cpp | 4 +- Source/Initialization/PlasmaInjector.H | 42 ++- Source/Initialization/PlasmaInjector.cpp | 330 +++++++++--------- Source/Initialization/TemperatureProperties.H | 3 +- .../Initialization/TemperatureProperties.cpp | 10 +- Source/Initialization/VelocityProperties.H | 3 +- Source/Initialization/VelocityProperties.cpp | 10 +- Source/Particles/PhysicalParticleContainer.H | 17 +- .../Particles/PhysicalParticleContainer.cpp | 317 ++++++++++------- Source/Particles/WarpXParticleContainer.H | 2 +- Source/Utils/Parser/ParserUtils.H | 223 +++++++++++- Source/Utils/Parser/ParserUtils.cpp | 52 ++- Source/Utils/SpeciesUtils.H | 4 +- Source/Utils/SpeciesUtils.cpp | 96 +++-- Source/Utils/WarpXMovingWindow.cpp | 3 +- 22 files changed, 951 insertions(+), 539 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index b9fedb4f050..a52a4c1d5e8 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -676,6 +676,19 @@ Particle initialization When ``.xmin`` and ``.xmax`` are set, they delimit the region within which particles are injected. If periodic boundary conditions are used in direction ``i``, then the default (i.e. if the range is not specified) range will be the simulation box, ``[geometry.prob_hi[i], geometry.prob_lo[i]]``. +* ``.injection_sources`` (``list of strings``) optional + Names of additional injection sources. By default, WarpX assumes one injection source per species, hence all of the input + parameters below describing the injection are parameters directly of the species. However, this option allows + additional sources, the names of which are specified here. For each source, the name of the source is added to the + input parameters below. For instance, with ``.injection_sources = source1 source2`` there can be the two input + parameters ``.source1.injection_style`` and ``.source2.injection_style``. + For the parameters of each source, the parameter with the name of the source will be used. + If it is not given, the value of the parameter without the source name will be used. This allows parameters used for all + sources to be specified once. For example, if the ``source1`` and ``source2`` have the same value of ``uz_m``, then it can be + set using ``.uz_m`` instead of setting it for each source. + Note that since by default ``.injection_style = none``, all injection sources can be input this way. + Note that if a moving window is used, the bulk velocity of all of the sources must be the same since it is used when updating the window. + * ``.injection_style`` (`string`; default: ``none``) Determines how the (macro-)particles will be injected in the simulation. The number of particles per cell is always given with respect to the coarsest level (level 0/mother grid), even if particles are immediately assigned to a refined patch. diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d index 3a8d2002a40..0943eee1d22 100644 --- a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d +++ b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d @@ -74,18 +74,22 @@ neutron_1.do_not_deposit = 1 my_constants.background_dens = 1.e26 my_constants.beam_dens = 1.e20 -deuterium_2.species_type = deuterium -deuterium_2.injection_style = "NRandomPerCell" -deuterium_2.num_particles_per_cell = 1000 -deuterium_2.profile = "parse_density_function" ## A tenth of the macroparticles in each cell is made of immobile high-density background deuteriums. ## The other nine tenths are made of fast low-density beam deuteriums. -deuterium_2.density_function(x,y,z) = if(y - floor(y) < 0.1, 10.*background_dens, 10./9.*beam_dens) -deuterium_2.momentum_distribution_type = "parse_momentum_function" -deuterium_2.momentum_function_ux(x,y,z) = 0. -deuterium_2.momentum_function_uy(x,y,z) = 0. -deuterium_2.momentum_function_uz(x,y,z) = "if(y - floor(y) < 0.1, - 0., sqrt(2*m_deuterium*Energy_step*(floor(z)**2))/(m_deuterium*clight))" +deuterium_2.species_type = deuterium +deuterium_2.injection_sources = high_density low_density +deuterium_2.profile = constant +deuterium_2.high_density.injection_style = "NRandomPerCell" +deuterium_2.high_density.num_particles_per_cell = 100 +deuterium_2.high_density.density = 10.*background_dens +deuterium_2.high_density.momentum_distribution_type = at_rest +deuterium_2.low_density.injection_style = "NRandomPerCell" +deuterium_2.low_density.num_particles_per_cell = 900 +deuterium_2.low_density.density = 10./9.*beam_dens +deuterium_2.low_density.momentum_distribution_type = "parse_momentum_function" +deuterium_2.low_density.momentum_function_ux(x,y,z) = 0. +deuterium_2.low_density.momentum_function_uy(x,y,z) = 0. +deuterium_2.low_density.momentum_function_uz(x,y,z) = sqrt(2*m_deuterium*Energy_step*(floor(z)**2))/(m_deuterium*clight) deuterium_2.do_not_push = 1 deuterium_2.do_not_deposit = 1 diff --git a/Examples/Tests/plasma_lens/PICMI_inputs_3d.py b/Examples/Tests/plasma_lens/PICMI_inputs_3d.py index 6114fb05de2..50d222bbf36 100644 --- a/Examples/Tests/plasma_lens/PICMI_inputs_3d.py +++ b/Examples/Tests/plasma_lens/PICMI_inputs_3d.py @@ -33,17 +33,25 @@ # Particles vel_z = 0.5*c -multiparticles_distribution = picmi.ParticleListDistribution(x = [0.05, 0.], - y = [0., 0.04], - z = [0.05, 0.05], - ux = [0., 0.], - uy = [0., 0.], - uz = [vel_z, vel_z], - weight = [1., 1.]) +offset_x_particle = picmi.ParticleListDistribution(x = [0.05], + y = [0.], + z = [0.05], + ux = [0.], + uy = [0.], + uz = [vel_z], + weight = [1.]) + +offset_y_particle = picmi.ParticleListDistribution(x = [0.], + y = [0.04], + z = [0.05], + ux = [0.], + uy = [0.], + uz = [vel_z], + weight = [1.]) electrons = picmi.Species(particle_type = 'electron', name = 'electrons', - initial_distribution = multiparticles_distribution) + initial_distribution = [offset_x_particle, offset_y_particle]) # Plasma lenses plasma_lenses = picmi.PlasmaLens(period = 0.5, diff --git a/Examples/Tests/plasma_lens/analysis.py b/Examples/Tests/plasma_lens/analysis.py index b410c6eb128..9ce82d903f8 100755 --- a/Examples/Tests/plasma_lens/analysis.py +++ b/Examples/Tests/plasma_lens/analysis.py @@ -92,12 +92,22 @@ def applylens(x0, vx0, vz0, gamma, lens_length, lens_strength): plasma_lens_strengths_B = np.zeros(len(plasma_lens_zstarts)) -x0 = float(ds.parameters.get('electrons.multiple_particles_pos_x').split()[0]) -y0 = float(ds.parameters.get('electrons.multiple_particles_pos_y').split()[1]) -z0 = float(ds.parameters.get('electrons.multiple_particles_pos_z').split()[0]) -ux0 = float(ds.parameters.get('electrons.multiple_particles_ux').split()[0])*c -uy0 = float(ds.parameters.get('electrons.multiple_particles_uy').split()[1])*c -uz0 = eval(ds.parameters.get('electrons.multiple_particles_uz').split()[0])*c +try: + # The picmi version + x0 = float(ds.parameters.get('electrons.dist0.multiple_particles_pos_x')) + y0 = float(ds.parameters.get('electrons.dist1.multiple_particles_pos_y')) + z0 = float(ds.parameters.get('electrons.dist0.multiple_particles_pos_z')) + ux0 = float(ds.parameters.get('electrons.dist0.multiple_particles_ux'))*c + uy0 = float(ds.parameters.get('electrons.dist1.multiple_particles_uy'))*c + uz0 = eval(ds.parameters.get('electrons.dist0.multiple_particles_uz'))*c +except TypeError: + # The inputs version + x0 = float(ds.parameters.get('electrons.multiple_particles_pos_x').split()[0]) + y0 = float(ds.parameters.get('electrons.multiple_particles_pos_y').split()[1]) + z0 = float(ds.parameters.get('electrons.multiple_particles_pos_z').split()[0]) + ux0 = float(ds.parameters.get('electrons.multiple_particles_ux').split()[0])*c + uy0 = float(ds.parameters.get('electrons.multiple_particles_uy').split()[1])*c + uz0 = eval(ds.parameters.get('electrons.multiple_particles_uz').split()[0])*c tt = 0. xx = x0 diff --git a/Python/pywarpx/Bucket.py b/Python/pywarpx/Bucket.py index 6beb97c8291..91a34b9d3d6 100644 --- a/Python/pywarpx/Bucket.py +++ b/Python/pywarpx/Bucket.py @@ -23,7 +23,7 @@ def _localsetattr(self, name, value): object.__setattr__(self, name, value) def add_new_attr(self, name, value): - """Names starting with "_" are make instance attributes. + """Names starting with "_" are made instance attributes. Otherwise the attribute is added to the args list. """ if name.startswith('_'): @@ -31,6 +31,15 @@ def add_new_attr(self, name, value): else: self.argvattrs[name] = value + def add_new_group_attr(self, group, name, value): + """The attribute is added to the args list in the form "group.name" if + group is not an empty string, otherwise as only "name". + """ + if group: + self.argvattrs[f'{group}.{name}'] = value + else: + self.argvattrs[name] = value + def __setattr__(self, name, value): self.add_new_attr(name, value) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 6b27069ea6b..520bc2250ab 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -271,7 +271,21 @@ def initialize_inputs(self, layout, pywarpx.Particles.particles_list.append(self.species) if self.initial_distribution is not None: - self.initial_distribution.initialize_inputs(self.species_number, layout, self.species, self.density_scale) + distributions_is_list = np.iterable(self.initial_distribution) + layout_is_list = np.iterable(layout) + if not distributions_is_list and not layout_is_list: + self.initial_distribution.initialize_inputs(self.species_number, layout, self.species, self.density_scale, '') + elif distributions_is_list and (layout_is_list or layout is None): + assert layout is None or (len(self.initial_distribution) == len(layout)),\ + Exception('The initial distribution and layout lists must have the same lenth') + source_names = [f'dist{i}' for i in range(len(self.initial_distribution))] + self.species.injection_sources = source_names + for i, dist in enumerate(self.initial_distribution): + layout_i = layout[i] if layout is not None else None + dist.initialize_inputs(self.species_number, layout_i, self.species, + self.density_scale, source_names[i]) + else: + raise Exception('The initial distribution and layout must both be scalars or both be lists') if injection_plane_position is not None: if injection_plane_normal_vector is not None: @@ -300,17 +314,17 @@ def init(self, kw): self.do_symmetrize = kw.pop('warpx_do_symmetrize', None) self.symmetrization_order = kw.pop('warpx_symmetrization_order', None) - def initialize_inputs(self, species_number, layout, species, density_scale): - species.injection_style = "gaussian_beam" - species.x_m = self.centroid_position[0] - species.y_m = self.centroid_position[1] - species.z_m = self.centroid_position[2] - species.x_rms = self.rms_bunch_size[0] - species.y_rms = self.rms_bunch_size[1] - species.z_rms = self.rms_bunch_size[2] + def initialize_inputs(self, species_number, layout, species, density_scale, source_name): + species.add_new_group_attr(source_name, 'injection_style', "gaussian_beam") + species.add_new_group_attr(source_name, 'x_m', self.centroid_position[0]) + species.add_new_group_attr(source_name, 'y_m', self.centroid_position[1]) + species.add_new_group_attr(source_name, 'z_m', self.centroid_position[2]) + species.add_new_group_attr(source_name, 'x_rms', self.rms_bunch_size[0]) + species.add_new_group_attr(source_name, 'y_rms', self.rms_bunch_size[1]) + species.add_new_group_attr(source_name, 'z_rms', self.rms_bunch_size[2]) # --- Only PseudoRandomLayout is supported - species.npart = layout.n_macroparticles + species.add_new_group_attr(source_name, 'npart', layout.n_macroparticles) # --- Calculate the total charge. Note that charge might be a string instead of a number. charge = species.charge @@ -318,9 +332,9 @@ def initialize_inputs(self, species_number, layout, species, density_scale): charge = constants.q_e elif charge == '-q_e': charge = -constants.q_e - species.q_tot = self.n_physical_particles*charge + species.add_new_group_attr(source_name, 'q_tot', self.n_physical_particles*charge) if density_scale is not None: - species.q_tot *= density_scale + species.add_new_group_attr(source_name, 'q_tot', density_scale) # --- The PICMI standard doesn't yet have a way of specifying these values. # --- They should default to the size of the domain. They are not typically @@ -334,26 +348,26 @@ def initialize_inputs(self, species_number, layout, species, density_scale): # --- Note that WarpX takes gamma*beta as input if np.any(np.not_equal(self.velocity_divergence, 0.)): - species.momentum_distribution_type = "radial_expansion" - species.u_over_r = self.velocity_divergence[0]/constants.c - #species.u_over_y = self.velocity_divergence[1]/constants.c - #species.u_over_z = self.velocity_divergence[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "radial_expansion") + species.add_new_group_attr(source_name, 'u_over_r', self.velocity_divergence[0]/constants.c) + #species.add_new_group_attr(source_name, 'u_over_y', self.velocity_divergence[1]/constants.c) + #species.add_new_group_attr(source_name, 'u_over_z', self.velocity_divergence[2]/constants.c) elif np.any(np.not_equal(self.rms_velocity, 0.)): - species.momentum_distribution_type = "gaussian" - species.ux_m = self.centroid_velocity[0]/constants.c - species.uy_m = self.centroid_velocity[1]/constants.c - species.uz_m = self.centroid_velocity[2]/constants.c - species.ux_th = self.rms_velocity[0]/constants.c - species.uy_th = self.rms_velocity[1]/constants.c - species.uz_th = self.rms_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussian") + species.add_new_group_attr(source_name, 'ux_m', self.centroid_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_m', self.centroid_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_m', self.centroid_velocity[2]/constants.c) + species.add_new_group_attr(source_name, 'ux_th', self.rms_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_th', self.rms_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_th', self.rms_velocity[2]/constants.c) else: - species.momentum_distribution_type = "constant" - species.ux = self.centroid_velocity[0]/constants.c - species.uy = self.centroid_velocity[1]/constants.c - species.uz = self.centroid_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "constant") + species.add_new_group_attr(source_name, 'ux', self.centroid_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy', self.centroid_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz', self.centroid_velocity[2]/constants.c) - species.do_symmetrize = self.do_symmetrize - species.symmetrization_order = self.symmetrization_order + species.add_new_group_attr(source_name, 'do_symmetrize', self.do_symmetrize) + species.add_new_group_attr(source_name, 'symmetrization_order', self.symmetrization_order) class DensityDistributionBase(object): @@ -369,98 +383,98 @@ def set_mangle_dict(self): # times self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) - def set_species_attributes(self, species, layout): + def set_species_attributes(self, species, layout, source_name): if isinstance(layout, GriddedLayout): # --- Note that the grid attribute of GriddedLayout is ignored - species.injection_style = "nuniformpercell" - species.num_particles_per_cell_each_dim = layout.n_macroparticle_per_cell + species.add_new_group_attr(source_name, 'injection_style', "nuniformpercell") + species.add_new_group_attr(source_name, 'num_particles_per_cell_each_dim', layout.n_macroparticle_per_cell) elif isinstance(layout, PseudoRandomLayout): assert (layout.n_macroparticles_per_cell is not None), Exception('WarpX only supports n_macroparticles_per_cell for the PseudoRandomLayout with this distribution') - species.injection_style = "nrandompercell" - species.num_particles_per_cell = layout.n_macroparticles_per_cell + species.add_new_group_attr(source_name, 'injection_style', "nrandompercell") + species.add_new_group_attr(source_name, 'num_particles_per_cell', layout.n_macroparticles_per_cell) else: raise Exception('WarpX does not support the specified layout for this distribution') - species.xmin = self.lower_bound[0] - species.xmax = self.upper_bound[0] - species.ymin = self.lower_bound[1] - species.ymax = self.upper_bound[1] - species.zmin = self.lower_bound[2] - species.zmax = self.upper_bound[2] + species.add_new_group_attr(source_name, 'xmin', self.lower_bound[0]) + species.add_new_group_attr(source_name, 'xmax', self.upper_bound[0]) + species.add_new_group_attr(source_name, 'ymin', self.lower_bound[1]) + species.add_new_group_attr(source_name, 'ymax', self.upper_bound[1]) + species.add_new_group_attr(source_name, 'zmin', self.lower_bound[2]) + species.add_new_group_attr(source_name, 'zmax', self.upper_bound[2]) if self.fill_in: - species.do_continuous_injection = 1 + species.add_new_group_attr(source_name, 'do_continuous_injection', 1) # --- Note that WarpX takes gamma*beta as input if (hasattr(self, "momentum_spread_expressions") and np.any(np.not_equal(self.momentum_spread_expressions, None)) ): species.momentum_distribution_type = 'gaussian_parse_momentum_function' - self.setup_parse_momentum_functions(species, self.momentum_expressions, '_m', self.directed_velocity) - self.setup_parse_momentum_functions(species, self.momentum_spread_expressions, '_th', [0.,0.,0.]) + self.setup_parse_momentum_functions(species, source_name, self.momentum_expressions, '_m', self.directed_velocity) + self.setup_parse_momentum_functions(species, source_name, self.momentum_spread_expressions, '_th', [0.,0.,0.]) elif (hasattr(self, "momentum_expressions") and np.any(np.not_equal(self.momentum_expressions, None)) ): - species.momentum_distribution_type = 'parse_momentum_function' - self.setup_parse_momentum_functions(species, self.momentum_expressions, '', self.directed_velocity) + species.add_new_group_attr(source_name, 'momentum_distribution_type', 'parse_momentum_function') + self.setup_parse_momentum_functions(species, source_name, self.momentum_expressions, '', self.directed_velocity) elif np.any(np.not_equal(self.rms_velocity, 0.)): - species.momentum_distribution_type = "gaussian" - species.ux_m = self.directed_velocity[0]/constants.c - species.uy_m = self.directed_velocity[1]/constants.c - species.uz_m = self.directed_velocity[2]/constants.c - species.ux_th = self.rms_velocity[0]/constants.c - species.uy_th = self.rms_velocity[1]/constants.c - species.uz_th = self.rms_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussian") + species.add_new_group_attr(source_name, 'ux_m', self.directed_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_m', self.directed_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_m', self.directed_velocity[2]/constants.c) + species.add_new_group_attr(source_name, 'ux_th', self.rms_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_th', self.rms_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_th', self.rms_velocity[2]/constants.c) else: - species.momentum_distribution_type = "constant" - species.ux = self.directed_velocity[0]/constants.c - species.uy = self.directed_velocity[1]/constants.c - species.uz = self.directed_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "constant") + species.add_new_group_attr(source_name, 'ux', self.directed_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy', self.directed_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz', self.directed_velocity[2]/constants.c) - def setup_parse_momentum_functions(self, species, expressions, suffix, defaults): + def setup_parse_momentum_functions(self, species, source_name, expressions, suffix, defaults): for sdir, idir in zip(['x', 'y', 'z'], [0, 1, 2]): if expressions[idir] is not None: expression = pywarpx.my_constants.mangle_expression(expressions[idir], self.mangle_dict) else: expression = f'{defaults[idir]}' - species.__setattr__(f'momentum_function_u{sdir}{suffix}(x,y,z)', f'({expression})/{constants.c}') + species.add_new_group_attr(source_name, f'momentum_function_u{sdir}{suffix}(x,y,z)', f'({expression})/{constants.c}') class UniformFluxDistribution(picmistandard.PICMI_UniformFluxDistribution, DensityDistributionBase): - def initialize_inputs(self, species_number, layout, species, density_scale): + def initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.fill_in = False self.set_mangle_dict() - self.set_species_attributes(species, layout) + self.set_species_attributes(species, layout, source_name) - species.flux_profile = "constant" - species.flux = self.flux + species.add_new_group_attr(source_name, 'flux_profile', "constant") + species.add_new_group_attr(source_name, 'flux', self.flux) if density_scale is not None: - species.flux *= density_scale - species.flux_normal_axis = self.flux_normal_axis - species.surface_flux_pos = self.surface_flux_position - species.flux_direction = self.flux_direction - species.flux_tmin = self.flux_tmin - species.flux_tmax = self.flux_tmax + species.add_new_group_attr(source_name, 'flux', density_scale) + species.add_new_group_attr(source_name, 'flux_normal_axis', self.flux_normal_axis) + species.add_new_group_attr(source_name, 'surface_flux_pos', self.surface_flux_position) + species.add_new_group_attr(source_name, 'flux_direction', self.flux_direction) + species.add_new_group_attr(source_name, 'flux_tmin', self.flux_tmin) + species.add_new_group_attr(source_name, 'flux_tmax', self.flux_tmax) # --- Use specific attributes for flux injection - species.injection_style = "nfluxpercell" + species.add_new_group_attr(source_name, 'injection_style', "nfluxpercell") assert (isinstance(layout, PseudoRandomLayout)), Exception('UniformFluxDistribution only supports the PseudoRandomLayout in WarpX') if self.gaussian_flux_momentum_distribution: - species.momentum_distribution_type = "gaussianflux" + species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussianflux") class UniformDistribution(picmistandard.PICMI_UniformDistribution, DensityDistributionBase): - def initialize_inputs(self, species_number, layout, species, density_scale): + def initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.set_mangle_dict() - self.set_species_attributes(species, layout) + self.set_species_attributes(species, layout, source_name) # --- Only constant density is supported by this class - species.profile = "constant" - species.density = self.density + species.add_new_group_attr(source_name, 'profile', "constant") + species.add_new_group_attr(source_name, 'density', self.density) if density_scale is not None: - species.density *= density_scale + species.add_new_group_attr(source_name, 'density', density_scale) class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDistributionBase): @@ -478,35 +492,35 @@ class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDist def init(self, kw): self.momentum_spread_expressions = kw.pop('warpx_momentum_spread_expressions', [None, None, None]) - def initialize_inputs(self, species_number, layout, species, density_scale): + def initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.set_mangle_dict() - self.set_species_attributes(species, layout) + self.set_species_attributes(species, layout, source_name) - species.profile = "parse_density_function" + species.add_new_group_attr(source_name, 'profile', "parse_density_function") expression = pywarpx.my_constants.mangle_expression(self.density_expression, self.mangle_dict) if density_scale is None: - species.__setattr__('density_function(x,y,z)', expression) + species.add_new_group_attr(source_name, 'density_function(x,y,z)', expression) else: - species.__setattr__('density_function(x,y,z)', "{}*({})".format(density_scale, expression)) + species.add_new_group_attr(source_name, 'density_function(x,y,z)', "{}*({})".format(density_scale, expression)) class ParticleListDistribution(picmistandard.PICMI_ParticleListDistribution): def init(self, kw): pass - def initialize_inputs(self, species_number, layout, species, density_scale): + def initialize_inputs(self, species_number, layout, species, density_scale, source_name): - species.injection_style = "multipleparticles" - species.multiple_particles_pos_x = self.x - species.multiple_particles_pos_y = self.y - species.multiple_particles_pos_z = self.z - species.multiple_particles_ux = np.array(self.ux)/constants.c - species.multiple_particles_uy = np.array(self.uy)/constants.c - species.multiple_particles_uz = np.array(self.uz)/constants.c - species.multiple_particles_weight = self.weight + species.add_new_group_attr(source_name, 'injection_style', "multipleparticles") + species.add_new_group_attr(source_name, 'multiple_particles_pos_x', self.x) + species.add_new_group_attr(source_name, 'multiple_particles_pos_y', self.y) + species.add_new_group_attr(source_name, 'multiple_particles_pos_z', self.z) + species.add_new_group_attr(source_name, 'multiple_particles_ux', np.array(self.ux)/constants.c) + species.add_new_group_attr(source_name, 'multiple_particles_uy', np.array(self.uy)/constants.c) + species.add_new_group_attr(source_name, 'multiple_particles_uz', np.array(self.uz)/constants.c) + species.add_new_group_attr(source_name, 'multiple_particles_weight', self.weight) if density_scale is not None: - species.multiple_particles_weight = self.weight*density_scale + species.add_new_group_attr(source_name, 'multiple_particles_weight', self.weight*density_scale) class ParticleDistributionPlanarInjector(picmistandard.PICMI_ParticleDistributionPlanarInjector): diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json index 382c6010c1c..23231eab789 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json @@ -1,23 +1,8 @@ { - "deuterium_1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 - }, - "deuterium_2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 3.356807324363973e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.0227810240779905e+29 + "lev=0": { + "rho": 0.0 }, - "helium4_1": { + "neutron_1": { "particle_momentum_x": 1.7519716491839538e-15, "particle_momentum_y": 1.7523289312260283e-15, "particle_momentum_z": 1.7480231586369996e-15, @@ -26,19 +11,25 @@ "particle_position_z": 325970.4138010667, "particle_weight": 4.421535775967805e-28 }, - "helium4_2": { - "particle_momentum_x": 1.5330942227771018e-15, - "particle_momentum_y": 1.5328473121602395e-15, - "particle_momentum_z": 1.7635828326228758e-15, - "particle_position_x": 137011.89739173267, - "particle_position_y": 136605.24328988983, - "particle_position_z": 290143.4673994485, - "particle_weight": 5.756530048087129e+18 + "tritium_1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.8875978729147693e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 }, - "lev=0": { - "rho": 0.0 + "deuterium_2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 3.356220762633467e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503, + "particle_weight": 1.0240001137713503e+30 }, - "neutron_1": { + "helium4_1": { "particle_momentum_x": 1.7519716491839538e-15, "particle_momentum_y": 1.7523289312260283e-15, "particle_momentum_z": 1.7480231586369996e-15, @@ -47,24 +38,33 @@ "particle_position_z": 325970.4138010667, "particle_weight": 4.421535775967805e-28 }, - "neutron_2": { - "particle_momentum_x": 1.5330942227771018e-15, - "particle_momentum_y": 1.5328473121602395e-15, - "particle_momentum_z": 1.549297051563983e-15, - "particle_position_x": 137011.89739173267, - "particle_position_y": 136605.24328988983, - "particle_position_z": 290143.4673994485, - "particle_weight": 5.756530048087129e+18 - }, - "tritium_1": { + "deuterium_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, "particle_weight": 1024.000000000021 }, + "neutron_2": { + "particle_momentum_x": 1.538345593941914e-15, + "particle_momentum_y": 1.536969107959402e-15, + "particle_momentum_z": 1.5535605933245691e-15, + "particle_position_x": 137088.2335716813, + "particle_position_y": 136370.55273167047, + "particle_position_z": 290148.26756873244, + "particle_weight": 6.427760042646557e+18 + }, + "helium4_2": { + "particle_momentum_x": 1.538345593941914e-15, + "particle_momentum_y": 1.536969107959402e-15, + "particle_momentum_z": 1.769309377628039e-15, + "particle_position_x": 137088.2335716813, + "particle_position_y": 136370.55273167047, + "particle_position_z": 290148.26756873244, + "particle_weight": 6.427760042646557e+18 + }, "tritium_2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -72,6 +72,6 @@ "particle_position_x": 409798.0158217681, "particle_position_y": 409670.9858143465, "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999999424347e+29 + "particle_weight": 1.0239999999357226e+29 } } diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index 5b6eacf2fc7..f27de6be1cc 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -27,8 +27,8 @@ WarpXFluidContainer::WarpXFluidContainer(int nlevs_max, int ispecies, const std: // Initialize injection objects const ParmParse pp_species_name(species_name); - SpeciesUtils::parseDensity(species_name, h_inj_rho, density_parser); - SpeciesUtils::parseMomentum(species_name, "none", h_inj_mom, + SpeciesUtils::parseDensity(species_name, "", h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, "", "none", h_inj_mom, ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel); if (h_inj_rho) { #ifdef AMREX_USE_GPU diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index cec84440159..616b1a74b17 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -45,7 +46,8 @@ public: /** Default constructor*/ PlasmaInjector () = default; - PlasmaInjector (int ispecies, const std::string& name, const amrex::Geometry& geom); + PlasmaInjector (int ispecies, const std::string& name, const amrex::Geometry& geom, + const std::string& src_name=""); // Default move and copy operations PlasmaInjector(const PlasmaInjector&) = delete; @@ -72,9 +74,8 @@ public: [[nodiscard]] amrex::XDim3 getMomentum ( amrex::Real x, amrex::Real y, amrex::Real z) const noexcept; - [[nodiscard]] amrex::Real getCharge () {return charge;} - [[nodiscard]] amrex::Real getMass () {return mass;} - [[nodiscard]] PhysicalSpecies getPhysicalSpecies() const {return physical_species;} + bool queryCharge (amrex::ParticleReal& a_charge) const; + bool queryMass (amrex::ParticleReal& a_mass) const; // bool: whether the initial injection of particles should be done // This routine is called during initialization of the plasma. @@ -138,17 +139,19 @@ public: amrex::Real density_min = std::numeric_limits::epsilon(); amrex::Real density_max = std::numeric_limits::max(); - InjectorPosition* getInjectorPosition (); - InjectorPosition* getInjectorFluxPosition (); - InjectorDensity* getInjectorDensity (); + [[nodiscard]] InjectorPosition* getInjectorPosition () const; + [[nodiscard]] InjectorPosition* getInjectorFluxPosition () const; + [[nodiscard]] InjectorDensity* getInjectorDensity () const; - InjectorFlux* getInjectorFlux (); - InjectorMomentum* getInjectorMomentumDevice (); - InjectorMomentum* getInjectorMomentumHost (); + [[nodiscard]] InjectorFlux* getInjectorFlux () const; + [[nodiscard]] InjectorMomentum* getInjectorMomentumDevice () const; + [[nodiscard]] InjectorMomentum* getInjectorMomentumHost () const; protected: - amrex::Real mass, charge; + bool mass_from_source = false; + bool charge_from_source = false; + amrex::ParticleReal mass, charge; PhysicalSpecies physical_species = PhysicalSpecies::unspecified; @@ -156,6 +159,7 @@ protected: int species_id; std::string species_name; + std::string source_name; std::unique_ptr h_inj_pos; InjectorPosition* d_inj_pos = nullptr; @@ -185,15 +189,15 @@ protected: std::unique_ptr h_mom_temp; std::unique_ptr h_mom_vel; - void setupSingleParticle (const amrex::ParmParse& pp_species_name); - void setupMultipleParticles (const amrex::ParmParse& pp_species_name); - void setupGaussianBeam (const amrex::ParmParse& pp_species_name); - void setupNRandomPerCell (const amrex::ParmParse& pp_species_name); - void setupNFluxPerCell (const amrex::ParmParse& pp_species_name); - void setupNuniformPerCell (const amrex::ParmParse& pp_species_name); - void setupExternalFile (const amrex::ParmParse& pp_species_name); + void setupSingleParticle (amrex::ParmParse const& pp_species); + void setupMultipleParticles (amrex::ParmParse const& pp_species); + void setupGaussianBeam (amrex::ParmParse const& pp_species); + void setupNRandomPerCell (amrex::ParmParse const& pp_species); + void setupNFluxPerCell (amrex::ParmParse const& pp_species); + void setupNuniformPerCell (amrex::ParmParse const& pp_species); + void setupExternalFile (amrex::ParmParse const& pp_species); - void parseFlux (const amrex::ParmParse& pp_species_name); + void parseFlux (amrex::ParmParse const& pp_species); }; #endif diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index e5b503af786..3b2ad76513a 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -46,10 +46,9 @@ using namespace amrex::literals; PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, - const amrex::Geometry& geom): - species_id{ispecies}, species_name{name} + const amrex::Geometry& geom, const std::string& src_name): + species_id{ispecies}, species_name{name}, source_name{src_name} { - const amrex::ParmParse pp_species_name(species_name); #ifdef AMREX_USE_GPU static_assert(std::is_trivially_copyable::value, @@ -60,7 +59,9 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, "InjectorMomentum must be trivially copyable"); #endif - pp_species_name.query("radially_weighted", radially_weighted); + const amrex::ParmParse pp_species(species_name); + + utils::parser::queryWithParser(pp_species, source_name, "radially_weighted", radially_weighted); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(radially_weighted, "ERROR: Only radially_weighted=true is supported"); // Unlimited boundaries @@ -102,43 +103,41 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, } # endif - utils::parser::queryWithParser(pp_species_name, "xmin", xmin); - utils::parser::queryWithParser(pp_species_name, "ymin", ymin); - utils::parser::queryWithParser(pp_species_name, "zmin", zmin); - utils::parser::queryWithParser(pp_species_name, "xmax", xmax); - utils::parser::queryWithParser(pp_species_name, "ymax", ymax); - utils::parser::queryWithParser(pp_species_name, "zmax", zmax); + utils::parser::queryWithParser(pp_species, source_name, "xmin", xmin); + utils::parser::queryWithParser(pp_species, source_name, "ymin", ymin); + utils::parser::queryWithParser(pp_species, source_name, "zmin", zmin); + utils::parser::queryWithParser(pp_species, source_name, "xmax", xmax); + utils::parser::queryWithParser(pp_species, source_name, "ymax", ymax); + utils::parser::queryWithParser(pp_species, source_name, "zmax", zmax); - utils::parser::queryWithParser(pp_species_name, "density_min", density_min); - utils::parser::queryWithParser(pp_species_name, "density_max", density_max); + utils::parser::queryWithParser(pp_species, source_name, "density_min", density_min); + utils::parser::queryWithParser(pp_species, source_name, "density_max", density_max); std::string injection_style = "none"; - // Parse injection style - pp_species_name.query("injection_style", injection_style); + utils::parser::query(pp_species, source_name, "injection_style", injection_style); std::transform(injection_style.begin(), - injection_style.end(), - injection_style.begin(), - ::tolower); - SpeciesUtils::extractSpeciesProperties(species_name, injection_style, charge, mass, physical_species); + injection_style.end(), + injection_style.begin(), + ::tolower); num_particles_per_cell_each_dim.assign(3, 0); if (injection_style == "singleparticle") { - setupSingleParticle(pp_species_name); + setupSingleParticle(pp_species); return; } else if (injection_style == "multipleparticles") { - setupMultipleParticles(pp_species_name); + setupMultipleParticles(pp_species); return; } else if (injection_style == "gaussian_beam") { - setupGaussianBeam(pp_species_name); + setupGaussianBeam(pp_species); } else if (injection_style == "nrandompercell") { - setupNRandomPerCell(pp_species_name); + setupNRandomPerCell(pp_species); } else if (injection_style == "nfluxpercell") { - setupNFluxPerCell(pp_species_name); + setupNFluxPerCell(pp_species); } else if (injection_style == "nuniformpercell") { - setupNuniformPerCell(pp_species_name); + setupNuniformPerCell(pp_species); } else if (injection_style == "external_file") { - setupExternalFile(pp_species_name); + setupExternalFile(pp_species); } else if (injection_style != "none") { SpeciesUtils::StringParseAbortMessage("Injection style", injection_style); } @@ -184,36 +183,26 @@ PlasmaInjector::~PlasmaInjector () PlasmaInjector::~PlasmaInjector () = default; #endif -void PlasmaInjector::setupSingleParticle (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupSingleParticle (amrex::ParmParse const& pp_species) { - utils::parser::getArrWithParser( - pp_species_name, "single_particle_pos", single_particle_pos, 0, 3); - utils::parser::getArrWithParser( - pp_species_name, "single_particle_u", single_particle_u, 0, 3); + utils::parser::getArrWithParser(pp_species, source_name, "single_particle_pos", single_particle_pos, 0, 3); + utils::parser::getArrWithParser(pp_species, source_name, "single_particle_u", single_particle_u, 0, 3); for (auto& x : single_particle_u) { x *= PhysConst::c; } - utils::parser::getWithParser( - pp_species_name, "single_particle_weight", single_particle_weight); + utils::parser::getWithParser(pp_species, source_name, "single_particle_weight", single_particle_weight); add_single_particle = true; } -void PlasmaInjector::setupMultipleParticles (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupMultipleParticles (amrex::ParmParse const& pp_species) { - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_pos_x", multiple_particles_pos_x); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_pos_y", multiple_particles_pos_y); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_pos_z", multiple_particles_pos_z); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_ux", multiple_particles_ux); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_uy", multiple_particles_uy); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_uz", multiple_particles_uz); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_weight", multiple_particles_weight); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_pos_x", multiple_particles_pos_x); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_pos_y", multiple_particles_pos_y); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_pos_z", multiple_particles_pos_z); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_ux", multiple_particles_ux); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_uy", multiple_particles_uy); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_uz", multiple_particles_uz); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_weight", multiple_particles_weight); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( ((multiple_particles_pos_x.size() == multiple_particles_pos_y.size()) && (multiple_particles_pos_x.size() == multiple_particles_pos_z.size()) && @@ -228,26 +217,26 @@ void PlasmaInjector::setupMultipleParticles (const amrex::ParmParse& pp_species_ add_multiple_particles = true; } -void PlasmaInjector::setupGaussianBeam (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupGaussianBeam (amrex::ParmParse const& pp_species) { - utils::parser::getWithParser(pp_species_name, "x_m", x_m); - utils::parser::getWithParser(pp_species_name, "y_m", y_m); - utils::parser::getWithParser(pp_species_name, "z_m", z_m); - utils::parser::getWithParser(pp_species_name, "x_rms", x_rms); - utils::parser::getWithParser(pp_species_name, "y_rms", y_rms); - utils::parser::getWithParser(pp_species_name, "z_rms", z_rms); - utils::parser::queryWithParser(pp_species_name, "x_cut", x_cut); - utils::parser::queryWithParser(pp_species_name, "y_cut", y_cut); - utils::parser::queryWithParser(pp_species_name, "z_cut", z_cut); - utils::parser::getWithParser(pp_species_name, "q_tot", q_tot); - utils::parser::getWithParser(pp_species_name, "npart", npart); - pp_species_name.query("do_symmetrize", do_symmetrize); - pp_species_name.query("symmetrization_order", symmetrization_order); + utils::parser::getWithParser(pp_species, source_name, "x_m", x_m); + utils::parser::getWithParser(pp_species, source_name, "y_m", y_m); + utils::parser::getWithParser(pp_species, source_name, "z_m", z_m); + utils::parser::getWithParser(pp_species, source_name, "x_rms", x_rms); + utils::parser::getWithParser(pp_species, source_name, "y_rms", y_rms); + utils::parser::getWithParser(pp_species, source_name, "z_rms", z_rms); + utils::parser::queryWithParser(pp_species, source_name, "x_cut", x_cut); + utils::parser::queryWithParser(pp_species, source_name, "y_cut", y_cut); + utils::parser::queryWithParser(pp_species, source_name, "z_cut", z_cut); + utils::parser::getWithParser(pp_species, source_name, "q_tot", q_tot); + utils::parser::getWithParser(pp_species, source_name, "npart", npart); + utils::parser::queryWithParser(pp_species, source_name, "do_symmetrize", do_symmetrize); + utils::parser::queryWithParser(pp_species, source_name, "symmetrization_order", symmetrization_order); const std::set valid_symmetries = {4,8}; WARPX_ALWAYS_ASSERT_WITH_MESSAGE( valid_symmetries.count(symmetrization_order), "Error: Symmetrization only supported to orders 4 or 8 "); gaussian_beam = true; - SpeciesUtils::parseMomentum(species_name, "gaussian_beam", h_inj_mom, + SpeciesUtils::parseMomentum(species_name, source_name, "gaussian_beam", h_inj_mom, ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel); @@ -265,10 +254,9 @@ void PlasmaInjector::setupGaussianBeam (const amrex::ParmParse& pp_species_name) #endif } -void PlasmaInjector::setupNRandomPerCell (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupNRandomPerCell (amrex::ParmParse const& pp_species) { - utils::parser::getWithParser( - pp_species_name, "num_particles_per_cell", num_particles_per_cell); + utils::parser::getWithParser(pp_species, source_name, "num_particles_per_cell", num_particles_per_cell); #if WARPX_DIM_RZ if (WarpX::n_rz_azimuthal_modes > 1) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -289,17 +277,17 @@ void PlasmaInjector::setupNRandomPerCell (const amrex::ParmParse& pp_species_nam #else d_inj_pos = h_inj_pos.get(); #endif - SpeciesUtils::parseDensity(species_name, h_inj_rho, density_parser); - SpeciesUtils::parseMomentum(species_name, "nrandompercell", h_inj_mom, + + SpeciesUtils::parseDensity(species_name, source_name, h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, source_name, "nrandompercell", h_inj_mom, ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel); } -void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupNFluxPerCell (amrex::ParmParse const& pp_species) { - utils::parser::getWithParser( - pp_species_name, "num_particles_per_cell", num_particles_per_cell_real); + utils::parser::getWithParser(pp_species, source_name, "num_particles_per_cell", num_particles_per_cell_real); #ifdef WARPX_DIM_RZ if (WarpX::n_rz_azimuthal_modes > 1) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -309,14 +297,11 @@ void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) "(Please visit PR#765 for more information.)"); } #endif - utils::parser::getWithParser( - pp_species_name, "surface_flux_pos", surface_flux_pos); - utils::parser::queryWithParser( - pp_species_name, "flux_tmin", flux_tmin); - utils::parser::queryWithParser( - pp_species_name, "flux_tmax", flux_tmax); + utils::parser::getWithParser(pp_species, source_name, "surface_flux_pos", surface_flux_pos); + utils::parser::queryWithParser(pp_species, source_name, "flux_tmin", flux_tmin); + utils::parser::queryWithParser(pp_species, source_name, "flux_tmax", flux_tmax); std::string flux_normal_axis_string; - pp_species_name.get("flux_normal_axis", flux_normal_axis_string); + utils::parser::get(pp_species, source_name, "flux_normal_axis", flux_normal_axis_string); flux_normal_axis = -1; #ifdef WARPX_DIM_RZ if (flux_normal_axis_string == "r" || flux_normal_axis_string == "R") { @@ -353,7 +338,7 @@ void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) #endif WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flux_normal_axis >= 0, "Error: Invalid value for flux_normal_axis. It must be " + flux_normal_axis_help); - pp_species_name.get("flux_direction", flux_direction); + utils::parser::getWithParser(pp_species, source_name, "flux_direction", flux_direction); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flux_direction == +1 || flux_direction == -1, "Error: flux_direction must be -1 or +1."); // Construct InjectorPosition with InjectorPositionRandom. @@ -369,15 +354,15 @@ void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) d_flux_pos = h_flux_pos.get(); #endif - parseFlux(pp_species_name); - SpeciesUtils::parseMomentum(species_name, "nfluxpercell", h_inj_mom, + parseFlux(pp_species); + SpeciesUtils::parseMomentum(species_name, source_name, "nfluxpercell", h_inj_mom, ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel, flux_normal_axis, flux_direction); } -void PlasmaInjector::setupNuniformPerCell (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupNuniformPerCell (amrex::ParmParse const& pp_species) { // Note that for RZ, three numbers are expected, r, theta, and z. // For 2D, only two are expected. The third is overwritten with 1. @@ -389,9 +374,8 @@ void PlasmaInjector::setupNuniformPerCell (const amrex::ParmParse& pp_species_na #else constexpr int num_required_ppc_each_dim = 3; #endif - utils::parser::getArrWithParser( - pp_species_name, "num_particles_per_cell_each_dim", - num_particles_per_cell_each_dim, 0, num_required_ppc_each_dim); + utils::parser::getArrWithParser(pp_species, source_name, "num_particles_per_cell_each_dim", num_particles_per_cell_each_dim, + 0, num_required_ppc_each_dim); #if WARPX_DIM_XZ num_particles_per_cell_each_dim.push_back(1); #endif @@ -425,14 +409,14 @@ void PlasmaInjector::setupNuniformPerCell (const amrex::ParmParse& pp_species_na num_particles_per_cell = num_particles_per_cell_each_dim[0] * num_particles_per_cell_each_dim[1] * num_particles_per_cell_each_dim[2]; - SpeciesUtils::parseDensity(species_name, h_inj_rho, density_parser); - SpeciesUtils::parseMomentum(species_name, "nuniformpercell", h_inj_mom, + SpeciesUtils::parseDensity(species_name, source_name, h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, source_name, "nuniformpercell", h_inj_mom, ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel); } -void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupExternalFile (amrex::ParmParse const& pp_species) { #ifndef WARPX_USE_OPENPMD WARPX_ABORT_WITH_MESSAGE( @@ -441,15 +425,15 @@ void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) #endif external_file = true; std::string str_injection_file; - pp_species_name.get("injection_file", str_injection_file); + utils::parser::get(pp_species, source_name, "injection_file", str_injection_file); // optional parameters - utils::parser::queryWithParser(pp_species_name, "q_tot", q_tot); - utils::parser::queryWithParser(pp_species_name, "z_shift",z_shift); + utils::parser::queryWithParser(pp_species, source_name, "q_tot", q_tot); + utils::parser::queryWithParser(pp_species, source_name, "z_shift",z_shift); #ifdef WARPX_USE_OPENPMD - const bool charge_is_specified = pp_species_name.contains("charge"); - const bool mass_is_specified = pp_species_name.contains("mass"); - const bool species_is_specified = pp_species_name.contains("species_type"); + const bool charge_is_specified = pp_species.contains("charge"); + const bool mass_is_specified = pp_species.contains("mass"); + const bool species_is_specified = pp_species.contains("species_type"); if (amrex::ParallelDescriptor::IOProcessor()) { m_openpmd_input_series = std::make_unique( @@ -465,70 +449,71 @@ void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) std::string const ps_name = it.particles.begin()->first; openPMD::ParticleSpecies ps = it.particles.begin()->second; - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ps.contains("charge") || charge_is_specified || species_is_specified, - std::string("'") + ps_name + - ".injection_file' does not contain a 'charge' species record. " - "Please specify '" + ps_name + ".charge' or " - "'" + ps_name + ".species_type' in your input file!\n"); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ps.contains("mass") || mass_is_specified || species_is_specified, - std::string("'") + ps_name + - ".injection_file' does not contain a 'mass' species record. " - "Please specify '" + ps_name + ".mass' or " - "'" + ps_name + ".species_type' in your input file!\n"); - - if (charge_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".charge' and '" + - ps_name + ".injection_file' specify a charge.\n'" + - ps_name + ".charge' will take precedence.\n"); - } - else if (species_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".species_type' and '" + - ps_name + ".injection_file' specify a charge.\n'" + - ps_name + ".species_type' will take precedence.\n"); + charge_from_source = ps.contains("charge"); + mass_from_source = ps.contains("mass"); + + if (charge_from_source) { + if (charge_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".charge' and '" + + ps_name + ".injection_file' specify a charge.\n'" + + ps_name + ".charge' will take precedence.\n"); + } + else if (species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".species_type' and '" + + ps_name + ".injection_file' specify a charge.\n'" + + ps_name + ".species_type' will take precedence.\n"); + } + else { + // TODO: Add ASSERT_WITH_MESSAGE to test if charge is a constant record + auto p_q_ptr = + ps["charge"][openPMD::RecordComponent::SCALAR].loadChunk(); + m_openpmd_input_series->flush(); + amrex::ParticleReal const p_q = p_q_ptr.get()[0]; + auto const charge_unit = static_cast(ps["charge"][openPMD::RecordComponent::SCALAR].unitSI()); + charge = p_q * charge_unit; + } } - else { - // TODO: Add ASSERT_WITH_MESSAGE to test if charge is a constant record - auto p_q_ptr = - ps["charge"][openPMD::RecordComponent::SCALAR].loadChunk(); - m_openpmd_input_series->flush(); - amrex::ParticleReal const p_q = p_q_ptr.get()[0]; - auto const charge_unit = static_cast(ps["charge"][openPMD::RecordComponent::SCALAR].unitSI()); - charge = p_q * charge_unit; - } - if (mass_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".mass' and '" + - ps_name + ".injection_file' specify a charge.\n'" + - ps_name + ".mass' will take precedence.\n"); - } - else if (species_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".species_type' and '" + - ps_name + ".injection_file' specify a mass.\n'" + - ps_name + ".species_type' will take precedence.\n"); - } - else { - // TODO: Add ASSERT_WITH_MESSAGE to test if mass is a constant record - auto p_m_ptr = - ps["mass"][openPMD::RecordComponent::SCALAR].loadChunk(); - m_openpmd_input_series->flush(); - amrex::ParticleReal const p_m = p_m_ptr.get()[0]; - auto const mass_unit = static_cast(ps["mass"][openPMD::RecordComponent::SCALAR].unitSI()); - mass = p_m * mass_unit; + if (mass_from_source) { + if (mass_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".mass' and '" + + ps_name + ".injection_file' specify a charge.\n'" + + ps_name + ".mass' will take precedence.\n"); + } + else if (species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".species_type' and '" + + ps_name + ".injection_file' specify a mass.\n'" + + ps_name + ".species_type' will take precedence.\n"); + } + else { + // TODO: Add ASSERT_WITH_MESSAGE to test if mass is a constant record + auto p_m_ptr = + ps["mass"][openPMD::RecordComponent::SCALAR].loadChunk(); + m_openpmd_input_series->flush(); + amrex::ParticleReal const p_m = p_m_ptr.get()[0]; + auto const mass_unit = static_cast(ps["mass"][openPMD::RecordComponent::SCALAR].unitSI()); + mass = p_m * mass_unit; + } } } // IOProcessor - // Broadcast charge and mass to non-IO processors - if (!charge_is_specified && !species_is_specified) - amrex::ParallelDescriptor::Bcast(&charge, 1, - amrex::ParallelDescriptor::IOProcessorNumber()); - if (!mass_is_specified && !species_is_specified) - amrex::ParallelDescriptor::Bcast(&mass, 1, - amrex::ParallelDescriptor::IOProcessorNumber()); + // Broadcast charge and mass to non-IO processors if read in from the file + if (!charge_is_specified && !species_is_specified) { + // Use ReduceBoolOr since Bcast(bool) doesn't compile + amrex::ParallelDescriptor::ReduceBoolOr(charge_from_source, amrex::ParallelDescriptor::IOProcessorNumber()); + if (charge_from_source) { + amrex::ParallelDescriptor::Bcast(&charge, 1, amrex::ParallelDescriptor::IOProcessorNumber()); + } + } + if (!mass_is_specified && !species_is_specified) { + amrex::ParallelDescriptor::ReduceBoolOr(mass_from_source, amrex::ParallelDescriptor::IOProcessorNumber()); + if (mass_from_source) { + amrex::ParallelDescriptor::Bcast(&mass, 1, amrex::ParallelDescriptor::IOProcessorNumber()); + } + } #else WARPX_ABORT_WITH_MESSAGE( "Plasma injection via external_file requires openPMD support: " @@ -539,20 +524,19 @@ void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) // Depending on injection type at runtime, initialize inj_flux // so that inj_flux->getFlux calls // InjectorFlux[Constant or Parser or etc.].getFlux. -void PlasmaInjector::parseFlux (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::parseFlux (amrex::ParmParse const& pp_species) { // parse flux information std::string flux_prof_s; - pp_species_name.get("flux_profile", flux_prof_s); + utils::parser::get(pp_species, source_name, "flux_profile", flux_prof_s); std::transform(flux_prof_s.begin(), flux_prof_s.end(), flux_prof_s.begin(), ::tolower); if (flux_prof_s == "constant") { - utils::parser::getWithParser(pp_species_name, "flux", flux); + utils::parser::getWithParser(pp_species, source_name, "flux", flux); // Construct InjectorFlux with InjectorFluxConstant. h_inj_flux.reset(new InjectorFlux((InjectorFluxConstant*)nullptr, flux)); } else if (flux_prof_s == "parse_flux_function") { - utils::parser::Store_parserString( - pp_species_name, "flux_function(x,y,z,t)", str_flux_function); + utils::parser::Store_parserString(pp_species, source_name, "flux_function(x,y,z,t)", str_flux_function); // Construct InjectorFlux with InjectorFluxParser. flux_parser = std::make_unique( utils::parser::makeParser(str_flux_function,{"x","y","z","t"})); @@ -595,38 +579,56 @@ bool PlasmaInjector::overlapsWith (const amrex::XDim3& lo, || (zmin > hi.z) || (zmax < lo.z) ); } +bool +PlasmaInjector::queryCharge (amrex::ParticleReal& a_charge) const +{ + if (charge_from_source) { + a_charge = charge; + } + return charge_from_source; +} + +bool +PlasmaInjector::queryMass (amrex::ParticleReal& a_mass) const +{ + if (mass_from_source) { + a_mass = mass; + } + return mass_from_source; +} + InjectorPosition* -PlasmaInjector::getInjectorPosition () +PlasmaInjector::getInjectorPosition () const { return d_inj_pos; } InjectorPosition* -PlasmaInjector::getInjectorFluxPosition () +PlasmaInjector::getInjectorFluxPosition () const { return d_flux_pos; } InjectorDensity* -PlasmaInjector::getInjectorDensity () +PlasmaInjector::getInjectorDensity () const { return d_inj_rho; } InjectorFlux* -PlasmaInjector::getInjectorFlux () +PlasmaInjector::getInjectorFlux () const { return d_inj_flux; } InjectorMomentum* -PlasmaInjector::getInjectorMomentumDevice () +PlasmaInjector::getInjectorMomentumDevice () const { return d_inj_mom; } InjectorMomentum* -PlasmaInjector::getInjectorMomentumHost () +PlasmaInjector::getInjectorMomentumHost () const { return h_inj_mom.get(); } diff --git a/Source/Initialization/TemperatureProperties.H b/Source/Initialization/TemperatureProperties.H index d9d9121c846..4d30b67c851 100644 --- a/Source/Initialization/TemperatureProperties.H +++ b/Source/Initialization/TemperatureProperties.H @@ -30,8 +30,9 @@ struct TemperatureProperties * information * * \param[in] pp: Reference to the parameter parser object for the species being initialized + * \param[in] source_name: Optional group name of the input parameters */ - TemperatureProperties (const amrex::ParmParse& pp); + TemperatureProperties (const amrex::ParmParse& pp, std::string const& source_name); /* Type of temperature initialization */ TemperatureInitType m_type; diff --git a/Source/Initialization/TemperatureProperties.cpp b/Source/Initialization/TemperatureProperties.cpp index 34e1d2b5fcc..ff73134a24e 100644 --- a/Source/Initialization/TemperatureProperties.cpp +++ b/Source/Initialization/TemperatureProperties.cpp @@ -17,17 +17,17 @@ * If temperature is a constant, store value. If a parser, make and * store the parser function */ -TemperatureProperties::TemperatureProperties (const amrex::ParmParse& pp) { +TemperatureProperties::TemperatureProperties (const amrex::ParmParse& pp, std::string const& source_name) { // Set defaults amrex::Real theta; std::string temp_dist_s = "constant"; std::string mom_dist_s; - pp.query("theta_distribution_type", temp_dist_s); - pp.query("momentum_distribution_type", mom_dist_s); + utils::parser::query(pp, source_name, "theta_distribution_type", temp_dist_s); + utils::parser::query(pp, source_name, "momentum_distribution_type", mom_dist_s); if (temp_dist_s == "constant") { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - utils::parser::queryWithParser(pp, "theta", theta), + utils::parser::queryWithParser(pp, source_name, "theta", theta), "Temperature parameter theta not specified"); // Do validation on theta value @@ -58,7 +58,7 @@ TemperatureProperties::TemperatureProperties (const amrex::ParmParse& pp) { } else if (temp_dist_s == "parser") { std::string str_theta_function; - utils::parser::Store_parserString(pp, "theta_function(x,y,z)", str_theta_function); + utils::parser::Store_parserString(pp, source_name, "theta_function(x,y,z)", str_theta_function); m_ptr_temperature_parser = std::make_unique( utils::parser::makeParser(str_theta_function,{"x","y","z"})); diff --git a/Source/Initialization/VelocityProperties.H b/Source/Initialization/VelocityProperties.H index a66d0fca653..453d97725e2 100644 --- a/Source/Initialization/VelocityProperties.H +++ b/Source/Initialization/VelocityProperties.H @@ -33,8 +33,9 @@ struct VelocityProperties * store the parser function * * \param[in] pp: Reference to the parameter parser object for the species being initialized + * \param[in] source_name: Optional group name of the input parameters */ - VelocityProperties (const amrex::ParmParse& pp); + VelocityProperties (const amrex::ParmParse& pp, std::string const& source_name); /* Type of velocity initialization */ VelocityInitType m_type; diff --git a/Source/Initialization/VelocityProperties.cpp b/Source/Initialization/VelocityProperties.cpp index 799b29e5be8..14b48c79524 100644 --- a/Source/Initialization/VelocityProperties.cpp +++ b/Source/Initialization/VelocityProperties.cpp @@ -11,13 +11,13 @@ #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" -VelocityProperties::VelocityProperties (const amrex::ParmParse& pp) +VelocityProperties::VelocityProperties (const amrex::ParmParse& pp, std::string const& source_name) { // Set defaults std::string vel_dist_s = "constant"; std::string vel_dir_s = "x"; - pp.query("bulk_vel_dir", vel_dir_s); + utils::parser::query(pp, source_name, "bulk_vel_dir", vel_dir_s); if(vel_dir_s[0] == '-'){ m_sign_dir = -1; } @@ -44,9 +44,9 @@ VelocityProperties::VelocityProperties (const amrex::ParmParse& pp) " other character."); } - pp.query("beta_distribution_type", vel_dist_s); + utils::parser::query(pp, source_name, "beta_distribution_type", vel_dist_s); if (vel_dist_s == "constant") { - utils::parser::queryWithParser(pp, "beta", m_velocity); + utils::parser::queryWithParser(pp, source_name, "beta", m_velocity); m_type = VelConstantValue; WARPX_ALWAYS_ASSERT_WITH_MESSAGE( m_velocity > -1 && m_velocity < 1, @@ -56,7 +56,7 @@ VelocityProperties::VelocityProperties (const amrex::ParmParse& pp) } else if (vel_dist_s == "parser") { std::string str_beta_function; - utils::parser::Store_parserString(pp, "beta_function(x,y,z)", str_beta_function); + utils::parser::Store_parserString(pp, source_name, "beta_function(x,y,z)", str_beta_function); m_ptr_velocity_parser = std::make_unique( utils::parser::makeParser(str_beta_function,{"x","y","z"})); diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index ec9834c4e7e..ef642fc718a 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -72,9 +72,9 @@ public: void InitIonizationModule () override; /* - * \brief Returns a pointer to the plasma injector. + * \brief Returns a pointer to the i'th plasma injector. */ - PlasmaInjector* GetPlasmaInjector () override; + PlasmaInjector* GetPlasmaInjector (int i) override; /** * \brief Evolve is the central function PhysicalParticleContainer that @@ -189,35 +189,40 @@ public: * number of particles per cell (in the cells of `part_realbox`). * The new particles are only created inside the intersection of `part_realbox` * with the local grid for the current proc. + * @param[in] the PlasmaInjector instance holding the input parameters * @param[in] lev the index of the refinement level * @param[in] part_realbox the box in which new particles should be created * (this box should correspond to an integer number of cells in each direction, * but its boundaries need not be aligned with the actual cells of the simulation) */ - void AddPlasma (int lev, amrex::RealBox part_realbox = amrex::RealBox()); + void AddPlasma (PlasmaInjector const& plasma_injector, int lev, amrex::RealBox part_realbox = amrex::RealBox()); /** * Create new macroparticles for this species, with a fixed * number of particles per cell in a plane. + * @param[in] the PlasmaInjector instance holding the input parameters * @param[in] dt time step size, used to partially advance the particles */ - void AddPlasmaFlux (amrex::Real dt); + void AddPlasmaFlux (PlasmaInjector const& plasma_injector, amrex::Real dt); void MapParticletoBoostedFrame (amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz, amrex::Real t_lab = 0.); void AddGaussianBeam ( + PlasmaInjector const& plasma_injector, amrex::Real x_m, amrex::Real y_m, amrex::Real z_m, amrex::Real x_rms, amrex::Real y_rms, amrex::Real z_rms, amrex::Real x_cut, amrex::Real y_cut, amrex::Real z_cut, amrex::Real q_tot, long npart, int do_symmetrize, int symmetrization_order); /** Load a particle beam from an external file + * @param[in] the PlasmaInjector instance holding the input parameters * @param[in] q_tot total charge of the particle species to be initialized * @param[in] z_shift optional shift to the z position of particles (useful for boosted frame runs) */ - void AddPlasmaFromFile (amrex::ParticleReal q_tot, + void AddPlasmaFromFile (PlasmaInjector & plasma_injector, + amrex::ParticleReal q_tot, amrex::ParticleReal z_shift); void CheckAndAddParticle ( @@ -355,7 +360,7 @@ public: protected: std::string species_name; - std::unique_ptr plasma_injector; + std::vector> plasma_injectors; // When true, adjust the transverse particle positions accounting // for the difference between the Lorentz transformed time of the diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 6df69eb4d9b..3fe1941775d 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -244,13 +244,81 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp { BackwardCompatibility(); - plasma_injector = std::make_unique(species_id, species_name, amr_core->Geom(0)); - physical_species = plasma_injector->getPhysicalSpecies(); - charge = plasma_injector->getCharge(); - mass = plasma_injector->getMass(); - const ParmParse pp_species_name(species_name); + std::string injection_style = "none"; + pp_species_name.query("injection_style", injection_style); + if (injection_style != "none") { + // The base plasma injector, whose input parameters have no source prefix. + // Only created if needed + plasma_injectors.push_back(std::make_unique(species_id, species_name, amr_core->Geom(0))); + } + + std::vector injection_sources; + pp_species_name.queryarr("injection_sources", injection_sources); + for (auto &source_name : injection_sources) { + plasma_injectors.push_back(std::make_unique(species_id, species_name, amr_core->Geom(0), + source_name)); + } + + // Setup the charge and mass. There are multiple ways that they can be specified, so checks are needed to + // ensure that a value is specified and warnings given if multiple values are specified. + // The ordering is that species.charge and species.mass take precedence over all other values. + // Next is charge and mass determined from species_type. + // Last is charge and mass from the plasma injector setup + bool charge_from_source = false; + bool mass_from_source = false; + for (auto const& plasma_injector : plasma_injectors) { + // For now, use the last value for charge and mass that is found. + // A check could be added for consistency of multiple values, but it'll probably never be needed + charge_from_source |= plasma_injector->queryCharge(charge); + mass_from_source |= plasma_injector->queryMass(mass); + } + + std::string physical_species_s; + const bool species_is_specified = pp_species_name.query("species_type", physical_species_s); + if (species_is_specified) { + const auto physical_species_from_string = species::from_string( physical_species_s ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(physical_species_from_string, + physical_species_s + " does not exist!"); + physical_species = physical_species_from_string.value(); + charge = species::get_charge( physical_species ); + mass = species::get_mass( physical_species ); + } + + // parse charge and mass (overriding values above) + const bool charge_is_specified = utils::parser::queryWithParser(pp_species_name, "charge", charge); + const bool mass_is_specified = utils::parser::queryWithParser(pp_species_name, "mass", mass); + + if (charge_is_specified && species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + species_name + ".charge' and " + + species_name + ".species_type' are specified.\n" + + species_name + ".charge' will take precedence.\n"); + } + if (mass_is_specified && species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + species_name + ".mass' and " + + species_name + ".species_type' are specified.\n" + + species_name + ".mass' will take precedence.\n"); + } + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + charge_from_source || + charge_is_specified || + species_is_specified, + "Need to specify at least one of species_type or charge for species '" + + species_name + "'." + ); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + mass_from_source || + mass_is_specified || + species_is_specified, + "Need to specify at least one of species_type or mass for species '" + + species_name + "'." + ); + pp_species_name.query("boost_adjust_transverse_positions", boost_adjust_transverse_positions); pp_species_name.query("do_backward_propagation", do_backward_propagation); pp_species_name.query("random_theta", m_rz_random_theta); @@ -374,7 +442,6 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) : WarpXParticleContainer(amr_core, 0) { - plasma_injector = std::make_unique(); } void @@ -448,6 +515,7 @@ void PhysicalParticleContainer::MapParticletoBoostedFrame ( void PhysicalParticleContainer::AddGaussianBeam ( + PlasmaInjector const& plasma_injector, const Real x_m, const Real y_m, const Real z_m, const Real x_rms, const Real y_rms, const Real z_rms, const Real x_cut, const Real y_cut, const Real z_cut, @@ -488,11 +556,11 @@ PhysicalParticleContainer::AddGaussianBeam ( constexpr Real y = 0._prt; const Real z = amrex::RandomNormal(z_m, z_rms); #endif - if (plasma_injector->insideBounds(x, y, z) && + if (plasma_injector.insideBounds(x, y, z) && std::abs( x - x_m ) <= x_cut * x_rms && std::abs( y - y_m ) <= y_cut * y_rms && std::abs( z - z_m ) <= z_cut * z_rms ) { - XDim3 u = plasma_injector->getMomentum(x, y, z); + XDim3 u = plasma_injector.getMomentum(x, y, z); u.x *= PhysConst::c; u.y *= PhysConst::c; u.z *= PhysConst::c; @@ -577,7 +645,8 @@ PhysicalParticleContainer::AddGaussianBeam ( } void -PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, +PhysicalParticleContainer::AddPlasmaFromFile(PlasmaInjector & plasma_injector, + ParticleReal q_tot, ParticleReal z_shift) { // Declare temporary vectors on the CPU @@ -592,10 +661,8 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, #ifdef WARPX_USE_OPENPMD //TODO: Make changes for read/write in multiple MPI ranks if (ParallelDescriptor::IOProcessor()) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(plasma_injector, - "AddPlasmaFromFile: plasma injector not initialized.\n"); // take ownership of the series and close it when done - auto series = std::move(plasma_injector->m_openpmd_input_series); + auto series = std::move(plasma_injector.m_openpmd_input_series); // assumption asserts: see PlasmaInjector openPMD::Iteration it = series->iterations.begin()->second; @@ -663,7 +730,7 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, #endif ParticleReal const z = ptr_z.get()[i]*position_unit_z + ptr_offset_z.get()[i]*position_offset_unit_z + z_shift; - if (plasma_injector->insideBounds(x, y, z)) { + if (plasma_injector.insideBounds(x, y, z)) { ParticleReal const ux = ptr_ux.get()[i]*momentum_unit_x/mass; ParticleReal const uz = ptr_uz.get()[i]*momentum_unit_z/mass; ParticleReal uy = 0.0_prt; @@ -701,7 +768,7 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, 1, attr, 0, attr_int, 1); #endif // WARPX_USE_OPENPMD - ignore_unused(q_tot, z_shift); + ignore_unused(plasma_injector, q_tot, z_shift); } void @@ -848,85 +915,85 @@ PhysicalParticleContainer::AddParticles (int lev) { WARPX_PROFILE("PhysicalParticleContainer::AddParticles()"); - if (plasma_injector->add_single_particle) { - if (WarpX::gamma_boost > 1.) { - MapParticletoBoostedFrame(plasma_injector->single_particle_pos[0], - plasma_injector->single_particle_pos[1], - plasma_injector->single_particle_pos[2], - plasma_injector->single_particle_u[0], - plasma_injector->single_particle_u[1], - plasma_injector->single_particle_u[2]); + for (auto const& plasma_injector : plasma_injectors) { + + if (plasma_injector->add_single_particle) { + if (WarpX::gamma_boost > 1.) { + MapParticletoBoostedFrame(plasma_injector->single_particle_pos[0], + plasma_injector->single_particle_pos[1], + plasma_injector->single_particle_pos[2], + plasma_injector->single_particle_u[0], + plasma_injector->single_particle_u[1], + plasma_injector->single_particle_u[2]); + } + amrex::Vector xp = {plasma_injector->single_particle_pos[0]}; + amrex::Vector yp = {plasma_injector->single_particle_pos[1]}; + amrex::Vector zp = {plasma_injector->single_particle_pos[2]}; + amrex::Vector uxp = {plasma_injector->single_particle_u[0]}; + amrex::Vector uyp = {plasma_injector->single_particle_u[1]}; + amrex::Vector uzp = {plasma_injector->single_particle_u[2]}; + amrex::Vector> attr = {{plasma_injector->single_particle_weight}}; + amrex::Vector> attr_int; + AddNParticles(lev, 1, xp, yp, zp, uxp, uyp, uzp, + 1, attr, 0, attr_int, 0); + return; } - amrex::Vector xp = {plasma_injector->single_particle_pos[0]}; - amrex::Vector yp = {plasma_injector->single_particle_pos[1]}; - amrex::Vector zp = {plasma_injector->single_particle_pos[2]}; - amrex::Vector uxp = {plasma_injector->single_particle_u[0]}; - amrex::Vector uyp = {plasma_injector->single_particle_u[1]}; - amrex::Vector uzp = {plasma_injector->single_particle_u[2]}; - amrex::Vector> attr = {{plasma_injector->single_particle_weight}}; - amrex::Vector> attr_int; - AddNParticles(lev, 1, xp, yp, zp, uxp, uyp, uzp, - 1, attr, 0, attr_int, 0); - return; - } - if (plasma_injector->add_multiple_particles) { - if (WarpX::gamma_boost > 1.) { - for (int i=0 ; i < plasma_injector->multiple_particles_pos_x.size() ; i++) { - MapParticletoBoostedFrame(plasma_injector->multiple_particles_pos_x[i], - plasma_injector->multiple_particles_pos_y[i], - plasma_injector->multiple_particles_pos_z[i], - plasma_injector->multiple_particles_ux[i], - plasma_injector->multiple_particles_uy[i], - plasma_injector->multiple_particles_uz[i]); + if (plasma_injector->add_multiple_particles) { + if (WarpX::gamma_boost > 1.) { + for (int i=0 ; i < plasma_injector->multiple_particles_pos_x.size() ; i++) { + MapParticletoBoostedFrame(plasma_injector->multiple_particles_pos_x[i], + plasma_injector->multiple_particles_pos_y[i], + plasma_injector->multiple_particles_pos_z[i], + plasma_injector->multiple_particles_ux[i], + plasma_injector->multiple_particles_uy[i], + plasma_injector->multiple_particles_uz[i]); + } } + amrex::Vector> attr; + attr.push_back(plasma_injector->multiple_particles_weight); + amrex::Vector> attr_int; + AddNParticles(lev, static_cast(plasma_injector->multiple_particles_pos_x.size()), + plasma_injector->multiple_particles_pos_x, + plasma_injector->multiple_particles_pos_y, + plasma_injector->multiple_particles_pos_z, + plasma_injector->multiple_particles_ux, + plasma_injector->multiple_particles_uy, + plasma_injector->multiple_particles_uz, + 1, attr, 0, attr_int, 0); } - amrex::Vector> attr; - attr.push_back(plasma_injector->multiple_particles_weight); - amrex::Vector> attr_int; - AddNParticles(lev, static_cast(plasma_injector->multiple_particles_pos_x.size()), - plasma_injector->multiple_particles_pos_x, - plasma_injector->multiple_particles_pos_y, - plasma_injector->multiple_particles_pos_z, - plasma_injector->multiple_particles_ux, - plasma_injector->multiple_particles_uy, - plasma_injector->multiple_particles_uz, - 1, attr, 0, attr_int, 0); - return; - } - if (plasma_injector->gaussian_beam) { - AddGaussianBeam(plasma_injector->x_m, - plasma_injector->y_m, - plasma_injector->z_m, - plasma_injector->x_rms, - plasma_injector->y_rms, - plasma_injector->z_rms, - plasma_injector->x_cut, - plasma_injector->y_cut, - plasma_injector->z_cut, - plasma_injector->q_tot, - plasma_injector->npart, - plasma_injector->do_symmetrize, - plasma_injector->symmetrization_order); - - - return; - } + if (plasma_injector->gaussian_beam) { + AddGaussianBeam(*plasma_injector, + plasma_injector->x_m, + plasma_injector->y_m, + plasma_injector->z_m, + plasma_injector->x_rms, + plasma_injector->y_rms, + plasma_injector->z_rms, + plasma_injector->x_cut, + plasma_injector->y_cut, + plasma_injector->z_cut, + plasma_injector->q_tot, + plasma_injector->npart, + plasma_injector->do_symmetrize, + plasma_injector->symmetrization_order); + } - if (plasma_injector->external_file) { - AddPlasmaFromFile(plasma_injector->q_tot, - plasma_injector->z_shift); - return; - } + if (plasma_injector->external_file) { + AddPlasmaFromFile(*plasma_injector, + plasma_injector->q_tot, + plasma_injector->z_shift); + } - if ( plasma_injector->doInjection() ) { - AddPlasma( lev ); + if ( plasma_injector->doInjection() ) { + AddPlasma(*plasma_injector, lev); + } } } void -PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) +PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int lev, RealBox part_realbox) { WARPX_PROFILE("PhysicalParticleContainer::AddPlasma()"); @@ -934,9 +1001,9 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const Geometry& geom = Geom(lev); if (!part_realbox.ok()) part_realbox = geom.ProbDomain(); - const int num_ppc = plasma_injector->num_particles_per_cell; + const int num_ppc = plasma_injector.num_particles_per_cell; #ifdef WARPX_DIM_RZ - Real rmax = std::min(plasma_injector->xmax, part_realbox.hi(0)); + Real rmax = std::min(plasma_injector.xmax, part_realbox.hi(0)); #endif const auto dx = geom.CellSizeArray(); @@ -963,18 +1030,18 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) fine_injection_box.coarsen(rrfac); } - InjectorPosition* inj_pos = plasma_injector->getInjectorPosition(); - InjectorDensity* inj_rho = plasma_injector->getInjectorDensity(); - InjectorMomentum* inj_mom = plasma_injector->getInjectorMomentumDevice(); + InjectorPosition* inj_pos = plasma_injector.getInjectorPosition(); + InjectorDensity* inj_rho = plasma_injector.getInjectorDensity(); + InjectorMomentum* inj_mom = plasma_injector.getInjectorMomentumDevice(); const Real gamma_boost = WarpX::gamma_boost; const Real beta_boost = WarpX::beta_boost; const Real t = WarpX::GetInstance().gett_new(lev); - const Real density_min = plasma_injector->density_min; - const Real density_max = plasma_injector->density_max; + const Real density_min = plasma_injector.density_min; + const Real density_max = plasma_injector.density_max; #ifdef WARPX_DIM_RZ const int nmodes = WarpX::n_rz_azimuthal_modes; - bool radially_weighted = plasma_injector->radially_weighted; + bool radially_weighted = plasma_injector.radially_weighted; #endif MFItInfo info; @@ -1448,16 +1515,16 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) } void -PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) +PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, amrex::Real dt) { WARPX_PROFILE("PhysicalParticleContainer::AddPlasmaFlux()"); const Geometry& geom = Geom(0); const amrex::RealBox& part_realbox = geom.ProbDomain(); - const amrex::Real num_ppc_real = plasma_injector->num_particles_per_cell_real; + const amrex::Real num_ppc_real = plasma_injector.num_particles_per_cell_real; #ifdef WARPX_DIM_RZ - Real rmax = std::min(plasma_injector->xmax, geom.ProbDomain().hi(0)); + Real rmax = std::min(plasma_injector.xmax, geom.ProbDomain().hi(0)); #endif const auto dx = geom.CellSizeArray(); @@ -1466,20 +1533,20 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) Real scale_fac = 0._rt; // Scale particle weight by the area of the emitting surface, within one cell #if defined(WARPX_DIM_3D) - scale_fac = dx[0]*dx[1]*dx[2]/dx[plasma_injector->flux_normal_axis]/num_ppc_real; + scale_fac = dx[0]*dx[1]*dx[2]/dx[plasma_injector.flux_normal_axis]/num_ppc_real; #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) scale_fac = dx[0]*dx[1]/num_ppc_real; // When emission is in the r direction, the emitting surface is a cylinder. // The factor 2*pi*r is added later below. - if (plasma_injector->flux_normal_axis == 0) scale_fac /= dx[0]; + if (plasma_injector.flux_normal_axis == 0) scale_fac /= dx[0]; // When emission is in the z direction, the emitting surface is an annulus // The factor 2*pi*r is added later below. - if (plasma_injector->flux_normal_axis == 2) scale_fac /= dx[1]; + if (plasma_injector.flux_normal_axis == 2) scale_fac /= dx[1]; // When emission is in the theta direction (flux_normal_axis == 1), // the emitting surface is a rectangle, within the plane of the simulation #elif defined(WARPX_DIM_1D_Z) scale_fac = dx[0]/num_ppc_real; - if (plasma_injector->flux_normal_axis == 2) scale_fac /= dx[0]; + if (plasma_injector.flux_normal_axis == 2) scale_fac /= dx[0]; #endif amrex::LayoutData* cost = WarpX::getCosts(0); @@ -1506,16 +1573,16 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) fine_injection_box.coarsen(rrfac); } - InjectorPosition* flux_pos = plasma_injector->getInjectorFluxPosition(); - InjectorFlux* inj_flux = plasma_injector->getInjectorFlux(); - InjectorMomentum* inj_mom = plasma_injector->getInjectorMomentumDevice(); + InjectorPosition* flux_pos = plasma_injector.getInjectorFluxPosition(); + InjectorFlux* inj_flux = plasma_injector.getInjectorFlux(); + InjectorMomentum* inj_mom = plasma_injector.getInjectorMomentumDevice(); constexpr int level_zero = 0; const amrex::Real t = WarpX::GetInstance().gett_new(level_zero); #ifdef WARPX_DIM_RZ const int nmodes = WarpX::n_rz_azimuthal_modes; const bool rz_random_theta = m_rz_random_theta; - bool radially_weighted = plasma_injector->radially_weighted; + bool radially_weighted = plasma_injector.radially_weighted; #endif MFItInfo info; @@ -1546,30 +1613,30 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) for (int dir=0; dirflux_normal_axis) { + if (dir == plasma_injector.flux_normal_axis) { #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (2*dir == plasma_injector->flux_normal_axis) { + if (2*dir == plasma_injector.flux_normal_axis) { // The above formula captures the following cases: // - flux_normal_axis=0 (emission along x/r) and dir=0 // - flux_normal_axis=2 (emission along z) and dir=1 #elif defined(WARPX_DIM_1D_Z) - if ( (dir==0) && (plasma_injector->flux_normal_axis==2) ) { + if ( (dir==0) && (plasma_injector.flux_normal_axis==2) ) { #endif - if (plasma_injector->flux_direction > 0) { - if (plasma_injector->surface_flux_pos < tile_realbox.lo(dir) || - plasma_injector->surface_flux_pos >= tile_realbox.hi(dir)) { + if (plasma_injector.flux_direction > 0) { + if (plasma_injector.surface_flux_pos < tile_realbox.lo(dir) || + plasma_injector.surface_flux_pos >= tile_realbox.hi(dir)) { no_overlap = true; break; } } else { - if (plasma_injector->surface_flux_pos <= tile_realbox.lo(dir) || - plasma_injector->surface_flux_pos > tile_realbox.hi(dir)) { + if (plasma_injector.surface_flux_pos <= tile_realbox.lo(dir) || + plasma_injector.surface_flux_pos > tile_realbox.hi(dir)) { no_overlap = true; break; } } - overlap_realbox.setLo( dir, plasma_injector->surface_flux_pos ); - overlap_realbox.setHi( dir, plasma_injector->surface_flux_pos ); + overlap_realbox.setLo( dir, plasma_injector.surface_flux_pos ); + overlap_realbox.setHi( dir, plasma_injector.surface_flux_pos ); overlap_box.setSmall( dir, 0 ); overlap_box.setBig( dir, 0 ); shifted[dir] = @@ -1754,7 +1821,7 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) const bool loc_do_field_ionization = do_field_ionization; const int loc_ionization_initial_level = ionization_initial_level; #ifdef WARPX_DIM_RZ - int const loc_flux_normal_axis = plasma_injector->flux_normal_axis; + int const loc_flux_normal_axis = plasma_injector.flux_normal_axis; #endif // Loop over all new particles and inject them (creates too many @@ -2308,7 +2375,7 @@ PhysicalParticleContainer::SplitParticles (int lev) { const auto GetPosition = GetParticlePosition(pti); - const amrex::Vector ppc_nd = plasma_injector->num_particles_per_cell_each_dim; + const amrex::Vector ppc_nd = plasma_injectors[0]->num_particles_per_cell_each_dim; const std::array& dx = WarpX::CellSize(lev); amrex::Vector split_offset = {dx[0]/2._rt, dx[1]/2._rt, @@ -2631,7 +2698,9 @@ PhysicalParticleContainer::ContinuousInjection (const RealBox& injection_box) { // Inject plasma on level 0. Paticles will be redistributed. const int lev=0; - AddPlasma(lev, injection_box); + for (auto const& plasma_injector : plasma_injectors) { + AddPlasma(*plasma_injector, lev, injection_box); + } } /* \brief Inject a flux of particles during the simulation @@ -2639,13 +2708,15 @@ PhysicalParticleContainer::ContinuousInjection (const RealBox& injection_box) void PhysicalParticleContainer::ContinuousFluxInjection (amrex::Real t, amrex::Real dt) { - if (plasma_injector->doFluxInjection()){ - // Check the optional parameters for start and stop of injection - if ( ((plasma_injector->flux_tmin<0) || (t>=plasma_injector->flux_tmin)) && - ((plasma_injector->flux_tmax<0) || (t< plasma_injector->flux_tmax)) ){ + for (auto const& plasma_injector : plasma_injectors) { + if (plasma_injector->doFluxInjection()){ + // Check the optional parameters for start and stop of injection + if ( ((plasma_injector->flux_tmin<0) || (t>=plasma_injector->flux_tmin)) && + ((plasma_injector->flux_tmax<0) || (t< plasma_injector->flux_tmax)) ){ - AddPlasmaFlux(dt); + AddPlasmaFlux(*plasma_injector, dt); + } } } } @@ -2975,9 +3046,13 @@ PhysicalParticleContainer::getIonizationFunc (const WarpXParIter& pti, ion_atomic_number}; } -PlasmaInjector* PhysicalParticleContainer::GetPlasmaInjector () +PlasmaInjector* PhysicalParticleContainer::GetPlasmaInjector (int i) { - return plasma_injector.get(); + if (i < 0 || i >= static_cast(plasma_injectors.size())) { + return nullptr; + } else { + return plasma_injectors[i].get(); + } } void PhysicalParticleContainer::resample (const int timestep, const bool verbose) diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 8ae6e4ef64b..82dcd720179 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -136,7 +136,7 @@ public: * \brief Virtual function that returns a pointer to the plasma injector, * for derived classes that define one (PhysicalParticleContainer). */ - virtual PlasmaInjector* GetPlasmaInjector () { return nullptr; } + virtual PlasmaInjector* GetPlasmaInjector (const int /*i*/) { return nullptr; } /** * Evolve is the central WarpXParticleContainer function that advances diff --git a/Source/Utils/Parser/ParserUtils.H b/Source/Utils/Parser/ParserUtils.H index fb86996bcd9..dbc4845d1bf 100644 --- a/Source/Utils/Parser/ParserUtils.H +++ b/Source/Utils/Parser/ParserUtils.H @@ -63,9 +63,28 @@ namespace utils::parser * \param query_string ParmParse.query will look for this string * \param stored_string variable in which the string to parse is stored */ + void Store_parserString( + amrex::ParmParse const& pp, + std::string const& query_string, + std::string& stored_string); + + /** + * \brief Parse a string (typically a mathematical expression) from the + * input file and store it into a variable. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * get(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] pp used to read the query_string `pp.=string` + * \param[in] group name of the optional group + * \param[in] query_string ParmParse.query will look for this string + * \param[out] stored_string variable in which the string to parse is stored + */ void Store_parserString( const amrex::ParmParse &pp, - std::string query_string, + std::string const& group, + std::string const& query_string, std::string& stored_string); @@ -299,6 +318,208 @@ namespace utils::parser } } + + /** Similar to amrex::ParmParse::query, but also supports math expressions for the value. + * + * amrex::ParmParse::query reads a name and a value from the input file. This function does the + * same, and applies the amrex::Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * queryWithParser(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored, either a scalar or vector + */ + template + int queryWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, T& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return queryWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return queryWithParser(a_pp, grp_str.c_str(), val); + } + } + + + template + int queryArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return queryArrWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return queryArrWithParser(a_pp, grp_str.c_str(), val); + } + } + + + /** Similar to amrex::ParmParse::query, but also supports math expressions for the value. + * + * amrex::ParmParse::query reads a name and a value from the input file. This function does the + * same, and applies the amrex::Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * queryArrWithParser(pp, "group", "name", val, start_ix, num_val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored, either a scalar or vector + * \param[in] start_ix start index in the list of inputs values (optional with arrays, default is + * amrex::ParmParse::FIRST for starting with the first input value) + * \param[in] num_val number of input values to use (optional with arrays, default is + * amrex::ParmParse::LAST for reading until the last input value) + */ + template + int queryArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val, + const int start_ix, const int num_val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return queryArrWithParser(a_pp, str, val, start_ix, num_val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return queryArrWithParser(a_pp, grp_str.c_str(), val, start_ix, num_val); + } + } + + /** Wraps around amrex::ParmParse::query, but also supports an option group name + * + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * get(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + */ + int query (const amrex::ParmParse& a_pp, std::string const& group, char const * str, std::string& val); + + /** Similar to amrex::ParmParse::get, but also supports math expressions for the value. + * + * amrex::ParmParse::get reads a name and a value from the input file. This function does the + * same, and applies the Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * getWithParser(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + */ + template + void getWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, T& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + getWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + getWithParser(a_pp, grp_str.c_str(), val); + } + } + + template + void getArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + getArrWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + getArrWithParser(a_pp, grp_str.c_str(), val); + } + } + + + /** Similar to amrex::ParmParse::get, but also supports math expressions for the value. + * + * amrex::ParmParse::get reads a name and a value from the input file. This function does the + * same, and applies the Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * getArrWithParser(pp, "group", "name", val, start_ix, num_val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + * \param[in] start_ix start index in the list of inputs values (optional with arrays, default is + * amrex::ParmParse::FIRST for starting with the first input value) + * \param[in] num_val number of input values to use (optional with arrays, default is + * amrex::ParmParse::LAST for reading until the last input value) + */ + template + void getArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val, + const int start_ix, const int num_val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + getArrWithParser(a_pp, str, val, start_ix, num_val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + getArrWithParser(a_pp, grp_str.c_str(), val, start_ix, num_val); + } + } + + /** Wraps around amrex::ParmParse::get, but also supports an option group name + * + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * get(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + */ + void get (amrex::ParmParse const& a_pp, std::string const& group, char const * str, std::string& val); + } #endif // WARPX_UTILS_PARSER_PARSERUTILS_H_ diff --git a/Source/Utils/Parser/ParserUtils.cpp b/Source/Utils/Parser/ParserUtils.cpp index aeacedc5b4f..6d430d4a619 100644 --- a/Source/Utils/Parser/ParserUtils.cpp +++ b/Source/Utils/Parser/ParserUtils.cpp @@ -19,8 +19,8 @@ #include void utils::parser::Store_parserString( - const amrex::ParmParse& pp, - std::string query_string, + amrex::ParmParse const& pp, + std::string const& query_string, std::string& stored_string) { std::vector f; @@ -32,6 +32,54 @@ void utils::parser::Store_parserString( f.clear(); } +void utils::parser::Store_parserString( + amrex::ParmParse const& a_pp, + std::string const& group, + std::string const& query_string, + std::string& stored_string) +{ + const bool is_specified_without_group = a_pp.contains(query_string.c_str()); + const std::string grp_str = group + "." + query_string; + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + utils::parser::Store_parserString(a_pp, query_string, stored_string); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + utils::parser::Store_parserString(a_pp, grp_str, stored_string); + } +} + +int utils::parser::query (const amrex::ParmParse& a_pp, std::string const& group, char const * str, std::string& val) +{ + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return a_pp.query(str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return a_pp.query(grp_str.c_str(), val); + } +} + +void utils::parser::get (const amrex::ParmParse& a_pp, std::string const& group, char const * str, std::string& val) +{ + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + a_pp.get(str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + a_pp.get(grp_str.c_str(), val); + } +} namespace { template< typename int_type > diff --git a/Source/Utils/SpeciesUtils.H b/Source/Utils/SpeciesUtils.H index f95525ecb4c..3f5d42910c4 100644 --- a/Source/Utils/SpeciesUtils.H +++ b/Source/Utils/SpeciesUtils.H @@ -21,11 +21,11 @@ namespace SpeciesUtils { std::string const& injection_style, amrex::Real& charge, amrex::Real& mass, PhysicalSpecies& physical_species); - void parseDensity (std::string const& species_name, + void parseDensity (std::string const& species_name, std::string const& source_name, std::unique_ptr& h_inj_rho, std::unique_ptr& density_parser); - void parseMomentum (std::string const& species_name, const std::string& style, + void parseMomentum (std::string const& species_name, std::string const& source_name, const std::string& style, std::unique_ptr& h_inj_mom, std::unique_ptr& ux_parser, std::unique_ptr& uy_parser, diff --git a/Source/Utils/SpeciesUtils.cpp b/Source/Utils/SpeciesUtils.cpp index 50729188e33..f8515592a07 100644 --- a/Source/Utils/SpeciesUtils.cpp +++ b/Source/Utils/SpeciesUtils.cpp @@ -77,20 +77,20 @@ namespace SpeciesUtils { // Depending on injection type at runtime, initialize inj_rho // so that inj_rho->getDensity calls // InjectorPosition[Constant or Predefined or etc.].getDensity. - void parseDensity (std::string const& species_name, + void parseDensity (std::string const& species_name, std::string const& source_name, std::unique_ptr& h_inj_rho, std::unique_ptr& density_parser) { - const amrex::ParmParse pp_species_name(species_name); + amrex::ParmParse pp_species(species_name); // parse density information std::string rho_prof_s; - pp_species_name.get("profile", rho_prof_s); + utils::parser::get(pp_species, source_name, "profile", rho_prof_s); std::transform(rho_prof_s.begin(), rho_prof_s.end(), rho_prof_s.begin(), ::tolower); if (rho_prof_s == "constant") { amrex::Real density; - utils::parser::getWithParser(pp_species_name, "density", density); + utils::parser::getWithParser(pp_species, source_name, "density", density); // Construct InjectorDensity with InjectorDensityConstant. h_inj_rho.reset(new InjectorDensity((InjectorDensityConstant*)nullptr, density)); } else if (rho_prof_s == "predefined") { @@ -98,8 +98,7 @@ namespace SpeciesUtils { h_inj_rho.reset(new InjectorDensity((InjectorDensityPredefined*)nullptr,species_name)); } else if (rho_prof_s == "parse_density_function") { std::string str_density_function; - utils::parser::Store_parserString( - pp_species_name, "density_function(x,y,z)", str_density_function); + utils::parser::Store_parserString(pp_species, source_name, "density_function(x,y,z)", str_density_function); // Construct InjectorDensity with InjectorDensityParser. density_parser = std::make_unique( utils::parser::makeParser(str_density_function,{"x","y","z"})); @@ -113,7 +112,7 @@ namespace SpeciesUtils { // Depending on injection type at runtime, initialize inj_mom // so that inj_mom->getMomentum calls // InjectorMomentum[Constant or Gaussian or etc.].getMomentum. - void parseMomentum (std::string const& species_name, const std::string& style, + void parseMomentum (std::string const& species_name, std::string const& source_name, const std::string& style, std::unique_ptr& h_inj_mom, std::unique_ptr& ux_parser, std::unique_ptr& uy_parser, @@ -127,15 +126,15 @@ namespace SpeciesUtils { { using namespace amrex::literals; - const amrex::ParmParse pp_species_name(species_name); + amrex::ParmParse pp_species(species_name); // parse momentum information std::string mom_dist_s; - pp_species_name.get("momentum_distribution_type", mom_dist_s); + utils::parser::get(pp_species, source_name, "momentum_distribution_type", mom_dist_s); std::transform(mom_dist_s.begin(), - mom_dist_s.end(), - mom_dist_s.begin(), - ::tolower); + mom_dist_s.end(), + mom_dist_s.begin(), + ::tolower); if (mom_dist_s == "at_rest") { constexpr amrex::Real ux = 0._rt; constexpr amrex::Real uy = 0._rt; @@ -146,9 +145,9 @@ namespace SpeciesUtils { amrex::Real ux = 0._rt; amrex::Real uy = 0._rt; amrex::Real uz = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux", ux); - utils::parser::queryWithParser(pp_species_name, "uy", uy); - utils::parser::queryWithParser(pp_species_name, "uz", uz); + utils::parser::queryWithParser(pp_species, source_name, "ux", ux); + utils::parser::queryWithParser(pp_species, source_name, "uy", uy); + utils::parser::queryWithParser(pp_species, source_name, "uz", uz); // Construct InjectorMomentum with InjectorMomentumConstant. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumConstant*)nullptr, ux, uy, uz)); } else if (mom_dist_s == "gaussian") { @@ -158,12 +157,12 @@ namespace SpeciesUtils { amrex::Real ux_th = 0._rt; amrex::Real uy_th = 0._rt; amrex::Real uz_th = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_m", ux_m); - utils::parser::queryWithParser(pp_species_name, "uy_m", uy_m); - utils::parser::queryWithParser(pp_species_name, "uz_m", uz_m); - utils::parser::queryWithParser(pp_species_name, "ux_th", ux_th); - utils::parser::queryWithParser(pp_species_name, "uy_th", uy_th); - utils::parser::queryWithParser(pp_species_name, "uz_th", uz_th); + utils::parser::queryWithParser(pp_species, source_name, "ux_m", ux_m); + utils::parser::queryWithParser(pp_species, source_name, "uy_m", uy_m); + utils::parser::queryWithParser(pp_species, source_name, "uz_m", uz_m); + utils::parser::queryWithParser(pp_species, source_name, "ux_th", ux_th); + utils::parser::queryWithParser(pp_species, source_name, "uy_th", uy_th); + utils::parser::queryWithParser(pp_species, source_name, "uz_th", uz_th); // Construct InjectorMomentum with InjectorMomentumGaussian. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussian*)nullptr, ux_m, uy_m, uz_m, ux_th, uy_th, uz_th)); @@ -176,12 +175,12 @@ namespace SpeciesUtils { amrex::Real ux_th = 0._rt; amrex::Real uy_th = 0._rt; amrex::Real uz_th = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_m", ux_m); - utils::parser::queryWithParser(pp_species_name, "uy_m", uy_m); - utils::parser::queryWithParser(pp_species_name, "uz_m", uz_m); - utils::parser::queryWithParser(pp_species_name, "ux_th", ux_th); - utils::parser::queryWithParser(pp_species_name, "uy_th", uy_th); - utils::parser::queryWithParser(pp_species_name, "uz_th", uz_th); + utils::parser::queryWithParser(pp_species, source_name, "ux_m", ux_m); + utils::parser::queryWithParser(pp_species, source_name, "uy_m", uy_m); + utils::parser::queryWithParser(pp_species, source_name, "uz_m", uz_m); + utils::parser::queryWithParser(pp_species, source_name, "ux_th", ux_th); + utils::parser::queryWithParser(pp_species, source_name, "uy_th", uy_th); + utils::parser::queryWithParser(pp_species, source_name, "uz_th", uz_th); // Construct InjectorMomentum with InjectorMomentumGaussianFlux. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussianFlux*)nullptr, ux_m, uy_m, uz_m, ux_th, uy_th, uz_th, @@ -193,32 +192,32 @@ namespace SpeciesUtils { amrex::Real ux_max = 0._rt; amrex::Real uy_max = 0._rt; amrex::Real uz_max = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_min", ux_min); - utils::parser::queryWithParser(pp_species_name, "uy_min", uy_min); - utils::parser::queryWithParser(pp_species_name, "uz_min", uz_min); - utils::parser::queryWithParser(pp_species_name, "ux_max", ux_max); - utils::parser::queryWithParser(pp_species_name, "uy_max", uy_max); - utils::parser::queryWithParser(pp_species_name, "uz_max", uz_max); + utils::parser::queryWithParser(pp_species, source_name, "ux_min", ux_min); + utils::parser::queryWithParser(pp_species, source_name, "uy_min", uy_min); + utils::parser::queryWithParser(pp_species, source_name, "uz_min", uz_min); + utils::parser::queryWithParser(pp_species, source_name, "ux_max", ux_max); + utils::parser::queryWithParser(pp_species, source_name, "uy_max", uy_max); + utils::parser::queryWithParser(pp_species, source_name, "uz_max", uz_max); // Construct InjectorMomentum with InjectorMomentumUniform. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumUniform*)nullptr, ux_min, uy_min, uz_min, ux_max, uy_max, uz_max)); } else if (mom_dist_s == "maxwell_boltzmann"){ - h_mom_temp = std::make_unique(pp_species_name); + h_mom_temp = std::make_unique(pp_species, source_name); const GetTemperature getTemp(*h_mom_temp); - h_mom_vel = std::make_unique(pp_species_name); + h_mom_vel = std::make_unique(pp_species, source_name); const GetVelocity getVel(*h_mom_vel); // Construct InjectorMomentum with InjectorMomentumBoltzmann. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumBoltzmann*)nullptr, getTemp, getVel)); } else if (mom_dist_s == "maxwell_juttner"){ - h_mom_temp = std::make_unique(pp_species_name); + h_mom_temp = std::make_unique(pp_species, source_name); const GetTemperature getTemp(*h_mom_temp); - h_mom_vel = std::make_unique(pp_species_name); + h_mom_vel = std::make_unique(pp_species, source_name); const GetVelocity getVel(*h_mom_vel); // Construct InjectorMomentum with InjectorMomentumJuttner. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumJuttner*)nullptr, getTemp, getVel)); } else if (mom_dist_s == "radial_expansion") { amrex::Real u_over_r = 0._rt; - utils::parser::queryWithParser(pp_species_name, "u_over_r", u_over_r); + utils::parser::queryWithParser(pp_species, source_name, "u_over_r", u_over_r); // Construct InjectorMomentum with InjectorMomentumRadialExpansion. h_inj_mom.reset(new InjectorMomentum ((InjectorMomentumRadialExpansion*)nullptr, u_over_r)); @@ -226,12 +225,9 @@ namespace SpeciesUtils { std::string str_momentum_function_ux; std::string str_momentum_function_uy; std::string str_momentum_function_uz; - utils::parser::Store_parserString(pp_species_name, "momentum_function_ux(x,y,z)", - str_momentum_function_ux); - utils::parser::Store_parserString(pp_species_name, "momentum_function_uy(x,y,z)", - str_momentum_function_uy); - utils::parser::Store_parserString(pp_species_name, "momentum_function_uz(x,y,z)", - str_momentum_function_uz); + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_ux(x,y,z)", str_momentum_function_ux); + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uy(x,y,z)", str_momentum_function_uy); + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uz(x,y,z)", str_momentum_function_uz); // Construct InjectorMomentum with InjectorMomentumParser. ux_parser = std::make_unique( utils::parser::makeParser(str_momentum_function_ux, {"x","y","z"})); @@ -250,17 +246,17 @@ namespace SpeciesUtils { std::string str_momentum_function_ux_th; std::string str_momentum_function_uy_th; std::string str_momentum_function_uz_th; - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_ux_m(x,y,z)", str_momentum_function_ux_m); - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uy_m(x,y,z)", str_momentum_function_uy_m); - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uz_m(x,y,z)", str_momentum_function_uz_m); - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_ux_th(x,y,z)", str_momentum_function_ux_th); - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uy_th(x,y,z)", str_momentum_function_uy_th); - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uz_th(x,y,z)", str_momentum_function_uz_th); // Construct InjectorMomentum with InjectorMomentumParser. ux_parser = std::make_unique( diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 60b31af2f14..42a97354248 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -81,7 +81,8 @@ WarpX::UpdateInjectionPosition (const amrex::Real a_dt) current_injection_position[dir] = pc.m_current_injection_position; #endif - PlasmaInjector* plasma_injector = pc.GetPlasmaInjector(); + // This only uses the base plasma injector + PlasmaInjector* plasma_injector = pc.GetPlasmaInjector(0); amrex::Real v_shift = 0._rt; if (plasma_injector != nullptr) From aee8dc0eb0bdf6c78ffd8160ebd2d6988ec25f8a Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Wed, 29 Nov 2023 13:07:44 -0800 Subject: [PATCH 004/176] Clang-Tidy Cache (#4453) Use ccache in the clang-tidy CI. This allows us to skip clang-tidy checks on files that are not changed as reported by ccache. --- .github/workflows/clang_tidy.yml | 44 +++++++------ .../source/makeMakefileForClangTidy.py | 65 +++++++++++++++++++ 2 files changed, 89 insertions(+), 20 deletions(-) create mode 100755 .github/workflows/source/makeMakefileForClangTidy.py diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index adfb827e4d8..dd6d79d1910 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -16,28 +16,26 @@ jobs: - name: install dependencies run: | .github/workflows/dependencies/clang14.sh - - name: build WarpX using clang-tidy + - name: set up cache + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-clang-tidy-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-clang-tidy-${{ github.job }}-git- + - name: build WarpX & run clang-tidy run: | - + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=300M + export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy + export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt + ccache -z export CXX=$(which clang++) export CC=$(which clang) - # The following wrapper ensures that only source files - # in WarpX/Source/* are actually processed by clang-tidy - #_______________________________ - cat > clang_tidy_wrapper << EOF - #!/bin/bash - REGEX="[a-z_A-Z0-9\/]*WarpX\/Source[a-z_A-Z0-9\/]+.cpp" - if [[ \$4 =~ \$REGEX ]];then - clang-tidy \$@ - fi - EOF - chmod +x clang_tidy_wrapper - #_____________________________________ - cmake -S . -B build_clang_tidy \ - -DCMAKE_CXX_CLANG_TIDY="$PWD/clang_tidy_wrapper;--system-headers=0;--config-file=$PWD/.clang-tidy" \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="1;2;3;RZ" \ -DWarpX_MPI=ON \ @@ -46,9 +44,15 @@ jobs: -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ - -DWarpX_PRECISION=SINGLE + -DWarpX_PRECISION=SINGLE \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + + cmake --build build_clang_tidy -j 2 - cmake --build build_clang_tidy -j 2 2> build_clang_tidy/clang-tidy.log + ${{github.workspace}}/.github/workflows/source/makeMakefileForClangTidy.py --input ${{github.workspace}}/ccache.log.txt + make -j2 -f clang-tidy-ccache-misses.mak \ + CLANG_TIDY=clang-tidy \ + CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*" - cat build_clang_tidy/clang-tidy.log - if [[ $(wc -m Date: Wed, 29 Nov 2023 19:16:43 -0800 Subject: [PATCH 005/176] AMReX: Update to `development` (#4455) ``` ./Tools/Release/updateAMReX.py ``` --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- run_test.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index aab28b74b80..9dc23468535 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -108,7 +108,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 9e35dc19489dc5d312e92781cb0471d282cf8370 && cd - + cd ../amrex && git checkout --detach 9b733ec45cd93a80234a7c98248b6eb4816589d5 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 build_nvhpc21-11-nvcc: diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index e11976205d1..f16c0f74e9f 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 9e35dc19489dc5d312e92781cb0471d282cf8370 +branch = 9b733ec45cd93a80234a7c98248b6eb4816589d5 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index c8e9a7e1a90..dbc3421d8b0 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 9e35dc19489dc5d312e92781cb0471d282cf8370 +branch = 9b733ec45cd93a80234a7c98248b6eb4816589d5 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index d382cf97094..f3866efef5b 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "9e35dc19489dc5d312e92781cb0471d282cf8370" +set(WarpX_amrex_branch "9b733ec45cd93a80234a7c98248b6eb4816589d5" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/run_test.sh b/run_test.sh index 485e07d4e2f..41fc96b0b05 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 9e35dc19489dc5d312e92781cb0471d282cf8370 && cd - +cd amrex && git checkout --detach 9b733ec45cd93a80234a7c98248b6eb4816589d5 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 4534db962527fe643b3c668f0ed7044c3b40b749 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 1 Dec 2023 09:11:32 -0800 Subject: [PATCH 006/176] Set `amrex.omp_threads = "nosmt"` (#4393) Improve the performance on Intel and AMD CPUs by default: Avoid oversubscribing the physical number of CPU threads, as is default in OpenMP. --- Docs/source/usage/parameters.rst | 6 ++++++ Source/Initialization/WarpXAMReXInit.cpp | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index a52a4c1d5e8..118fada45d9 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -167,6 +167,12 @@ Overall simulation parameters For all regular WarpX operations, we therefore do explicit memory transfers without the need for managed memory and thus changed the AMReX default to false. `Please also see the documentation in AMReX `__. +* ``amrex.omp_threads`` (``system``, ``nosmt`` or positive integer; default is ``nosmt``) + An integer number can be set in lieu of the ``OMP_NUM_THREADS`` environment variable to control the number of OpenMP threads to use for the ``OMP`` compute backend on CPUs. + By default, we use the ``nosmt`` option, which overwrites the OpenMP default of spawning one thread per logical CPU core, and instead only spawns a number of threads equal to the number of physical CPU cores on the machine. + If set, the environment variable ``OMP_NUM_THREADS`` takes precedence over ``system`` and ``nosmt``, but not over integer numbers set in this option. + + Signal Handling ^^^^^^^^^^^^^^^ diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index 8502bdc4ae6..c153595604f 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -29,6 +29,10 @@ namespace { bool the_arena_is_managed = false; // AMReX' default: true pp_amrex.queryAdd("the_arena_is_managed", the_arena_is_managed); + // https://amrex-codes.github.io/amrex/docs_html/InputsComputeBackends.html + std::string omp_threads = "nosmt"; // AMReX' default: system + pp_amrex.queryAdd("omp_threads", omp_threads); + // Work-around: // If warpx.numprocs is used for the domain decomposition, we will not use blocking factor // to generate grids. Nonetheless, AMReX has asserts in place that validate that the From 008dc7193761e1fda41cf5ff42960e92605d6764 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Fri, 1 Dec 2023 11:23:47 -0600 Subject: [PATCH 007/176] Initial commit (#4460) --- Source/Diagnostics/BTDiagnostics.H | 3 +-- Source/Diagnostics/BoundaryScrapingDiagnostics.H | 3 +-- Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H | 3 +-- .../ComputeDiagFunctors/BackTransformParticleFunctor.H | 3 +-- Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H | 3 +-- Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H | 3 +-- .../ComputeDiagFunctors/ComputeParticleDiagFunctor.H | 3 +-- Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H | 3 +-- Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H | 3 +-- Source/Diagnostics/ComputeDiagFunctors/JFunctor.H | 3 +-- Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H | 3 +-- Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H | 3 +-- .../Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H | 3 +-- Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H | 3 +-- Source/Diagnostics/FullDiagnostics.H | 3 +-- .../MacroscopicProperties/MacroscopicProperties.H | 3 +-- 16 files changed, 16 insertions(+), 32 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index 4198846fa5b..f72e7aa8b07 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -24,8 +24,7 @@ #include #include -class -BTDiagnostics final : public Diagnostics +class BTDiagnostics final : public Diagnostics { public: diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.H b/Source/Diagnostics/BoundaryScrapingDiagnostics.H index 097835297dd..60d184c30e2 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.H +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.H @@ -14,8 +14,7 @@ /** collect the particles that are absorbed at the embedded boundary, throughout the simulation */ -class -BoundaryScrapingDiagnostics final : public Diagnostics +class BoundaryScrapingDiagnostics final : public Diagnostics { public: diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H index ac361db98fa..ef61d96f0aa 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H @@ -29,8 +29,7 @@ * lab-frame field data is then stored in mf_dst. */ -class -BackTransformFunctor final : public ComputeDiagFunctor +class BackTransformFunctor final : public ComputeDiagFunctor { public: /** Constructor description diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H index afc3e6e8511..dc14d8a38c8 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H @@ -195,8 +195,7 @@ struct LorentzTransformParticles * \brief BackTransform functor to select particles and Lorentz Transform them * and store in particle buffers */ -class -BackTransformParticleFunctor final : public ComputeParticleDiagFunctor +class BackTransformParticleFunctor final : public ComputeParticleDiagFunctor { public: BackTransformParticleFunctor(WarpXParticleContainer *pc_src, std::string species_name, int num_buffers); diff --git a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H index 58823d7bb17..dd5bb239ecf 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF and store result in mf_out. */ -class -CellCenterFunctor final : public ComputeDiagFunctor +class CellCenterFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H index 68f9c38bc8e..c254a341122 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H @@ -13,8 +13,7 @@ * \brief Functor to compute a diagnostic and store the result in existing * MultiFab */ -class -ComputeDiagFunctor +class ComputeDiagFunctor { public: ComputeDiagFunctor( int ncomp, amrex::IntVect crse_ratio) : diff --git a/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H index 3309dbed584..e5f8c22b95c 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H @@ -15,8 +15,7 @@ /** * \brief Functor to compute a diagnostic and store the result in existing ParticleContainer. */ -class -ComputeParticleDiagFunctor +class ComputeParticleDiagFunctor { public: diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H index ebf890f6545..3d04a56742b 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H @@ -10,8 +10,7 @@ /** * \brief Functor to compute divB into mf_out. */ -class -DivBFunctor final : public ComputeDiagFunctor +class DivBFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H index 76e5270b972..312ccaa5cd6 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H @@ -10,8 +10,7 @@ /** * \brief Functor to compute divE into mf_out. */ -class -DivEFunctor final : public ComputeDiagFunctor +class DivEFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H index c42f63f1369..3080b500712 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF for current density and store result in mf_out. */ -class -JFunctor final : public ComputeDiagFunctor +class JFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H index af218d2667b..1b8785af7b7 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF and store result in mf_out. */ -class -PartPerCellFunctor final : public ComputeDiagFunctor +class PartPerCellFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H index b4d64117dd7..9718c9c7163 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF and store result in mf_out. */ -class -PartPerGridFunctor final : public ComputeDiagFunctor +class PartPerGridFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H index 12a17ff13af..56c4372fad8 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H @@ -12,8 +12,7 @@ /** * \brief Functor to calculate per-cell averages of particle properties. */ -class -ParticleReductionFunctor final : public ComputeDiagFunctor +class ParticleReductionFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H index 46f8a3be251..31641fe23e9 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to compute charge density rho into mf_out */ -class -RhoFunctor final : public ComputeDiagFunctor +class RhoFunctor final : public ComputeDiagFunctor { public: diff --git a/Source/Diagnostics/FullDiagnostics.H b/Source/Diagnostics/FullDiagnostics.H index 4464d46ebac..42b60c213f2 100644 --- a/Source/Diagnostics/FullDiagnostics.H +++ b/Source/Diagnostics/FullDiagnostics.H @@ -6,8 +6,7 @@ #include -class -FullDiagnostics final : public Diagnostics +class FullDiagnostics final : public Diagnostics { public: FullDiagnostics (int i, std::string name); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H index b26a918c9ae..610a17b6998 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H @@ -27,8 +27,7 @@ * \brief This class contains the macroscopic properties of the medium needed to * evaluate macroscopic Maxwell equation. */ -class -MacroscopicProperties +class MacroscopicProperties { public: MacroscopicProperties (); // constructor From d4baf948d59cbfad3b1c0898e37149035dc00a79 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Fri, 1 Dec 2023 09:29:19 -0800 Subject: [PATCH 008/176] New Cache Strategy (#4456) If a cache is updated during a job, GitHub does not store the new cache unless it's not found with the primary key. To work around that, we use the git hash in the cache name. When the cache action tries to load the cache using the primary key, it's always a miss. It will then try to load a cache using the restore-keys. If the cache is updated during CI, the new version will be stored under the primary key's name. In this new strategy, a new PR will use the cache associated with the latest development branch. The updated cache will be stored and become private to that PR. Any updates to the PR will also create new updated caches. But we don't need to keep all the copies. We only need to keep the latest version of the cache associated with the PR. So we use a workflow (clean-cache.yml) to clean up the old caches. After a PR is closed, we use clean-cache-postpr.yml to clean up all the caches associated with the PR. There is no easy way to get the PR number of the PR that triggers the cleanup workflow. Thus, we store the PR number as an artifact for the cleanup workflow. It turns out ccache encounters internal errors in some CIs on Ubuntu 20.04. Switching to the latest ccache solves the issues. --- .github/workflows/clang_tidy.yml | 19 ++- .github/workflows/cleanup-cache-postpr.yml | 40 +++++++ .github/workflows/cleanup-cache.yml | 63 ++++++++++ .github/workflows/codeql.yml | 39 ++++++- .github/workflows/cuda.yml | 76 ++++++++---- .github/workflows/dependencies/ccache.sh | 13 +++ .github/workflows/dependencies/clang14.sh | 4 +- .github/workflows/dependencies/dpcpp.sh | 4 +- .github/workflows/dependencies/gcc.sh | 4 +- .github/workflows/dependencies/gcc12.sh | 4 +- .../dependencies/gcc12_blaspp_lapackpp.sh | 4 +- .github/workflows/dependencies/hip.sh | 12 +- .github/workflows/dependencies/icc.sh | 4 +- .github/workflows/dependencies/nvcc11-3.sh | 12 +- .github/workflows/dependencies/nvcc11-8.sh | 12 +- .github/workflows/dependencies/nvhpc.sh | 12 +- .github/workflows/dependencies/pyfull.sh | 4 +- .github/workflows/hip.yml | 53 ++++++--- .github/workflows/intel.yml | 75 ++++++++---- .github/workflows/macos.yml | 29 ++++- .github/workflows/post-pr.yml | 20 ++++ .github/workflows/ubuntu.yml | 109 +++++++++++------- 22 files changed, 457 insertions(+), 155 deletions(-) create mode 100644 .github/workflows/cleanup-cache-postpr.yml create mode 100644 .github/workflows/cleanup-cache.yml create mode 100755 .github/workflows/dependencies/ccache.sh create mode 100644 .github/workflows/post-pr.yml diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index dd6d79d1910..c0d039b7f17 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -20,9 +20,9 @@ jobs: uses: actions/cache@v3 with: path: ~/.cache/ccache - key: ccache-clang-tidy-${{ github.job }}-git-${{ github.sha }} + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-clang-tidy-${{ github.job }}-git- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX & run clang-tidy run: | export CCACHE_COMPRESS=1 @@ -56,3 +56,18 @@ jobs: ccache -s du -hs ~/.cache/ccache + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml new file mode 100644 index 00000000000..978e9c28f04 --- /dev/null +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -0,0 +1,40 @@ +name: CleanUpCachePostPR + +on: + workflow_run: + workflows: [PostPR] + types: + - completed + +jobs: + CleanUpCcacheCachePostPR: + name: Clean Up Ccache Cache Post PR + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Clean up ccache + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + + gh run download ${{ github.event.workflow_run.id }} -n pr_number + pr_number=`cat pr_number.txt` + BRANCH=refs/pull/${pr_number}/merge + + # Setting this to not fail the workflow while deleting cache keys. + set +e + + keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1) + # $keys might contain spaces. Thus we set IFS to \n. + IFS=$'\n' + for k in $keys + do + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm + done + unset IFS diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml new file mode 100644 index 00000000000..6421bbf4215 --- /dev/null +++ b/.github/workflows/cleanup-cache.yml @@ -0,0 +1,63 @@ +name: CleanUpCache + +on: + workflow_run: + workflows: [🧹 clang-tidy, 🔍 CodeQL, 🐧 CUDA, 🐧 HIP, 🐧 Intel, 🍏 macOS, 🐧 OpenMP] + types: + - completed + +jobs: + CleanUpCcacheCache: + name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }} + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + - name: Clean up ccache + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + + # push or pull_request or schedule or ... + EVENT=${{ github.event.workflow_run.event }} + + # Triggering workflow run name (e.g., LinuxClang) + WORKFLOW_NAME="${{ github.event.workflow_run.name }}" + + if [[ $EVENT == "pull_request" ]]; then + gh run download ${{ github.event.workflow_run.id }} -n pr_number + pr_number=`cat pr_number.txt` + BRANCH=refs/pull/${pr_number}/merge + else + BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }} + fi + + # Setting this to not fail the workflow while deleting cache keys. + set +e + + # In our cache keys, substring after `-git-` is git hash, substring + # before that is a unique id for jobs (e.g., `ccache-LinuxClang-configure-2d`). + # The goal is to keep the last used key of each job and delete all others. + + # something like ccache-LinuxClang- + keyprefix="ccache-${WORKFLOW_NAME}-" + + cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq) + + # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d". + # It might also contain spaces. Thus we set IFS to \n. + IFS=$'\n' + for j in $cached_jobs + do + old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2) + for k in $old_keys + do + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm + done + done + unset IFS diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 55d0e473d27..436df798d3b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -35,13 +35,22 @@ jobs: if: ${{ matrix.language == 'cpp' }} run: | sudo apt-get update - sudo apt-get install --yes cmake openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libadios-openmpi-dev + sudo apt-get install --yes cmake openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libadios-openmpi-dev ccache python -m pip install --upgrade pip python -m pip install --upgrade wheel python -m pip install --upgrade cmake export CMAKE="$HOME/.local/bin/cmake" && echo "CMAKE=$CMAKE" >> $GITHUB_ENV + - name: Set Up Cache + if: ${{ matrix.language == 'cpp' }} + uses: actions/cache@v3 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: Configure (C++) if: ${{ matrix.language == 'cpp' }} run: | @@ -61,6 +70,19 @@ jobs: - name: Build (C++) if: ${{ matrix.language == 'cpp' }} run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + $CMAKE --build build -j 2 + + ccache -s + du -hs ~/.cache/ccache + + # Make sure CodeQL has something to do + touch Source/Utils/WarpXVersion.cpp + export CCACHE_DISABLE=1 $CMAKE --build build -j 2 - name: Perform CodeQL Analysis @@ -88,3 +110,18 @@ jobs: uses: github/codeql-action/upload-sarif@v2 with: sarif_file: sarif-results/${{ matrix.language }}.sarif + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 9dc23468535..3ad7978fb2b 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -29,18 +29,18 @@ jobs: .github/workflows/dependencies/nvcc11-3.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-cuda-nvcc-${{ hashFiles('.github/workflows/cuda.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-cuda-nvcc-${{ hashFiles('.github/workflows/cuda.yml') }}- - ccache-cuda-nvcc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: install openPMD-api run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" cmake-easyinstall --prefix=/usr/local \ @@ -53,6 +53,10 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} which nvcc || echo "nvcc not in PATH!" @@ -78,6 +82,9 @@ jobs: python3 -m pip wheel . python3 -m pip install *.whl + ccache -s + du -hs ~/.cache/ccache + # make sure legacy build system continues to build, i.e., that we don't forget # to add new .cpp files build_nvcc_gnumake: @@ -91,18 +98,18 @@ jobs: .github/workflows/dependencies/nvcc11-8.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-cuda-gnumake-${{ hashFiles('.github/workflows/cuda.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-cuda-gnumake-${{ hashFiles('.github/workflows/cuda.yml') }}- - ccache-cuda-gnumake- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} which nvcc || echo "nvcc not in PATH!" @@ -111,6 +118,9 @@ jobs: cd ../amrex && git checkout --detach 9b733ec45cd93a80234a7c98248b6eb4816589d5 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 + ccache -s + du -hs ~/.cache/ccache + build_nvhpc21-11-nvcc: name: NVHPC@21.11 NVCC/NVC++ Release [tests] runs-on: ubuntu-20.04 @@ -124,18 +134,18 @@ jobs: run: .github/workflows/dependencies/nvhpc.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-cuda-nvhpc-${{ hashFiles('.github/workflows/cuda.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-cuda-nvhpc-${{ hashFiles('.github/workflows/cuda.yml') }}- - ccache-cuda-nvhpc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + source /etc/profile.d/modules.sh module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.11 which nvcc || echo "nvcc not in PATH!" @@ -174,3 +184,21 @@ jobs: #export PYWARPX_LIB_DIR=$PWD/build/lib/site-packages/pywarpx/ #python3 -m pip wheel . #python3 -m pip install *.whl + + ccache -s + du -hs ~/.cache/ccache + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/dependencies/ccache.sh b/.github/workflows/dependencies/ccache.sh new file mode 100755 index 00000000000..30a155ff20c --- /dev/null +++ b/.github/workflows/dependencies/ccache.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +if [[ $# -eq 2 ]]; then + CVER=$1 +else + CVER=4.8.3 +fi + +wget https://github.com/ccache/ccache/releases/download/v${CVER}/ccache-${CVER}-linux-x86_64.tar.xz +tar xvf ccache-${CVER}-linux-x86_64.tar.xz +sudo cp -f ccache-${CVER}-linux-x86_64/ccache /usr/local/bin/ diff --git a/.github/workflows/dependencies/clang14.sh b/.github/workflows/dependencies/clang14.sh index 18274d5e46c..27597e06c5d 100755 --- a/.github/workflows/dependencies/clang14.sh +++ b/.github/workflows/dependencies/clang14.sh @@ -15,7 +15,6 @@ echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries sudo apt-get -qqq update sudo apt-get install -y \ cmake \ - ccache \ clang-14 \ clang-tidy-14 \ libblas-dev \ @@ -29,6 +28,9 @@ sudo apt-get install -y \ libomp-dev \ ninja-build +# ccache +$(dirname "$0")/ccache.sh + # cmake-easyinstall # sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall diff --git a/.github/workflows/dependencies/dpcpp.sh b/.github/workflows/dependencies/dpcpp.sh index 9ecc5e4ca19..3b146405b4b 100755 --- a/.github/workflows/dependencies/dpcpp.sh +++ b/.github/workflows/dependencies/dpcpp.sh @@ -34,7 +34,6 @@ for itry in {1..5} do sudo apt-get install -y --no-install-recommends \ build-essential \ - ccache \ cmake \ intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel \ g++ gfortran \ @@ -58,3 +57,6 @@ sudo rm -rf /opt/intel/oneapi/mkl/latest/lib/intel64/*.a \ du -sh /opt/intel/oneapi/ du -sh /opt/intel/oneapi/*/* df -h + +# ccache +$(dirname "$0")/ccache.sh diff --git a/.github/workflows/dependencies/gcc.sh b/.github/workflows/dependencies/gcc.sh index d845463a682..fb2351f8a92 100755 --- a/.github/workflows/dependencies/gcc.sh +++ b/.github/workflows/dependencies/gcc.sh @@ -16,9 +16,11 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ gnupg \ ninja-build \ pkg-config \ wget + +# ccache +$(dirname "$0")/ccache.sh diff --git a/.github/workflows/dependencies/gcc12.sh b/.github/workflows/dependencies/gcc12.sh index 01f30f07a31..fb46c6b811b 100755 --- a/.github/workflows/dependencies/gcc12.sh +++ b/.github/workflows/dependencies/gcc12.sh @@ -16,7 +16,6 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ g++-12 \ gnupg \ @@ -28,3 +27,6 @@ sudo apt-get install -y \ ninja-build \ pkg-config \ wget + +# ccache +$(dirname "$0")/ccache.sh diff --git a/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh b/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh index 65f029ea67b..9ff3d615a3e 100755 --- a/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh +++ b/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh @@ -16,7 +16,6 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ g++-12 \ gnupg \ @@ -30,6 +29,9 @@ sudo apt-get install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + # cmake-easyinstall # sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index 2e7830368ca..9d0206efb76 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -43,6 +43,9 @@ sudo apt-get install -y --no-install-recommends \ rocprim-dev \ rocrand-dev +# ccache +$(dirname "$0")/ccache.sh + # activate # source /etc/profile.d/rocm.sh @@ -56,12 +59,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/icc.sh b/.github/workflows/dependencies/icc.sh index 5242a836a42..74f0db4b46a 100755 --- a/.github/workflows/dependencies/icc.sh +++ b/.github/workflows/dependencies/icc.sh @@ -17,12 +17,14 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ gnupg \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + # Ref.: https://github.com/rscohn2/oneapi-ci sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB diff --git a/.github/workflows/dependencies/nvcc11-3.sh b/.github/workflows/dependencies/nvcc11-3.sh index ed9fd211128..92e2717e425 100755 --- a/.github/workflows/dependencies/nvcc11-3.sh +++ b/.github/workflows/dependencies/nvcc11-3.sh @@ -26,6 +26,9 @@ sudo apt-get install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb sudo dpkg -i cuda-keyring_1.0-1_all.deb @@ -55,12 +58,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/nvcc11-8.sh b/.github/workflows/dependencies/nvcc11-8.sh index 33af2c1ade5..6089360392b 100755 --- a/.github/workflows/dependencies/nvcc11-8.sh +++ b/.github/workflows/dependencies/nvcc11-8.sh @@ -26,6 +26,9 @@ sudo apt-get install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb sudo dpkg -i cuda-keyring_1.0-1_all.deb @@ -55,12 +58,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/nvhpc.sh b/.github/workflows/dependencies/nvhpc.sh index 9c6166d386d..be2ff29ca1e 100755 --- a/.github/workflows/dependencies/nvhpc.sh +++ b/.github/workflows/dependencies/nvhpc.sh @@ -25,6 +25,9 @@ sudo apt install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | \ sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt update -y && \ @@ -46,12 +49,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/pyfull.sh b/.github/workflows/dependencies/pyfull.sh index b0a4f607a51..12678763da5 100755 --- a/.github/workflows/dependencies/pyfull.sh +++ b/.github/workflows/dependencies/pyfull.sh @@ -16,7 +16,6 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ clang \ cmake \ gnupg \ @@ -35,6 +34,9 @@ sudo apt-get install -y \ python3-setuptools \ wget +# ccache +$(dirname "$0")/ccache.sh + # cmake-easyinstall # sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 40e5055cd8b..f7378bfa775 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -21,19 +21,19 @@ jobs: run: .github/workflows/dependencies/hip.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-hip-3dsp-${{ hashFiles('.github/workflows/hip.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-hip-3dsp-${{ hashFiles('.github/workflows/hip.yml') }}- - ccache-hip-3dsp- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + source /etc/profile.d/rocm.sh hipcc --version which clang @@ -63,6 +63,9 @@ jobs: python3 -m pip wheel . python3 -m pip install *.whl + ccache -s + du -hs ~/.cache/ccache + build_hip_2d_dp: name: HIP 2D DP runs-on: ubuntu-20.04 @@ -77,19 +80,19 @@ jobs: run: .github/workflows/dependencies/hip.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-hip-2ddp-${{ hashFiles('.github/workflows/hip.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-hip-2ddp-${{ hashFiles('.github/workflows/hip.yml') }}- - ccache-hip-2ddp- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + source /etc/profile.d/rocm.sh hipcc --version which clang @@ -119,3 +122,21 @@ jobs: export PYWARPX_LIB_DIR=$PWD/build_2d/lib/site-packages/pywarpx/ python3 -m pip wheel . python3 -m pip install *.whl + + ccache -s + du -hs ~/.cache/ccache + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index b4a8869d498..9124715fe18 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -23,18 +23,19 @@ jobs: .github/workflows/dependencies/icc.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-intel-icc-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-intel-icc-${{ hashFiles('.github/workflows/intel.yml') }}- - ccache-intel-icc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=200M + export CCACHE_DEPEND=1 + ccache -z + set +eu source /opt/intel/oneapi/setvars.sh set -eu @@ -66,6 +67,9 @@ jobs: cmake --build build_sp -j 2 cmake --build build_sp --target pip_install + ccache -s + du -hs ~/.cache/ccache + build_icpx: name: oneAPI ICX SP runs-on: ubuntu-20.04 @@ -85,19 +89,20 @@ jobs: .github/workflows/dependencies/dpcpp.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-intel-icpx-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-intel-icpx-${{ hashFiles('.github/workflows/intel.yml') }}- - ccache-intel-icpx- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + export CCACHE_DEPEND=1 + ccache -z + set +e source /opt/intel/oneapi/setvars.sh set -e @@ -118,6 +123,9 @@ jobs: cmake --build build_sp -j 2 cmake --build build_sp --target pip_install + ccache -s + du -hs ~/.cache/ccache + - name: run pywarpx run: | set +e @@ -145,19 +153,20 @@ jobs: .github/workflows/dependencies/dpcpp.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-intel-dpcc-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-intel-dpcc-${{ hashFiles('.github/workflows/intel.yml') }}- - ccache-intel-dpcc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + export CCACHE_DEPEND=1 + ccache -z + set +e source /opt/intel/oneapi/setvars.sh set -e @@ -177,8 +186,26 @@ jobs: -DWarpX_PRECISION=SINGLE cmake --build build_sp -j 2 + ccache -s + du -hs ~/.cache/ccache + # Skip this as it will copy the binary artifacts and we are tight on disk space # python3 -m pip install --upgrade pip # python3 -m pip install --upgrade build packaging setuptools wheel # PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ python3 -m pip wheel . # python3 -m pip install *.whl + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 68ea4737107..c5f4391634f 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -59,16 +59,18 @@ jobs: python3 -m pip install --upgrade mpi4py - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: path: /Users/runner/Library/Caches/ccache - key: ccache-macos-appleclang-${{ hashFiles('.github/workflows/macos.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-macos-appleclang-${{ hashFiles('.github/workflows/macos.yml') }}- - ccache-macos-appleclang- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + source py-venv/bin/activate cmake -S . -B build_dp \ @@ -89,9 +91,26 @@ jobs: cmake --build build_sp -j 3 cmake --build build_sp --target pip_install + ccache -s + - name: run pywarpx run: | source py-venv/bin/activate export OMP_NUM_THREADS=1 mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml new file mode 100644 index 00000000000..f5b914033b7 --- /dev/null +++ b/.github/workflows/post-pr.yml @@ -0,0 +1,20 @@ +name: PostPR +on: + pull_request: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 8fa83c99bef..6b8e26111b8 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -20,18 +20,18 @@ jobs: .github/workflows/dependencies/gcc.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-cxxminimal-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-cxxminimal-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-cxxminimal- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + cmake -S . -B build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;3" \ @@ -42,6 +42,9 @@ jobs: ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + ccache -s + du -hs ~/.cache/ccache + build_1D_2D: name: GCC 1D & 2D w/ MPI runs-on: ubuntu-22.04 @@ -57,18 +60,18 @@ jobs: .github/workflows/dependencies/gcc12.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-1D-2D-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-1D-2D-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-1D-2D- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + cmake -S . -B build \ -GNinja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -80,6 +83,9 @@ jobs: ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d + ccache -s + du -hs ~/.cache/ccache + build_3D_sp: name: GCC 3D & RZ w/ MPI, single precision runs-on: ubuntu-22.04 @@ -94,18 +100,17 @@ jobs: .github/workflows/dependencies/gcc12_blaspp_lapackpp.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-3D-RZ-SP-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-3D-RZ-SP-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-3D-RZ-SP- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=200M + ccache -z # we need to define this *after* having installed the dependencies, # because the compilation of blaspp raises warnings. @@ -125,6 +130,9 @@ jobs: ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + ccache -s + du -hs ~/.cache/ccache + build_gcc_ablastr: name: GCC ABLASTR w/o MPI runs-on: ubuntu-20.04 @@ -140,24 +148,27 @@ jobs: sudo apt-get install -y libopenmpi-dev openmpi-bin - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-gccablastr-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-gccablastr-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-gccablastr- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + cmake -S . -B build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_APP=OFF \ -DWarpX_LIB=OFF cmake --build build -j 2 + ccache -s + du -hs ~/.cache/ccache + build_pyfull: name: Clang pywarpx runs-on: ubuntu-20.04 @@ -174,18 +185,18 @@ jobs: .github/workflows/dependencies/pyfull.sh - name: CCache Cache uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-pyfull-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-pyfull-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-pyfull- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools wheel @@ -199,7 +210,25 @@ jobs: -DWarpX_QED_TABLE_GEN=ON cmake --build build -j 2 --target pip_install + ccache -s + du -hs ~/.cache/ccache + - name: run pywarpx run: | export OMP_NUM_THREADS=1 mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v3 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 From 5e55592564ff44f0dd5aa4fce46ae79fcfee12ea Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Fri, 1 Dec 2023 11:47:47 -0600 Subject: [PATCH 009/176] Make geometry.dims abort message more user-friendly (#4459) --- Source/Utils/WarpXUtil.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index c65603f4d02..327998a61f3 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -289,11 +289,17 @@ void CheckDims () #endif const ParmParse pp_geometry("geometry"); std::string dims; - pp_geometry.get("dims", dims); std::string dims_error = "The selected WarpX executable was built as '"; dims_error.append(dims_compiled).append("'-dimensional, but the "); - dims_error.append("inputs file declares 'geometry.dims = ").append(dims).append("'.\n"); - dims_error.append("Please re-compile with a different WarpX_DIMS option or select the right executable name."); + if (pp_geometry.contains("dims")) { + pp_geometry.get("dims", dims); + dims_error.append("inputs file declares 'geometry.dims = ").append(dims).append("'.\n"); + dims_error.append("Please re-compile with a different WarpX_DIMS option or select the right executable name."); + } else { + dims = "Not specified"; + dims_error.append("inputs file does not declare 'geometry.dims'. Please add 'geometry.dims = "); + dims_error.append(dims_compiled).append("' to inputs file."); + } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(dims == dims_compiled, dims_error); } From 5736887248a1e9d48dd78be088ada97a6519ae28 Mon Sep 17 00:00:00 2001 From: Thomas Marks Date: Fri, 1 Dec 2023 15:31:07 -0500 Subject: [PATCH 010/176] Fix typo in "write_diagnostics_on_restart" (#4463) * fix type on "write diagnostics on restart" write_diagnostics_on_restart was write_diagonstics_on_restart * fix all instances of typo * Delete install.sh --- Docs/source/usage/parameters.rst | 2 +- Source/Initialization/WarpXInitData.cpp | 2 +- Source/WarpX.H | 2 +- Source/WarpX.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 118fada45d9..05da0a7c5d3 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -3281,7 +3281,7 @@ The checkpoint capability can be turned with regular diagnostics: ``. Name of the checkpoint file to restart from. Returns an error if the folder does not exist or if it is not properly formatted. -* ``warpx.write_diagonstics_on_restart`` (`bool`) optional (default `false`) +* ``warpx.write_diagnostics_on_restart`` (`bool`) optional (default `false`) When `true`, write the diagnostics after restart at the time of the restart. Intervals parser diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 91e8b3f6319..bdafdf9936b 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -485,7 +485,7 @@ WarpX::InitData () AddExternalFields(); } - if (restart_chkfile.empty() || write_diagonstics_on_restart) { + if (restart_chkfile.empty() || write_diagnostics_on_restart) { // Write full diagnostics before the first iteration. multi_diags->FilterComputePackFlush(istep[0] - 1); diff --git a/Source/WarpX.H b/Source/WarpX.H index 7d0ca8500cd..de23a450567 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1619,7 +1619,7 @@ private: std::string restart_chkfile; /** When `true`, write the diagnostics after restart at the time of the restart. */ - bool write_diagonstics_on_restart = false; + bool write_diagnostics_on_restart = false; amrex::VisMF::Header::Version plotfile_headerversion = amrex::VisMF::Header::Version_v1; amrex::VisMF::Header::Version slice_plotfile_headerversion = amrex::VisMF::Header::Version_v1; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 9ec29b07f01..adc7423a157 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -615,7 +615,7 @@ WarpX::ReadParameters () } } - pp_warpx.query("write_diagonstics_on_restart", write_diagonstics_on_restart); + pp_warpx.query("write_diagnostics_on_restart", write_diagnostics_on_restart); pp_warpx.queryarr("checkpoint_signals", signals_in); #if defined(__linux__) || defined(__APPLE__) From 5ca5837d8ccc46d8e88afe68f2f4dd79e509963f Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Fri, 1 Dec 2023 17:35:15 -0800 Subject: [PATCH 011/176] Bugfix in `fields.py` `mesh()` with ghost cells included (#4466) * fix issue in `fields.py` when retrieving the mesh in 1d with ghost cells included * fix general bug with `mesh()` --- Python/pywarpx/fields.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index 5eeeda50ea7..988bdb7203f 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -160,15 +160,8 @@ def mesh(self, direction): if self.include_ghosts: # The ghost cells are added to the upper and lower end of the global domain. nghosts = self.mf.n_grow_vect() - ilo = list(ilo) - ihi = list(ihi) - min_box = self.mf.box_array().minimal_box() - imax = min_box.big_end - for i in range(self.dim): - if ilo[i] == 0: - ilo[i] -= nghosts[i] - if ihi[i] == imax[i]: - ihi[i] += nghosts[i] + ilo -= nghosts[idir] + ihi += nghosts[idir] # Cell size in the direction warpx = libwarpx.libwarpx_so.get_instance() From d2633ba42aa322d960dea3997544352dbcb33671 Mon Sep 17 00:00:00 2001 From: Thomas Marks Date: Fri, 1 Dec 2023 21:39:12 -0500 Subject: [PATCH 012/176] fix preprocessor defs for 1D in two locations (#4465) --- .../ComputeDiagFunctors/BackTransformParticleFunctor.H | 2 +- Source/Initialization/WarpXInitData.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H index dc14d8a38c8..ed867520375 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H @@ -154,7 +154,7 @@ struct LorentzTransformParticles dst.m_aos[i_dst].pos(0) = xp; dst.m_aos[i_dst].pos(1) = zp; amrex::ignore_unused(yp); -#elif defined (WARPX_DIM_1D) +#elif defined (WARPX_DIM_1D_Z) dst.m_aos[i_dst].pos(0) = zp; amrex::ignore_unused(xp, yp); #else diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index bdafdf9936b..93cf38f7c75 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1570,7 +1570,7 @@ WarpX::ReadExternalFieldFromFile ( void WarpX::ReadExternalFieldFromFile (std::string , amrex::MultiFab* ,std::string, std::string) { -#if defined(WARPX_DIM_1D) +#if defined(WARPX_DIM_1D_Z) WARPX_ABORT_WITH_MESSAGE("Reading fields from openPMD files is not supported in 1D"); #elif defined(WARPX_DIM_XZ) WARPX_ABORT_WITH_MESSAGE("Reading from openPMD for external fields is not known to work with XZ (see #3828)"); From a393c7bfdc3b3a9ddaa7c8fcdc1d340d26612f49 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 4 Dec 2023 10:50:25 -0800 Subject: [PATCH 013/176] Add external current handling to Ohm's law solver (#4405) * add `WarpX::getedgelengths` and WarpX::getfaceareas` functions to return pointers to those multifabs * add external current support to the hybrid-PIC solver * add external current specification (for hybrid-PIC scheme) to picmi * add RZ support for `FiniteDifferenceSolver::CalculateCurrentAmpere` * allow an initial Bz field to be set in RZ * code cleanup and addition of CI test * avoid lambda capture issue * update documentation to show external current implementation * revert unwanted changes * restore RZ support that was lost during rebase * fix segfault when EB support is OFF * fix codeQL issue * add some details to docs * the external current only needs to be calculated once per field solve step * add description of `hybrid_pic_model.J[x/y/z]_external_grid_function(x, y, z, t)` to the documentation --- .../theory/kinetic_fluid_hybrid_model.rst | 46 +++- Docs/source/usage/parameters.rst | 21 +- Python/pywarpx/picmi.py | 20 +- .../FiniteDifferenceSolver.H | 4 + .../HybridPICModel/HybridPICModel.H | 23 ++ .../HybridPICModel/HybridPICModel.cpp | 207 +++++++++++++++++- .../HybridPICSolveE.cpp | 37 ++-- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 3 + Source/WarpX.H | 3 + 9 files changed, 329 insertions(+), 35 deletions(-) diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index 37e4955d665..d70e30a9821 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -35,25 +35,51 @@ neglecting the displacement current term :cite:p:`c-NIELSON1976`, giving, \mu_0\vec{J} = \vec{\nabla}\times\vec{B}, -where :math:`\vec{J} = \vec{J}_i - \vec{J}_e` is the total electrical current, i.e. -the sum of electron and ion currents. Since ions are treated in the regular -PIC manner, the ion current, :math:`\vec{J}_i`, is known during a simulation. Therefore, +where :math:`\vec{J} = \sum_{s\neq e}\vec{J}_s + \vec{J}_e + \vec{J}_{ext}` is the total electrical current, +i.e. the sum of electron and ion currents as well as any external current (not captured through plasma +particles). Since ions are treated in the regular +PIC manner, the ion current, :math:`\sum_{s\neq e}\vec{J}_s`, is known during a simulation. Therefore, given the magnetic field, the electron current can be calculated. -If we now further assume electrons are inertialess, the electron momentum -equation yields, +The electron momentum transport equation (obtained from multiplying the Vlasov equation by mass and +integrating over velocity), also called the generalized Ohm's law, is given by: .. math:: - \frac{d(n_em_e\vec{V}_e)}{dt} = 0 = -en_e\vec{E}-\vec{J}_e\times\vec{B}-\nabla\cdot\vec{P}_e+en_e\vec{\eta}\cdot\vec{J}, + en_e\vec{E} = \frac{m}{e}\frac{\partial \vec{J}_e}{\partial t} + \frac{m}{e^2}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e -where :math:`\vec{V_e}=\vec{J}_e/(en_e)`, :math:`\vec{P}_e` is the electron pressure -tensor and :math:`\vec{\eta}` is the resistivity tensor. An expression for the electric field -(generalized Ohm's law) can be obtained from the above as: +where :math:`\vec{U}_e = \vec{J}_e/(en_e)` is the electron fluid velocity, +:math:`{\overleftrightarrow P}_e` is the electron pressure tensor and +:math:`\vec{R}_e` is the drag force due to collisions between electrons and ions. +Applying the above momentum equation to the Maxwell-Faraday equation (:math:`\frac{\partial\vec{B}}{\partial t} = -\nabla\times\vec{E}`) +and substituting in :math:`\vec{J}` calculated from the Maxwell-Ampere equation, gives, .. math:: - \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla\cdot\vec{P}_e \right)+\vec{\eta}\cdot\vec{J}. + \frac{\partial\vec{J}_e}{\partial t} = -\frac{1}{\mu_0}\nabla\times\left(\nabla\times\vec{E}\right) - \frac{\partial\vec{J}_{ext}}{\partial t} - \sum_{s\neq e}\frac{\partial\vec{J}_s}{\partial t}. + +Plugging this back into the generalized Ohm' law gives: + + .. math:: + + \left(en_e +\frac{m}{e\mu_0}\nabla\times\nabla\times\right)\vec{E} =& + - \frac{m}{e}\left( \frac{\partial\vec{J}_{ext}}{\partial t} + \sum_{s\neq e}\frac{\partial\vec{J}_s}{\partial t} \right) \\ + &+ \frac{m}{e^2}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e. + +If we now further assume electrons are inertialess (i.e. :math:`m=0`), the above equation simplifies to, + + .. math:: + + en_e\vec{E} = -\vec{J}_e\times\vec{B}-\nabla\cdot{\overleftrightarrow P}_e+\vec{R}_e. + +Making the further simplifying assumptions that the electron pressure is isotropic and that +the electron drag term can be written as a simple resistance +i.e. :math:`\vec{R}_e = en_e\vec{\eta}\cdot\vec{J}`, brings us to the implemented form of +Ohm's law: + + .. math:: + + \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla P_e \right)+\vec{\eta}\cdot\vec{J}. Lastly, if an electron temperature is given from which the electron pressure can be calculated, the model is fully constrained and can be evolved given initial diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 05da0a7c5d3..4bc8fdfc7cc 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2126,25 +2126,28 @@ Maxwell solver: kinetic-fluid hybrid ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * ``hybrid_pic_model.elec_temp`` (`float`) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the electron temperature, in eV, used to calculate - the electron pressure (see :ref:`here `). + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the electron temperature, in eV, used to calculate + the electron pressure (see :ref:`here `). * ``hybrid_pic_model.n0_ref`` (`float`) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the reference density, in :math:`m^{-3}`, used to calculate - the electron pressure (see :ref:`here `). + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the reference density, in :math:`m^{-3}`, used to calculate + the electron pressure (see :ref:`here `). * ``hybrid_pic_model.gamma`` (`float`) optional (default ``5/3``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the exponent used to calculate - the electron pressure (see :ref:`here `). + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the exponent used to calculate + the electron pressure (see :ref:`here `). * ``hybrid_pic_model.plasma_resistivity(rho)`` (`float` or `str`) optional (default ``0``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma resistivity in :math:`\Omega m`. + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma resistivity in :math:`\Omega m`. + +* ``hybrid_pic_model.J[x/y/z]_external_grid_function(x, y, z, t)`` (`float` or `str`) optional (default ``0``) + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the external current (on the grid) in :math:`A/m^2`. * ``hybrid_pic_model.n_floor`` (`float`) optional (default ``1``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma density floor, in :math:`m^{-3}`, which is useful since the generalized Ohm's law used to calculate the E-field includes a :math:`1/n` term. + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma density floor, in :math:`m^{-3}`, which is useful since the generalized Ohm's law used to calculate the E-field includes a :math:`1/n` term. * ``hybrid_pic_model.substeps`` (`int`) optional (default ``100``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the number of sub-steps to take during the B-field update. + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the number of sub-steps to take during the B-field update. .. note:: diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 520bc2250ab..140bde7efd6 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1136,9 +1136,14 @@ class HybridPICSolver(picmistandard.base._ClassWithInit): substeps: int, default=100 Number of substeps to take when updating the B-field. + + Jx/y/z_external_function: str + Function of space and time specifying external (non-plasma) currents. """ def __init__(self, grid, Te=None, n0=None, gamma=None, - n_floor=None, plasma_resistivity=None, substeps=None, **kw): + n_floor=None, plasma_resistivity=None, substeps=None, + Jx_external_function=None, Jy_external_function=None, + Jz_external_function=None, **kw): self.grid = grid self.method = "hybrid" @@ -1150,6 +1155,10 @@ def __init__(self, grid, Te=None, n0=None, gamma=None, self.substeps = substeps + self.Jx_external_function = Jx_external_function + self.Jy_external_function = Jy_external_function + self.Jz_external_function = Jz_external_function + self.handle_init(kw) def initialize_inputs(self): @@ -1166,6 +1175,15 @@ def initialize_inputs(self): 'plasma_resistivity(rho)', self.plasma_resistivity ) pywarpx.hybridpicmodel.substeps = self.substeps + pywarpx.hybridpicmodel.__setattr__( + 'Jx_external_grid_function(x,y,z,t)', self.Jx_external_function + ) + pywarpx.hybridpicmodel.__setattr__( + 'Jy_external_grid_function(x,y,z,t)', self.Jy_external_function + ) + pywarpx.hybridpicmodel.__setattr__( + 'Jz_external_grid_function(x,y,z,t)', self.Jz_external_function + ) class ElectrostaticSolver(picmistandard.PICMI_ElectrostaticSolver): diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index d877db5253d..612267b50a3 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -139,6 +139,7 @@ class FiniteDifferenceSolver * \param[out] Efield vector of electric field MultiFabs updated at a given level * \param[in] Jfield vector of total current MultiFabs at a given level * \param[in] Jifield vector of ion current density MultiFabs at a given level + * \param[in] Jextfield vector of external current density MultiFabs at a given level * \param[in] Bfield vector of magnetic field MultiFabs at a given level * \param[in] rhofield scalar ion charge density Multifab at a given level * \param[in] Pefield scalar electron pressure MultiFab at a given level @@ -150,6 +151,7 @@ class FiniteDifferenceSolver void HybridPICSolveE ( std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3>& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, @@ -233,6 +235,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, @@ -337,6 +340,7 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 399177c82bb..6bcee1bec94 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -46,6 +46,20 @@ public: void InitData (); + /** + * \brief + * Function to evaluate the external current expressions and populate the + * external current multifab. Note the external current can be a function + * of time and therefore this should be re-evaluated at every step. + */ + void GetCurrentExternal ( + amrex::Vector, 3>> const& edge_lengths + ); + void GetCurrentExternal ( + std::array< std::unique_ptr, 3> const& edge_lengths, + int lev + ); + /** * \brief * Function to calculate the total current based on Ampere's law while @@ -133,10 +147,19 @@ public: std::unique_ptr m_resistivity_parser; amrex::ParserExecutor<1> m_eta; + /** External current */ + std::string m_Jx_ext_grid_function = "0.0"; + std::string m_Jy_ext_grid_function = "0.0"; + std::string m_Jz_ext_grid_function = "0.0"; + std::array< std::unique_ptr, 3> m_J_external_parser; + std::array< amrex::ParserExecutor<4>, 3> m_J_external; + bool m_external_field_has_time_dependence = false; + // Declare multifabs specifically needed for the hybrid-PIC model amrex::Vector< std::unique_ptr > rho_fp_temp; amrex::Vector, 3 > > current_fp_temp; amrex::Vector, 3 > > current_fp_ampere; + amrex::Vector, 3 > > current_fp_external; amrex::Vector< std::unique_ptr > electron_pressure_fp; // Helper functions to retrieve hybrid-PIC multifabs diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index a94593ded83..57999335ae6 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -42,6 +42,11 @@ void HybridPICModel::ReadParameters () // convert electron temperature from eV to J m_elec_temp *= PhysConst::q_e; + + // external currents + pp_hybrid.query("Jx_external_grid_function(x,y,z,t)", m_Jx_ext_grid_function); + pp_hybrid.query("Jy_external_grid_function(x,y,z,t)", m_Jy_ext_grid_function); + pp_hybrid.query("Jz_external_grid_function(x,y,z,t)", m_Jz_ext_grid_function); } void HybridPICModel::AllocateMFs (int nlevs_max) @@ -50,6 +55,7 @@ void HybridPICModel::AllocateMFs (int nlevs_max) rho_fp_temp.resize(nlevs_max); current_fp_temp.resize(nlevs_max); current_fp_ampere.resize(nlevs_max); + current_fp_external.resize(nlevs_max); } void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm, @@ -87,6 +93,15 @@ void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const Distri WarpX::AllocInitMultiFab(current_fp_ampere[lev][2], amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_fp_ampere[z]", 0.0_rt); + // the external current density multifab is made nodal to avoid needing to interpolate + // to a nodal grid as has to be done for the ion and total current density multifabs + WarpX::AllocInitMultiFab(current_fp_external[lev][0], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, ngJ, lev, "current_fp_external[x]", 0.0_rt); + WarpX::AllocInitMultiFab(current_fp_external[lev][1], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, ngJ, lev, "current_fp_external[y]", 0.0_rt); + WarpX::AllocInitMultiFab(current_fp_external[lev][2], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, ngJ, lev, "current_fp_external[z]", 0.0_rt); + #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE( (ncomps == 1), @@ -101,6 +116,7 @@ void HybridPICModel::ClearLevel (int lev) for (int i = 0; i < 3; ++i) { current_fp_temp[lev][i].reset(); current_fp_ampere[lev][i].reset(); + current_fp_external[lev][i].reset(); } } @@ -110,6 +126,22 @@ void HybridPICModel::InitData () utils::parser::makeParser(m_eta_expression, {"rho"})); m_eta = m_resistivity_parser->compile<1>(); + m_J_external_parser[0] = std::make_unique( + utils::parser::makeParser(m_Jx_ext_grid_function,{"x","y","z","t"})); + m_J_external_parser[1] = std::make_unique( + utils::parser::makeParser(m_Jy_ext_grid_function,{"x","y","z","t"})); + m_J_external_parser[2] = std::make_unique( + utils::parser::makeParser(m_Jz_ext_grid_function,{"x","y","z","t"})); + m_J_external[0] = m_J_external_parser[0]->compile<4>(); + m_J_external[1] = m_J_external_parser[1]->compile<4>(); + m_J_external[2] = m_J_external_parser[2]->compile<4>(); + + // check if the external current parsers depend on time + for (int i=0; i<3; i++) { + const std::set J_ext_symbols = m_J_external_parser[i]->symbols(); + m_external_field_has_time_dependence += J_ext_symbols.count("t"); + } + auto & warpx = WarpX::GetInstance(); // Get the grid staggering of the fields involved in calculating E @@ -181,6 +213,178 @@ void HybridPICModel::InitData () Ey_IndexType[1] = 1; Ez_IndexType[1] = 1; #endif + + // Initialize external current - note that this approach skips the check + // if the current is time dependent which is what needs to be done to + // write time independent fields on the first step. + std::array< std::unique_ptr, 3 > edge_lengths; + + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) + { +#ifdef AMREX_USE_EB + auto& edge_lengths_x = warpx.getedgelengths(lev, 0); + edge_lengths[0] = std::make_unique( + edge_lengths_x, amrex::make_alias, 0, edge_lengths_x.nComp() + ); + auto& edge_lengths_y = warpx.getedgelengths(lev, 1); + edge_lengths[1] = std::make_unique( + edge_lengths_y, amrex::make_alias, 0, edge_lengths_y.nComp() + ); + auto& edge_lengths_z = warpx.getedgelengths(lev, 2); + edge_lengths[2] = std::make_unique( + edge_lengths_z, amrex::make_alias, 0, edge_lengths_z.nComp() + ); +#endif + GetCurrentExternal(edge_lengths, lev); + } +} + +void HybridPICModel::GetCurrentExternal ( + amrex::Vector, 3>> const& edge_lengths) +{ + if (!m_external_field_has_time_dependence) return; + + auto& warpx = WarpX::GetInstance(); + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) + { + GetCurrentExternal(edge_lengths[lev], lev); + } +} + + +void HybridPICModel::GetCurrentExternal ( + std::array< std::unique_ptr, 3> const& edge_lengths, + int lev) +{ + // This logic matches closely to WarpX::InitializeExternalFieldsOnGridUsingParser + // except that the parsers include time dependence. + auto & warpx = WarpX::GetInstance(); + + auto t = warpx.gett_new(lev); + + auto dx_lev = warpx.Geom(lev).CellSizeArray(); + const RealBox& real_box = warpx.Geom(lev).ProbDomain(); + + auto& mfx = current_fp_external[lev][0]; + auto& mfy = current_fp_external[lev][1]; + auto& mfz = current_fp_external[lev][2]; + + const amrex::IntVect x_nodal_flag = mfx->ixType().toIntVect(); + const amrex::IntVect y_nodal_flag = mfy->ixType().toIntVect(); + const amrex::IntVect z_nodal_flag = mfz->ixType().toIntVect(); + + // avoid implicit lambda capture + auto Jx_external = m_J_external[0]; + auto Jy_external = m_J_external[1]; + auto Jz_external = m_J_external[2]; + + for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); + const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); + const amrex::Box& tbz = mfi.tilebox( z_nodal_flag, mfz->nGrowVect() ); + + auto const& mfxfab = mfx->array(mfi); + auto const& mfyfab = mfy->array(mfi); + auto const& mfzfab = mfz->array(mfi); + +#ifdef AMREX_USE_EB + amrex::Array4 const& lx = edge_lengths[0]->array(mfi); + amrex::Array4 const& ly = edge_lengths[1]->array(mfi); + amrex::Array4 const& lz = edge_lengths[2]->array(mfi); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ignore_unused(ly); +#endif +#else + amrex::ignore_unused(edge_lengths); +#endif + + amrex::ParallelFor (tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // skip if node is covered by an embedded boundary +#ifdef AMREX_USE_EB + if (lx(i, j, k) <= 0) return; +#endif + // Shift required in the x-, y-, or z- position + // depending on the index type of the multifab +#if defined(WARPX_DIM_1D_Z) + const amrex::Real x = 0._rt; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real fac_x = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - x_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; +#else + const amrex::Real fac_x = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real fac_y = (1._rt - x_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; + const amrex::Real fac_z = (1._rt - x_nodal_flag[2]) * dx_lev[2] * 0.5_rt; + const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; +#endif + // Initialize the x-component of the field. + mfxfab(i,j,k) = Jx_external(x,y,z,t); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // skip if node is covered by an embedded boundary +#ifdef AMREX_USE_EB + if (ly(i, j, k) <= 0) return; +#endif +#if defined(WARPX_DIM_1D_Z) + const amrex::Real x = 0._rt; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real fac_x = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - y_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; +#elif defined(WARPX_DIM_3D) + const amrex::Real fac_x = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real fac_y = (1._rt - y_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; + const amrex::Real fac_z = (1._rt - y_nodal_flag[2]) * dx_lev[2] * 0.5_rt; + const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; +#endif + // Initialize the y-component of the field. + mfyfab(i,j,k) = Jy_external(x,y,z,t); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // skip if node is covered by an embedded boundary +#ifdef AMREX_USE_EB + if (lz(i, j, k) <= 0) return; +#endif +#if defined(WARPX_DIM_1D_Z) + const amrex::Real x = 0._rt; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real fac_x = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - z_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; +#elif defined(WARPX_DIM_3D) + const amrex::Real fac_x = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real fac_y = (1._rt - z_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; + const amrex::Real fac_z = (1._rt - z_nodal_flag[2]) * dx_lev[2] * 0.5_rt; + const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; +#endif + // Initialize the z-component of the field. + mfzfab(i,j,k) = Jz_external(x,y,z,t); + } + ); + } } void HybridPICModel::CalculateCurrentAmpere ( @@ -265,7 +469,8 @@ void HybridPICModel::HybridPICSolveE ( // Solve E field in regular cells warpx.get_pointer_fdtd_solver_fp(lev)->HybridPICSolveE( - Efield, current_fp_ampere[lev], Jfield, Bfield, rhofield, + Efield, current_fp_ampere[lev], Jfield, current_fp_external[lev], + Bfield, rhofield, electron_pressure_fp[lev], edge_lengths, lev, this, include_resistivity_term ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index ff220887099..84d5b773097 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -369,6 +369,7 @@ void FiniteDifferenceSolver::HybridPICSolveE ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 >& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, @@ -382,14 +383,14 @@ void FiniteDifferenceSolver::HybridPICSolveE ( #ifdef WARPX_DIM_RZ HybridPICSolveECylindrical ( - Efield, Jfield, Jifield, Bfield, rhofield, Pefield, + Efield, Jfield, Jifield, Jextfield, Bfield, rhofield, Pefield, edge_lengths, lev, hybrid_model, include_resistivity_term ); #else HybridPICSolveECartesian ( - Efield, Jfield, Jifield, Bfield, rhofield, Pefield, + Efield, Jfield, Jifield, Jextfield, Bfield, rhofield, Pefield, edge_lengths, lev, hybrid_model, include_resistivity_term ); @@ -406,6 +407,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, @@ -484,6 +486,9 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Array4 const& Jir = Jifield[0]->const_array(mfi); Array4 const& Jit = Jifield[1]->const_array(mfi); Array4 const& Jiz = Jifield[2]->const_array(mfi); + Array4 const& Jextr = Jextfield[0]->const_array(mfi); + Array4 const& Jextt = Jextfield[1]->const_array(mfi); + Array4 const& Jextz = Jextfield[2]->const_array(mfi); Array4 const& Br = Bfield[0]->const_array(mfi); Array4 const& Bt = Bfield[1]->const_array(mfi); Array4 const& Bz = Bfield[2]->const_array(mfi); @@ -508,16 +513,16 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // calculate enE = (J - Ji) x B enE_nodal(i, j, 0, 0) = ( - (jt_interp - jit_interp) * Bz_interp - - (jz_interp - jiz_interp) * Bt_interp + (jt_interp - jit_interp - Jextt(i, j, 0)) * Bz_interp + - (jz_interp - jiz_interp - Jextz(i, j, 0)) * Bt_interp ); enE_nodal(i, j, 0, 1) = ( - (jz_interp - jiz_interp) * Br_interp - - (jr_interp - jir_interp) * Bz_interp + (jz_interp - jiz_interp - Jextz(i, j, 0)) * Br_interp + - (jr_interp - jir_interp - Jextr(i, j, 0)) * Bz_interp ); enE_nodal(i, j, 0, 2) = ( - (jr_interp - jir_interp) * Bt_interp - - (jt_interp - jit_interp) * Br_interp + (jr_interp - jir_interp - Jextr(i, j, 0)) * Bt_interp + - (jt_interp - jit_interp - Jextt(i, j, 0)) * Br_interp ); }); @@ -674,6 +679,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, @@ -746,6 +752,9 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Array4 const& Jix = Jifield[0]->const_array(mfi); Array4 const& Jiy = Jifield[1]->const_array(mfi); Array4 const& Jiz = Jifield[2]->const_array(mfi); + Array4 const& Jextx = Jextfield[0]->const_array(mfi); + Array4 const& Jexty = Jextfield[1]->const_array(mfi); + Array4 const& Jextz = Jextfield[2]->const_array(mfi); Array4 const& Bx = Bfield[0]->const_array(mfi); Array4 const& By = Bfield[1]->const_array(mfi); Array4 const& Bz = Bfield[2]->const_array(mfi); @@ -770,16 +779,16 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // calculate enE = (J - Ji) x B enE_nodal(i, j, k, 0) = ( - (jy_interp - jiy_interp) * Bz_interp - - (jz_interp - jiz_interp) * By_interp + (jy_interp - jiy_interp - Jexty(i, j, k)) * Bz_interp + - (jz_interp - jiz_interp - Jextz(i, j, k)) * By_interp ); enE_nodal(i, j, k, 1) = ( - (jz_interp - jiz_interp) * Bx_interp - - (jx_interp - jix_interp) * Bz_interp + (jz_interp - jiz_interp - Jextz(i, j, k)) * Bx_interp + - (jx_interp - jix_interp - Jextx(i, j, k)) * Bz_interp ); enE_nodal(i, j, k, 2) = ( - (jx_interp - jix_interp) * By_interp - - (jy_interp - jiy_interp) * Bx_interp + (jx_interp - jix_interp - Jextx(i, j, k)) * By_interp + - (jy_interp - jiy_interp - Jexty(i, j, k)) * Bx_interp ); }); diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 9a5449dfef8..00c2ddab73a 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -54,6 +54,9 @@ void WarpX::HybridPICEvolveFields () // Get requested number of substeps to use int sub_steps = m_hybrid_pic_model->m_substeps / 2; + // Get the external current + m_hybrid_pic_model->GetCurrentExternal(m_edge_lengths); + // Reference hybrid-PIC multifabs auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; auto& current_fp_temp = m_hybrid_pic_model->current_fp_temp; diff --git a/Source/WarpX.H b/Source/WarpX.H index de23a450567..04327752de8 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -517,6 +517,9 @@ public: const amrex::MultiFab& getEfield_avg_cp (int lev, int direction) {return *Efield_avg_cp[lev][direction];} const amrex::MultiFab& getBfield_avg_cp (int lev, int direction) {return *Bfield_avg_cp[lev][direction];} + const amrex::MultiFab& getedgelengths (int lev, int direction) {return *m_edge_lengths[lev][direction];} + const amrex::MultiFab& getfaceareas (int lev, int direction) {return *m_face_areas[lev][direction];} + [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} From ffbdd962814796684a2990373ad7b38c796c8055 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 4 Dec 2023 16:39:46 -0800 Subject: [PATCH 014/176] Release 23.12 (#4469) * AMReX: 23.12 * pyAMReX: 23.12 * WarpX: 23.12 --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 3ad7978fb2b..b94b19141a6 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 9b733ec45cd93a80234a7c98248b6eb4816589d5 && cd - + cd ../amrex && git checkout --detach 23.12 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b3eae1663c..e6aa3c8174f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 23.11) +project(WarpX VERSION 23.12) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index f6fa22f40f8..9a2def319e4 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -79,9 +79,9 @@ # built documents. # # The short X.Y version. -version = u'23.11' +version = u'23.12' # The full version, including alpha/beta/rc tags. -release = u'23.11' +release = u'23.12' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index 62294fcaae2..47028642cce 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '23.11', + version = '23.12', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index f16c0f74e9f..38ea288eacd 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 9b733ec45cd93a80234a7c98248b6eb4816589d5 +branch = 23.12 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index dbc3421d8b0..191eab3bca2 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 9b733ec45cd93a80234a7c98248b6eb4816589d5 +branch = 23.12 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index f3866efef5b..d7790805608 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 23.11 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 23.12 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "9b733ec45cd93a80234a7c98248b6eb4816589d5" +set(WarpX_amrex_branch "23.12" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 7752a82758b..792d52327c1 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 23.11 CONFIG REQUIRED) + find_package(pyAMReX 23.12 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "58dc1ed58226ab683a9bdea858da2a196cd1570f" +set(WarpX_pyamrex_branch "23.12" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 41fc96b0b05..b0aa7ff909e 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 9b733ec45cd93a80234a7c98248b6eb4816589d5 && cd - +cd amrex && git checkout --detach 23.12 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index bb92d69b417..5228b7b3521 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '23.11', + version = '23.12', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From 85735511546bae4662580b3290b6a56c5adf0e7e Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Mon, 4 Dec 2023 18:09:36 -0800 Subject: [PATCH 015/176] Remove brew cache (#4470) The size of the brew cache is huge. It can be as big as more than 6 GB. Since we only have 10 GB GitHub cache space, the brew cache is disabled in the macOS CI. It should not increase the time for the macOS CI too much. Without any cache, it usually takes less than 10 minutes to install all the brew packages. --- .github/workflows/macos.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index c5f4391634f..ec1b43a1a30 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -18,22 +18,8 @@ jobs: #CMAKE_GENERATOR: Ninja steps: - uses: actions/checkout@v3 - - name: Brew Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key - with: - path: | - /usr/local/bin - /usr/local/lib - /usr/local/share - /Users/runner/Library/Caches/Homebrew - key: brew-macos-appleclang-${{ hashFiles('.github/workflows/macos.yml') }} - restore-keys: | - brew-macos-appleclang- - name: install dependencies run: | - brew --cache set +e brew unlink gcc brew update From 75132049f33ab49ca9d9441da94bc744007c2b1d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 4 Dec 2023 22:09:23 -0800 Subject: [PATCH 016/176] Doc: Restructure Examples (#4467) * Doc: Restructure Examples Restructure examples by science case. * Doc: Fix visualized field name. * Doc: `:py:function:` with trailing () Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- .../workflows/source/wrongFileNameInExamples | 1 + Docs/source/dataanalysis/examples | 1 - Docs/source/refs.bib | 103 +++++++ Docs/source/usage/examples.rst | 271 ++++++------------ .../usage/examples/capacitive_discharge | 1 + Docs/source/usage/examples/gaussian_beam | 1 + Docs/source/usage/examples/langmuir | 1 + Docs/source/usage/examples/laser_ion | 1 + Docs/source/usage/examples/lwfa | 1 + .../source/usage/examples/ohm_solver_EM_modes | 1 + .../examples/ohm_solver_ion_Landau_damping | 1 + .../examples/ohm_solver_ion_beam_instability | 1 + .../examples/ohm_solver_magnetic_reconnection | 1 + Docs/source/usage/examples/plasma_mirror | 1 + Docs/source/usage/examples/pwfa | 1 + Docs/source/usage/examples/uniform_plasma | 1 + Docs/source/usage/python.rst | 18 +- .../capacitive_discharge/README.md | 7 - .../capacitive_discharge/README.rst | 54 ++++ .../laser_acceleration/README.md | 12 - .../laser_acceleration/README.rst | 94 ++++++ .../laser_acceleration/plot_3d.py | 40 +++ .../Physics_applications/laser_ion/README.rst | 65 +++++ .../laser_ion/{inputs => inputs_2d} | 0 .../plasma_acceleration/README.rst | 70 +++++ .../plasma_mirror/README.rst | 58 ++++ .../uniform_plasma/README.rst | 73 +++++ Examples/Tests/gaussian_beam/README.rst | 47 +++ Examples/Tests/langmuir/README.md | 26 -- Examples/Tests/langmuir/README.rst | 146 ++++++++++ Examples/Tests/ohm_solver_EM_modes/README.rst | 45 +++ .../ohm_solver_ion_Landau_damping/README.rst | 25 ++ .../README.rst | 36 +++ .../README.rst | 44 +++ Regression/WarpX-tests.ini | 2 +- 35 files changed, 1009 insertions(+), 241 deletions(-) delete mode 120000 Docs/source/dataanalysis/examples create mode 120000 Docs/source/usage/examples/capacitive_discharge create mode 120000 Docs/source/usage/examples/gaussian_beam create mode 120000 Docs/source/usage/examples/langmuir create mode 120000 Docs/source/usage/examples/laser_ion create mode 120000 Docs/source/usage/examples/lwfa create mode 120000 Docs/source/usage/examples/ohm_solver_EM_modes create mode 120000 Docs/source/usage/examples/ohm_solver_ion_Landau_damping create mode 120000 Docs/source/usage/examples/ohm_solver_ion_beam_instability create mode 120000 Docs/source/usage/examples/ohm_solver_magnetic_reconnection create mode 120000 Docs/source/usage/examples/plasma_mirror create mode 120000 Docs/source/usage/examples/pwfa create mode 120000 Docs/source/usage/examples/uniform_plasma delete mode 100644 Examples/Physics_applications/capacitive_discharge/README.md create mode 100644 Examples/Physics_applications/capacitive_discharge/README.rst delete mode 100644 Examples/Physics_applications/laser_acceleration/README.md create mode 100644 Examples/Physics_applications/laser_acceleration/README.rst create mode 100755 Examples/Physics_applications/laser_acceleration/plot_3d.py create mode 100644 Examples/Physics_applications/laser_ion/README.rst rename Examples/Physics_applications/laser_ion/{inputs => inputs_2d} (100%) create mode 100644 Examples/Physics_applications/plasma_acceleration/README.rst create mode 100644 Examples/Physics_applications/plasma_mirror/README.rst create mode 100644 Examples/Physics_applications/uniform_plasma/README.rst create mode 100644 Examples/Tests/gaussian_beam/README.rst delete mode 100644 Examples/Tests/langmuir/README.md create mode 100644 Examples/Tests/langmuir/README.rst create mode 100644 Examples/Tests/ohm_solver_EM_modes/README.rst create mode 100644 Examples/Tests/ohm_solver_ion_Landau_damping/README.rst create mode 100644 Examples/Tests/ohm_solver_ion_beam_instability/README.rst create mode 100644 Examples/Tests/ohm_solver_magnetic_reconnection/README.rst diff --git a/.github/workflows/source/wrongFileNameInExamples b/.github/workflows/source/wrongFileNameInExamples index e116b9df084..0de69d69c9c 100755 --- a/.github/workflows/source/wrongFileNameInExamples +++ b/.github/workflows/source/wrongFileNameInExamples @@ -18,6 +18,7 @@ do [[ ${file:0:12} != PICMI_inputs ]] && [[ ${file:0:8 } != analysis ]] && [[ ${file: -4} != yaml ]] && + [[ ${file:0:4 } != plot ]] && [[ ${file:0:6 } != README ]] then files+=($file) diff --git a/Docs/source/dataanalysis/examples b/Docs/source/dataanalysis/examples deleted file mode 120000 index 76b45c18c59..00000000000 --- a/Docs/source/dataanalysis/examples +++ /dev/null @@ -1 +0,0 @@ -../../../Examples \ No newline at end of file diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index fdbbe57cf72..6c94869caa5 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -20,6 +20,32 @@ @article{Geddes2008 journal = {Journal of Physics: Conference Series} } +@article{TajimaDawson1982, + author = {Tajima, T. and Dawson, J. M.}, + title = "{Laser accelerator by plasma waves}", + journal = {AIP Conference Proceedings}, + volume = {91}, + number = {1}, + pages = {69-93}, + year = {1982}, + month = {09}, + abstract = "{Parallel intense laser beam ω0, k0 and ω1, k1 shone on a plasma with frequency separation equal to the plasma frequency ωp is capable of creating a coherent large electrostatic field and accelerating particles to high energies in large flux. The photon beam excites through the forward Raman scattering large amplitude plasmons whose phase velocity is equal to (ω−ω1)/(k0−k1), close to c in an underdense plasma. The plasmon traps electrons with electrostatic field EL=γ1/2⊥ mcωp/c, of the order of a few GeV/cm for plasma density to 1018 cm−3. Because of the phase velocity of the field close to c this field carries trapped electrons to high energies: W=2mc2(ω0/ωp)2. Preaccelerated particles (ions, for examples) coherent with the plasmon fields can also be accelerated. The (multiple) forward Raman instability saturates only when a sizable electron population is trapped and most of the electromagnetic energy is cascaded down to the frequency close to the cut‐off (ωp).}", + issn = {0094-243X}, + doi = {10.1063/1.33805}, + url = {https://doi.org/10.1063/1.33805} +} + +@article{Esarey1996, + author={Esarey, E. and Sprangle, P. and Krall, J. and Ting, A.}, + journal={IEEE Transactions on Plasma Science}, + title={Overview of plasma-based accelerator concepts}, + year={1996}, + volume={24}, + number={2}, + pages={252-288}, + doi={10.1109/27.509991} +} + @techreport{Geddes2009, title={Laser plasma particle accelerators: Large fields for smaller facility sources}, author={Geddes, Cameron GR and Cormier-Michel, Estelle and Esarey, Eric H and Schroeder, Carl B and Vay, Jean-Luc and Leemans, Wim P and Bruhwiler, David L and Cary, John R and Cowan, Ben and Durant, Marc and others}, @@ -496,3 +522,80 @@ @book{Stix1992 year = {1992}, bdsk-url-1 = {https://books.google.com/books?id=OsOWJ8iHpmMC} } + +@article{Macchi2013, + title = {Ion acceleration by superintense laser-plasma interaction}, + author = {Macchi, Andrea and Borghesi, Marco and Passoni, Matteo}, + journal = {Rev. Mod. Phys.}, + volume = {85}, + issue = {2}, + pages = {751--793}, + numpages = {0}, + year = {2013}, + month = {May}, + publisher = {American Physical Society}, + doi = {10.1103/RevModPhys.85.751}, + url = {https://link.aps.org/doi/10.1103/RevModPhys.85.751} +} + +@article{Wilks2001, + author = {Wilks, S. C. and Langdon, A. B. and Cowan, T. E. and Roth, M. and Singh, M. and Hatchett, S. and Key, M. H. and Pennington, D. and MacKinnon, A. and Snavely, R. A.}, + title = "{Energetic proton generation in ultra-intense laser–solid interactions}", + journal = {Physics of Plasmas}, + volume = {8}, + number = {2}, + pages = {542-549}, + year = {2001}, + month = {02}, + abstract = "{An explanation for the energetic ions observed in the PetaWatt experiments is presented. In solid target experiments with focused intensities exceeding 1020 W/cm2, high-energy electron generation, hard bremsstrahlung, and energetic protons have been observed on the backside of the target. In this report, an attempt is made to explain the physical process present that will explain the presence of these energetic protons, as well as explain the number, energy, and angular spread of the protons observed in experiment. In particular, we hypothesize that hot electrons produced on the front of the target are sent through to the back off the target, where they ionize the hydrogen layer there. These ions are then accelerated by the hot electron cloud, to tens of MeV energies in distances of order tens of μm, whereupon they end up being detected in the radiographic and spectrographic detectors.}", + issn = {1070-664X}, + doi = {10.1063/1.1333697}, + url = {https://doi.org/10.1063/1.1333697}, + eprint = {https://pubs.aip.org/aip/pop/article-pdf/8/2/542/12669088/542\_1\_online.pdf}, +} + +@article{Bulanov2008, + title = {Accelerating monoenergetic protons from ultrathin foils by flat-top laser pulses in the directed-Coulomb-explosion regime}, + author = {Bulanov, S. S. and Brantov, A. and Bychenkov, V. Yu. and Chvykov, V. and Kalinchenko, G. and Matsuoka, T. and Rousseau, P. and Reed, S. and Yanovsky, V. and Litzenberg, D. W. and Krushelnick, K. and Maksimchuk, A.}, + journal = {Phys. Rev. E}, + volume = {78}, + issue = {2}, + pages = {026412}, + numpages = {6}, + year = {2008}, + month = {Aug}, + publisher = {American Physical Society}, + doi = {10.1103/PhysRevE.78.026412}, + url = {https://link.aps.org/doi/10.1103/PhysRevE.78.026412} +} + +@article{Dromey2004, + author = {Dromey, B. and Kar, S. and Zepf, M. and Foster, P.}, + title = "{The plasma mirror—A subpicosecond optical switch for ultrahigh power lasers}", + journal = {Review of Scientific Instruments}, + volume = {75}, + number = {3}, + pages = {645-649}, + year = {2004}, + month = {02}, + abstract = "{Plasma mirrors are devices capable of switching very high laser powers on subpicosecond time scales with a dynamic range of 20–30 dB. A detailed study of their performance in the near-field of the laser beam is presented, a setup relevant to improving the pulse contrast of modern ultrahigh power lasers (TW–PW). The conditions under which high reflectivity can be achieved and focusability of the reflected beam retained are identified. At higher intensities a region of high specular reflectivity with rapidly decreasing focusability was observed, suggesting that specular reflectivity alone is not an adequate guide to the ideal range of plasma mirror operation. It was found that to achieve high reflectivity with negligible phasefront distortion of the reflected beam the inequality csΔt\\<λLaser must be met (cs: sound speed, Δt: time from plasma formation to the peak of the pulse). The achievable contrast enhancement is given by the ratio of plasma mirror reflectivity to cold reflectivity.}", + issn = {0034-6748}, + doi = {10.1063/1.1646737}, + url = {https://doi.org/10.1063/1.1646737}, + eprint = {https://pubs.aip.org/aip/rsi/article-pdf/75/3/645/8814694/645\_1\_online.pdf}, +} + +@article{Roedel2010, + title = {High repetition rate plasma mirror for temporal contrast enhancement of terawatt femtosecond laser pulses by three orders of magnitude}, + volume = {103}, + ISSN = {1432-0649}, + url = {http://dx.doi.org/10.1007/s00340-010-4329-7}, + DOI = {10.1007/s00340-010-4329-7}, + number = {2}, + journal = {Applied Physics B}, + publisher = {Springer Science and Business Media LLC}, + author = {R\"{o}del, C. and Heyer, M. and Behmke, M. and K\"{u}bel, M. and J\"{a}ckel, O. and Ziegler, W. and Ehrt, D. and Kaluza, M. C. and Paulus, G. G.}, + year = {2010}, + month = nov, + pages = {295–302} +} diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index 48d37db98c4..dc0a076aaaa 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -7,252 +7,150 @@ This section allows you to **download input files** that correspond to different We provide two kinds of inputs: -* AMReX ``inputs`` files, :ref:`with parameters described here `, * PICMI python input files, `with parameters described here `__. +* AMReX ``inputs`` files, :ref:`with parameters described here `, -For a complete list of all example input files, have a look at our ``Examples/`` directory. -It contains folders and subfolders with self-describing names that you can try. All these input files are automatically tested, so they should always be up-to-date. +For a complete list of all example input files, also have a look at our `Examples/ `__ directory. +It contains folders and subfolders with self-describing names that you can try. +All these input files are automatically tested, so they should always be up-to-date. -Beam-driven electron acceleration ---------------------------------- -AMReX ``inputs``: +Plasma-Based Acceleration +------------------------- -* :download:`2D case <../../../Examples/Physics_applications/plasma_acceleration/inputs_2d>` -* :download:`2D case in boosted frame <../../../Examples/Physics_applications/plasma_acceleration/inputs_2d_boost>` -* :download:`3D case in boosted frame <../../../Examples/Physics_applications/plasma_acceleration/inputs_3d_boost>` +.. toctree:: + :maxdepth: 1 -PICMI: + examples/lwfa/README.rst + examples/pwfa/README.rst -* :download:`Without mesh refinement <../../../Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py>` -* :download:`With mesh refinement <../../../Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_mr.py>` +Coming soon: -Laser-driven electron acceleration ----------------------------------- +* LWFA: External injection in the boosted frame +* LWFA: Ionization injection in the lab frame using a LASY data file +* PWFA: External injection in the boosted frame +* PWFA: Self-injection in the lab frame +* MR case? -AMReX ``inputs``: -* :download:`1D case <../../../Examples/Physics_applications/laser_acceleration/inputs_1d>` -* :download:`2D case <../../../Examples/Physics_applications/laser_acceleration/inputs_2d>` -* :download:`2D case in boosted frame <../../../Examples/Physics_applications/laser_acceleration/inputs_2d_boost>` -* :download:`3D case <../../../Examples/Physics_applications/laser_acceleration/inputs_3d>` -* :download:`RZ case <../../../Examples/Physics_applications/laser_acceleration/inputs_rz>` +Laser-Plasma Interaction +------------------------ -PICMI (Python) scripts: +.. toctree:: + :maxdepth: 1 -* :download:`1D case <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_1d.py>` -* :download:`2D case with mesh refinement <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_2d.py>` -* :download:`3D case <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py>` -* :download:`RZ case <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py>` + examples/laser_ion/README.rst + examples/plasma_mirror/README.rst -Plasma mirror -------------- +Coming soon: -:download:`2D case <../../../Examples/Physics_applications/plasma_mirror/inputs_2d>` +* MVA (3D & RZ) +* MR for the planar example? -Laser-ion acceleration ----------------------- -:download:`2D case <../../../Examples/Physics_applications/laser_ion/inputs>` +Particle Accelerator & Beam Physics +----------------------------------- -.. note:: +.. toctree:: + :maxdepth: 1 - The resolution of this 2D case is extremely low by default. - You will need a computing cluster for adequate resolution of the target density, see comments in the input file. + examples/gaussian_beam/README.rst -.. warning:: +Coming soon: - It is strongly advised to set the parameters ``.zmin / zmax / xmin / ...`` when working with highly dense targets that are limited in one or multiple dimensions. - The particle creation routine will first create particles everywhere between these limits (`defaulting to box size if unset`), setting particles to invalid only afterwards based on the density profile. - Not setting these parameters can quickly lead to memory overflows. +* Beam-Beam Collision +* Beam Transport or Injector +* Cathode/source -Uniform plasma --------------- -:download:`2D case <../../../Examples/Physics_applications/uniform_plasma/inputs_2d>` -:download:`3D case <../../../Examples/Physics_applications/uniform_plasma/inputs_3d>` +High Energy Astrophysical Plasma Physics +---------------------------------------- -Capacitive discharge --------------------- +.. toctree:: + :maxdepth: 1 -The Monte-Carlo collision (MCC) model can be used to simulate electron and ion collisions with a neutral background gas. In particular this can be used to study capacitive discharges between parallel plates. The implementation has been tested against the benchmark results from :cite:t:`ex-Turner2013`. The figure below shows a comparison of the ion density as calculated in WarpX (in June 2022 with `PR #3118 `_) compared to the literature results (which can be found `here `__). + examples/ohm_solver_magnetic_reconnection/README.rst -.. figure:: https://user-images.githubusercontent.com/40245517/171573007-f7d733c7-c0de-490c-9ed6-ff4c02154358.png - :alt: MCC benchmark against Turner et. al. (2013). - :width: 80% -An input file to reproduce the benchmark calculations is linked below. -To run a given case ``-n``, from 1 to 4, execute: +Microelectronics +---------------- - .. code-block:: bash +`ARTEMIS (Adaptive mesh Refinement Time-domain ElectrodynaMIcs Solver) `__ is based on WarpX and couples the Maxwell's equations implementation in WarpX with classical equations that describe quantum material behavior (such as, LLG equation for micromagnetics and London equation for superconducting materials) for quantifying the performance of `next-generation microelectronics `__. - python3 PICMI_inputs_1d.py -n 1 +* `ARTEMIS examples `__ +* `ARTEMIS manual `__ -Once the simulation completes an output file ``avg_ion_density.npy`` will be created which can be compared to the literature results as in the plot above. Running case 1 on 4 processors takes roughly 20 minutes to complete. -* :download:`input file <../../../Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py>` +Nuclear Fusion +-------------- .. note:: - This example needs `additional calibration data for cross sections `__. - Download this data alongside your inputs file and update the paths in the inputs file: - - .. code-block:: bash + TODO - git clone https://github.com/ECP-WarpX/warpx-data.git +Coming soon: -Test cases ----------- +* Microchannel +* Magnetically Confined Plasma with a Single Coil - Magnetic bottle: simple geometry with an external field -PICMI (Python) test cases included that can be used as a reference: - -* :download:`Gaussian beam <../../../Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py>` -* :download:`Langmuir plasma wave test in 3d <../../../Examples/Tests/langmuir/PICMI_inputs_langmuir_rt.py>` -* :download:`Langmuir plasma wave test in RZ <../../../Examples/Tests/langmuir/PICMI_inputs_langmuir_rz_multimode_analyze.py>` -* :download:`Langmuir plasma wave test in 2D <../../../Examples/Tests/langmuir/PICMI_inputs_langmuir2d.py>` - -Manipulating fields via Python ------------------------------- - -An example of using Python to access the simulation charge density, solve the Poisson equation (using ``superLU``) and write the resulting electrostatic potential back to the simulation is given in the input file below. This example uses the ``fields.py`` module included in the ``pywarpx`` library. -* :download:`Direct Poisson solver example <../../../Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py>` - -An example of initializing the fields by accessing their data through Python, advancing the simulation for a chosen number of time steps, and plotting the fields again through Python. The simulation runs with 128 regular cells, 8 guard cells, and 10 PML cells, in each direction. Moreover, it uses div(E) and div(B) cleaning both in the regular grid and in the PML and initializes all available electromagnetic fields (E,B,F,G) identically. - -* :download:`Unit pulse with PML <../../../Examples/Tests/python_wrappers/PICMI_inputs_2d.py>` - -.. _examples-hybrid-model: - -Kinetic-fluid hybrid model +Fundamental Plasma Physics -------------------------- -Several examples and benchmarks of the kinetic-fluid hybrid model are shown below. The first few examples are replications -of the verification tests described in :cite:t:`ex-MUNOZ2018`. -The hybrid-PIC model was added to WarpX in `PR #3665 `_ - the figures below -were generated at that time. - -Electromagnetic modes -^^^^^^^^^^^^^^^^^^^^^ - -In this example a simulation is seeded with a thermal plasma while an initial magnetic field is applied in either the -:math:`z` or :math:`x` direction. The simulation is progressed for a large number of steps and the resulting fields are -analyzed for mode excitations. - -Right and left circularly polarized electromagnetic waves are supported through the cyclotron motion of the ions, except -in a region of thermal resonances as indicated on the plot below. +.. toctree:: + :maxdepth: 1 -.. figure:: https://user-images.githubusercontent.com/40245517/216207688-9c39374a-9e69-45b8-a588-35b087b83d27.png - :alt: Parallel EM modes in thermal ion plasma - :width: 70% + examples/langmuir/README.rst + examples/capacitive_discharge/README.rst -Perpendicularly propagating modes are also supported, commonly referred to as ion-Bernstein modes. +Coming soon: -.. figure:: https://user-images.githubusercontent.com/40245517/231217944-7d12b8d4-af4b-44f8-a1b9-a2b59ce3a1c2.png - :alt: Perpendicular EM modes in thermal ion plasma - :width: 50% +* Expanding Sphere example -The input file for these examples and the corresponding analysis can be found at: - -* :download:`EM modes input <../../../Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_EM_modes/analysis.py>` - -The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the parallel propagating or -ion-Bernstein modes as indicated below. - - .. code-block:: bash - - python3 PICMI_inputs.py -dim {1/2/3} --bdir {x/y/z} - -A RZ-geometry example case for normal modes propagating along an applied magnetic field in a cylinder is also available. -The analytical solution for these modes are described in :cite:t:`ex-Stix1992` Chapter 6, Sec. 2. - -.. figure:: https://user-images.githubusercontent.com/40245517/259251824-33e78375-81d8-410d-a147-3fa0498c66be.png - :alt: Normal EM modes in a metallic cylinder - :width: 90% - -The input file for this example and corresponding analysis can be found at: - -* :download:`Cylinderical modes input <../../../Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_EM_modes/analysis_rz.py>` - -Ion beam R instability -^^^^^^^^^^^^^^^^^^^^^^ - -In this example a low density ion beam interacts with a "core" plasma population which induces an instability. -Based on the relative density between the beam and the core plasma a resonant or non-resonant condition can -be accessed. The figures below show the evolution of the y-component of the magnetic field as the beam and -core plasma interact. - -.. figure:: https://user-images.githubusercontent.com/40245517/217923933-6bdb65cb-7d26-40d8-8687-7dd75274bd48.png - :alt: Resonant ion beam R instability - :width: 70% - -.. figure:: https://user-images.githubusercontent.com/40245517/217925983-b91d6482-69bc-43c1-8c7d-23ebe7c69d49.png - :alt: Non-resonant ion beam R instability - :width: 70% - -The growth rates of the strongest growing modes for the resonant case are compared -to theory (dashed lines) in the figure below. - -.. figure:: https://github.com/ECP-WarpX/WarpX/assets/40245517/a94bb6e5-30e9-4d8f-9e6b-844dc8f51d17 - :alt: Resonant ion beam R instability growth rates - :width: 50% +.. _examples-hybrid-model: -The input file for these examples and the corresponding analysis can be found at: +Kinetic-fluid Hybrid Models +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* :download:`Ion beam R instability input <../../../Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_ion_beam_instability/analysis.py>` +Several examples and benchmarks of the kinetic-fluid hybrid model are shown below. +The first few examples are replications of the verification tests described in :cite:t:`ex-MUNOZ2018`. +The hybrid-PIC model was added to WarpX in `PR #3665 `_ - the figures in the examples below were generated at that time. -The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the resonant or non-resonant -condition as indicated below. +.. toctree:: + :maxdepth: 1 - .. code-block:: bash + examples/ohm_solver_EM_modes/README.rst + examples/ohm_solver_ion_beam_instability/README.rst + examples/ohm_solver_ion_Landau_damping/README.rst - python3 PICMI_inputs.py -dim {1/2/3} --resonant -Ion Landau damping -^^^^^^^^^^^^^^^^^^ +High-Performance Computing and Numerics +--------------------------------------- -Landau damping is a well known process in which electrostatic (acoustic) waves -are damped by transferring energy to particles satisfying a resonance condition. -The process can be simulated by seeding a plasma with a specific acoustic mode -(density perturbation) and tracking the strength of the mode as a function of -time. The figure below shows a set of such simulations with parameters matching -those described in section 4.5 of :cite:t:`ex-MUNOZ2018`. The straight lines show -the theoretical damping rate for the given temperature ratios. +The following examples are commonly used to study the performance of WarpX, e.g., for computing efficiency, scalability, and I/O patterns. +While all prior examples are used for such studies as well, the examples here need less explanation on the physics, less-detail tuning on load balancing, and often simply scale (weak or strong) by changing the number of cells, AMReX block size and number of compute units. -.. figure:: https://user-images.githubusercontent.com/40245517/230523935-3c8d63bd-ee69-4639-b111-f06dad5587f6.png - :alt: Ion Landau damping - :width: 70% +.. toctree:: + :maxdepth: 1 -The input file for these examples and the corresponding analysis can be found at: + examples/uniform_plasma/README.rst -* :download:`Ion Landau damping input <../../../Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py>` -The same input script can be used for 1d, 2d or 3d simulations and to sweep different -temperature ratios. +Manipulating fields via Python +------------------------------ - .. code-block:: bash +.. note:: - python3 PICMI_inputs.py -dim {1/2/3} --temp_ratio {value} + TODO: The section needs to be sorted into either science cases (above) or later sections (workflows and Python API details). -Magnetic reconnection -^^^^^^^^^^^^^^^^^^^^^ +An example of using Python to access the simulation charge density, solve the Poisson equation (using ``superLU``) and write the resulting electrostatic potential back to the simulation is given in the input file below. This example uses the ``fields.py`` module included in the ``pywarpx`` library. -Hybrid-PIC codes are often used to simulate magnetic reconnection in space -plasmas. An example of magnetic reconnection from a force-free sheet is -provided, based on the simulation described in :cite:t:`ex-Le2016`. +* :download:`Direct Poisson solver example <../../../Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py>` -.. figure:: https://user-images.githubusercontent.com/40245517/229639784-b5d3b596-3550-4570-8761-8d9a67aa4b3b.gif - :alt: Magnetic reconnection - :width: 70% +An example of initializing the fields by accessing their data through Python, advancing the simulation for a chosen number of time steps, and plotting the fields again through Python. The simulation runs with 128 regular cells, 8 guard cells, and 10 PML cells, in each direction. Moreover, it uses div(E) and div(B) cleaning both in the regular grid and in the PML and initializes all available electromagnetic fields (E,B,F,G) identically. -The input file for this example and corresponding analysis can be found at: +* :download:`Unit pulse with PML <../../../Examples/Tests/python_wrappers/PICMI_inputs_2d.py>` -* :download:`Magnetic reconnection input <../../../Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py>` Many Further Examples, Demos and Tests -------------------------------------- @@ -261,5 +159,8 @@ WarpX runs over 200 integration tests on a variety of modeling cases, which vali Please see the `Examples/Tests/ `__ directory for many more examples. +Example References +------------------ + .. bibliography:: :keyprefix: ex- diff --git a/Docs/source/usage/examples/capacitive_discharge b/Docs/source/usage/examples/capacitive_discharge new file mode 120000 index 00000000000..dd1493f8f70 --- /dev/null +++ b/Docs/source/usage/examples/capacitive_discharge @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/capacitive_discharge \ No newline at end of file diff --git a/Docs/source/usage/examples/gaussian_beam b/Docs/source/usage/examples/gaussian_beam new file mode 120000 index 00000000000..a03c49d4dda --- /dev/null +++ b/Docs/source/usage/examples/gaussian_beam @@ -0,0 +1 @@ +../../../../Examples/Tests/gaussian_beam \ No newline at end of file diff --git a/Docs/source/usage/examples/langmuir b/Docs/source/usage/examples/langmuir new file mode 120000 index 00000000000..f0f0c5d7bc3 --- /dev/null +++ b/Docs/source/usage/examples/langmuir @@ -0,0 +1 @@ +../../../../Examples/Tests/langmuir \ No newline at end of file diff --git a/Docs/source/usage/examples/laser_ion b/Docs/source/usage/examples/laser_ion new file mode 120000 index 00000000000..8d9aa37f076 --- /dev/null +++ b/Docs/source/usage/examples/laser_ion @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/laser_ion \ No newline at end of file diff --git a/Docs/source/usage/examples/lwfa b/Docs/source/usage/examples/lwfa new file mode 120000 index 00000000000..4ec620fb20e --- /dev/null +++ b/Docs/source/usage/examples/lwfa @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/laser_acceleration \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_EM_modes b/Docs/source/usage/examples/ohm_solver_EM_modes new file mode 120000 index 00000000000..485be7241ae --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_EM_modes @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_EM_modes \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_ion_Landau_damping b/Docs/source/usage/examples/ohm_solver_ion_Landau_damping new file mode 120000 index 00000000000..c70b062da77 --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_ion_Landau_damping @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_ion_Landau_damping \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_ion_beam_instability b/Docs/source/usage/examples/ohm_solver_ion_beam_instability new file mode 120000 index 00000000000..3864ca292b5 --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_ion_beam_instability @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_ion_beam_instability \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_magnetic_reconnection b/Docs/source/usage/examples/ohm_solver_magnetic_reconnection new file mode 120000 index 00000000000..1dd74c99f84 --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_magnetic_reconnection @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_magnetic_reconnection \ No newline at end of file diff --git a/Docs/source/usage/examples/plasma_mirror b/Docs/source/usage/examples/plasma_mirror new file mode 120000 index 00000000000..46f96c3eb8f --- /dev/null +++ b/Docs/source/usage/examples/plasma_mirror @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/plasma_mirror \ No newline at end of file diff --git a/Docs/source/usage/examples/pwfa b/Docs/source/usage/examples/pwfa new file mode 120000 index 00000000000..044974be6ce --- /dev/null +++ b/Docs/source/usage/examples/pwfa @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/plasma_acceleration \ No newline at end of file diff --git a/Docs/source/usage/examples/uniform_plasma b/Docs/source/usage/examples/uniform_plasma new file mode 120000 index 00000000000..fe1c540dfa9 --- /dev/null +++ b/Docs/source/usage/examples/uniform_plasma @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/uniform_plasma \ No newline at end of file diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index dee10651813..a32aa02aa42 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -252,13 +252,13 @@ An important object is ``Simulation.extension.warpx``, which is available during This object is the Python equivalent to the central ``WarpX`` simulation class and provides access to field ``MultiFab`` and ``ParticleContainer`` data. -.. function:: pywarpx.picmi.Simulation.extension.warpx.getistep +.. py:function:: pywarpx.picmi.Simulation.extension.warpx.getistep() -.. function:: pywarpx.picmi.Simulation.extension.warpx.gett_new +.. py:function:: pywarpx.picmi.Simulation.extension.warpx.gett_new() -.. function:: pywarpx.picmi.Simulation.extension.warpx.evolve +.. py:function:: pywarpx.picmi.Simulation.extension.warpx.evolve() -.. autofunction:: pywarpx.picmi.Simulation.extension.finalize +.. autofunction:: pywarpx.picmi.Simulation.extension.finalize() These and other classes are provided through `pyAMReX `__. After the simulation is initialized, pyAMReX can be accessed via @@ -274,13 +274,13 @@ After the simulation is initialized, pyAMReX can be accessed via # for a 3D simulation amr = libwarpx.amr # picks the right 1d, 2d or 3d variant -.. function:: amr.ParallelDescriptor.NProcs() +.. py:function:: amr.ParallelDescriptor.NProcs() -.. function:: amr.ParallelDescriptor.MyProc() +.. py:function:: amr.ParallelDescriptor.MyProc() -.. function:: amr.ParallelDescriptor.IOProcessor() +.. py:function:: amr.ParallelDescriptor.IOProcessor() -.. function:: amr.ParallelDescriptor.IOProcessorNumber() +.. py:function:: amr.ParallelDescriptor.IOProcessorNumber() Particles can be added to the simulation at specific positions and with specific attribute values: @@ -333,7 +333,7 @@ with scraped particle data. The embedded boundary conditions can be modified when using the electrostatic solver. -.. function:: pywarpx.picmi.Simulation.extension.warpx.set_potential_on_eb +.. py:function:: pywarpx.picmi.Simulation.extension.warpx.set_potential_on_eb() Using Python Input as a Preprocessor ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Examples/Physics_applications/capacitive_discharge/README.md b/Examples/Physics_applications/capacitive_discharge/README.md deleted file mode 100644 index f5653b2831a..00000000000 --- a/Examples/Physics_applications/capacitive_discharge/README.md +++ /dev/null @@ -1,7 +0,0 @@ -The examples in this directory are based on the benchmark cases from Turner et -al. (Phys. Plasmas 20, 013507, 2013). -See the 'Examples' section in the documentation for previously computed -comparisons between WarpX and the literature results. -The 1D PICMI input file can be used to reproduce the results from Turner et al. -for a given case, N, by executing: - `python3 PICMI_inputs_1d.py -n N` diff --git a/Examples/Physics_applications/capacitive_discharge/README.rst b/Examples/Physics_applications/capacitive_discharge/README.rst new file mode 100644 index 00000000000..708b4528cd5 --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/README.rst @@ -0,0 +1,54 @@ +.. _examples-capacitive-discharge: + +Capacitive Discharge +==================== + +The examples in this directory are based on the benchmark cases from Turner et al. (Phys. Plasmas 20, 013507, 2013) :cite:p:`ex-Turner2013`. + +The Monte-Carlo collision (MCC) model can be used to simulate electron and ion collisions with a neutral background gas. +In particular this can be used to study capacitive discharges between parallel plates. +The implementation has been tested against the benchmark results from :cite:t:`ex-Turner2013`. + +.. note:: + + This example needs `additional calibration data for cross sections `__. + Download this data alongside your inputs file and update the paths in the inputs file: + + .. code-block:: bash + + git clone https://github.com/ECP-WarpX/warpx-data.git + + +Run +--- + +The 1D PICMI input file can be used to reproduce the results from Turner et al. for a given case, ``N`` from 1 to 4, by executing ``python3 PICMI_inputs_1d.py -n N``, e.g., + +.. code-block:: bash + + python3 PICMI_inputs_1d.py -n 1 + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: PICMI_inputs_1d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py``. + + +Analyze +------- + +Once the simulation completes an output file ``avg_ion_density.npy`` will be created which can be compared to the literature results as in the plot below. +Running case ``1`` on four CPU processors takes roughly 20 minutes to complete. + + +Visualize +--------- + +The figure below shows a comparison of the ion density as calculated in WarpX (in June 2022 with `PR #3118 `_) compared to the literature results (which can be found `in the supplementary materials of Turner et al. `__). + +.. figure:: https://user-images.githubusercontent.com/40245517/171573007-f7d733c7-c0de-490c-9ed6-ff4c02154358.png + :alt: MCC benchmark against :cite:t:`ex-Turner2013`. + :width: 80% + + MCC benchmark against :cite:t:`ex-Turner2013`. diff --git a/Examples/Physics_applications/laser_acceleration/README.md b/Examples/Physics_applications/laser_acceleration/README.md deleted file mode 100644 index 01a87fdce8c..00000000000 --- a/Examples/Physics_applications/laser_acceleration/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Examples of laser acceleration simulations. - -Examples are provided using the executable or Python-driven version of WarpX. - -## Using the executable version: - - inputs.2d: 2d simulation of LWFA in the laboratory frame. - - inputs.3d: 3d simulation of LWFA in the laboratory frame. - - inputs.2d.boost: 2d simulation of LWFA in a boosted frame. This script also - uses rigid injection and a parser for the plasma density. - -## Using the python-driven version: - - laser_acceleration_PICMI.py diff --git a/Examples/Physics_applications/laser_acceleration/README.rst b/Examples/Physics_applications/laser_acceleration/README.rst new file mode 100644 index 00000000000..5b5ef4e7e43 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/README.rst @@ -0,0 +1,94 @@ +.. _examples-lwfa: + +Laser-Wakefield Acceleration of Electrons +========================================= + +This example shows how to model a laser-wakefield accelerator (LWFA) :cite:p:`ex-TajimaDawson1982,ex-Esarey1996`. + +Laser-wakefield acceleration is best performed in 3D and quasi-cylindrical (RZ) geometry, which ensures that the plasma wavelength of the wakefield is modelled with the right scale lengths. +RZ modeling enables efficient modeling if effects of asymmetry shall be ignored (e.g., asymmetric beams and transverse profiles, hosing of the injected beam, etc.). + +For LWFA scenarios with long propagation lengths, use the :ref:`boosted frame method `. +An example can be seen in the :ref:`PWFA example `. + + +Run +--- + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: 3D + + This example can be run **either** as: + + * **Python** script: ``python3 PICMI_inputs_3d.py`` or + * WarpX **executable** using an input file: ``warpx.3d inputs_3d max_step=400`` + + .. tab-set:: + + .. tab-item:: Python: Script + + .. literalinclude:: PICMI_inputs_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py``. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_3d + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_3d``. + + .. tab-item:: RZ + + This example can be run **either** as: + + * **Python** script: ``python3 PICMI_inputs_rz.py`` or + * WarpX **executable** using an input file: ``warpx.rz inputs_3d max_step=400`` + + .. tab-set:: + + .. tab-item:: Python: Script + + .. literalinclude:: PICMI_inputs_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py``. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_rz + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_rz``. + +Analyze +------- + +.. note:: + + This section is TODO. + +We run the following script to analyze correctness: + +.. dropdown:: Script ``analysis_3d.py`` + + .. literalinclude:: analysis_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/analysis_3d.py``. + + +Visualize +--------- + +You can run the following script to visualize the beam evolution over time: + +.. dropdown:: Script ``plot_3d.py`` + + .. literalinclude:: plot_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/plot_3d.py diags/diag1000400/``. + +.. figure:: https://user-images.githubusercontent.com/1353258/287800852-f994a020-4ecc-4987-bffc-2cb7df6144a9.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTEiLCJleHAiOjE3MDE3MTYyNjksIm5iZiI6MTcwMTcxNTk2OSwicGF0aCI6Ii8xMzUzMjU4LzI4NzgwMDg1Mi1mOTk0YTAyMC00ZWNjLTQ5ODctYmZmYy0yY2I3ZGY2MTQ0YTkucG5nP1gtQW16LUFsZ29yaXRobT1BV1M0LUhNQUMtU0hBMjU2JlgtQW16LUNyZWRlbnRpYWw9QUtJQUlXTkpZQVg0Q1NWRUg1M0ElMkYyMDIzMTIwNCUyRnVzLWVhc3QtMSUyRnMzJTJGYXdzNF9yZXF1ZXN0JlgtQW16LURhdGU9MjAyMzEyMDRUMTg1MjQ5WiZYLUFtei1FeHBpcmVzPTMwMCZYLUFtei1TaWduYXR1cmU9NDkyNWJkMTg2NWM3ZjcwZjVkMjlmNDE1NmRjNWEyZWM5MzgxMWJhZTVjMGMxNjdkZDg1Zjk0NmQ1NGEwMjNiMiZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmYWN0b3JfaWQ9MCZrZXlfaWQ9MCZyZXBvX2lkPTAifQ.C_NQceQcqiCDzBoSzIjm3c8QdTLNDdtjJmkQjkhW4c8 + :alt: (top) Electric field of the laser pulse and (bottom) absolute density. + + (top) Electric field of the laser pulse and (bottom) absolute density. diff --git a/Examples/Physics_applications/laser_acceleration/plot_3d.py b/Examples/Physics_applications/laser_acceleration/plot_3d.py new file mode 100755 index 00000000000..00222ff43c8 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/plot_3d.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Axel Huebl +# License: BSD-3-Clause-LBNL +# +# This is a script plots the wakefield of an LWFA simulation. + +import sys + +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + + +def plot_lwfa(): + # this will be the name of the plot file + fn = sys.argv[1] + + # Read the file + ds = yt.load(fn) + + # plot the laser field and absolute density + fields = ["Ey", "rho"] + normal = "y" + sl = yt.SlicePlot(ds, normal=normal, fields=fields) + for field in fields: + sl.set_log(field, False) + + sl.set_figure_size((4, 8)) + fig = sl.export_to_mpl_figure(nrows_ncols=(2, 1)) + fig.tight_layout() + plt.show() + +if __name__ == "__main__": + plot_lwfa() diff --git a/Examples/Physics_applications/laser_ion/README.rst b/Examples/Physics_applications/laser_ion/README.rst new file mode 100644 index 00000000000..25e25f70d84 --- /dev/null +++ b/Examples/Physics_applications/laser_ion/README.rst @@ -0,0 +1,65 @@ +.. _examples-laser-ion: + +Laser-Ion Acceleration with a Planar Target +=========================================== + +This example shows how to model laser-ion acceleration with planar targets of solid density :cite:p:`ex-Wilks2001,Bulanov2008,ex-Macchi2013`. +The acceleration mechanism in this scenario depends on target parameters + +Although laser-ion acceleration requires full 3D modeling for adequate description of the acceleration dynamics, especially the acceleration field lengths and decay times, this example models a 2D example. +2D modeling can often hint at a qualitative overview of the dynamics, but mostly saves computational costs since the plasma frequency (and Debye length) of the plasma determines the resolution need in laser-solid interaction modeling. + +.. note:: + + TODO: The Python (PICMI) input file needs to be created. + +.. note:: + + The resolution of this 2D case is extremely low by default. + You will need a computing cluster for adequate resolution of the target density, see comments in the input file. + +.. warning:: + + It is strongly advised to set the parameters ``.zmin / zmax / xmin / ...`` when working with highly dense targets that are limited in one or multiple dimensions. + The particle creation routine will first create particles everywhere between these limits (`defaulting to box size if unset`), setting particles to invalid only afterwards based on the density profile. + Not setting these parameters can quickly lead to memory overflows. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: (*TODO*) or +* WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created following the ``inputs_2d`` file. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/inputs_2d``. + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Physics_applications/laser_ion/inputs b/Examples/Physics_applications/laser_ion/inputs_2d similarity index 100% rename from Examples/Physics_applications/laser_ion/inputs rename to Examples/Physics_applications/laser_ion/inputs_2d diff --git a/Examples/Physics_applications/plasma_acceleration/README.rst b/Examples/Physics_applications/plasma_acceleration/README.rst new file mode 100644 index 00000000000..508df75c6e7 --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/README.rst @@ -0,0 +1,70 @@ +.. _examples-pwfa: + +Beam-Driven Wakefield Acceleration of Electrons +=============================================== + +This example shows how to model a beam-driven plasma-wakefield accelerator (PWFA) :cite:p:`TajimaDawson1982,Esarey1996`. + +PWFA is best performed in 3D and quasi-cylindrical (RZ) geometry, which ensures that the plasma wavelength of the wakefield is modelled with the right scale lengths. +RZ modeling enables efficient modeling if effects of asymmetry shall be ignored (e.g., asymmetric beams and transverse profiles, hosing of the injected beam, etc.). + +Additionally, to speed up computation, this example uses the :ref:`boosted frame method ` to effectively model long acceleration lengths. + +Alternatively, an other common approximation for PWFAs is quasi-static modeling, e.g., if effects such as self-injection can be ignored. +In the Beam, Plasma & Accelerator Simulation Toolkit (BLAST), `HiPACE++ `__ provides such methods. + +.. note:: + + TODO: The Python (PICMI) input file should use the boosted frame method, like the ``inputs_3d_boost`` file. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: ``python3 PICMI_inputs_plasma_acceleration.py`` or +* WarpX **executable** using an input file: ``warpx.3d inputs_3d_boost`` + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should use the boosted frame method, like the ``inputs_3d_boost`` file. + + .. literalinclude:: PICMI_inputs_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py``. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_3d + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/inputs_3d_boost``. + +Analyze +------- + +.. note:: + + This section is TODO. + +We run the following script to analyze correctness: + +.. dropdown:: Script ``analysis_3d.py`` + + .. literalinclude:: analysis_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/analysis_3d.py``. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Physics_applications/plasma_mirror/README.rst b/Examples/Physics_applications/plasma_mirror/README.rst new file mode 100644 index 00000000000..ca3d6627c42 --- /dev/null +++ b/Examples/Physics_applications/plasma_mirror/README.rst @@ -0,0 +1,58 @@ +.. _examples-plasma-mirror: + +Plasma-Mirror +============= + +This example shows how to model a plasma mirror, using a planar target of solid density :cite:p:`Dromey2004,Roedel2010`. + +Although laser-solid interaction modeling requires full 3D modeling for adequate description of the dynamics at play, this example models a 2D example. +2D modeling provide a qualitative overview of the dynamics, but mostly saves computational costs since the plasma frequency (and Debye length) of the surface plasma determines the resolution need in laser-solid interaction modeling. + +.. note:: + + TODO: The Python (PICMI) input file needs to be created. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: (*TODO*) or +* WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created, like the ``inputs`` file. + + .. literalinclude:: PICMI_plasma_mirror_2d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/plasma_mirror/PICMI_plasma_mirror_2d.py``. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/plasma_mirror/inputs_2d``. + + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Physics_applications/uniform_plasma/README.rst b/Examples/Physics_applications/uniform_plasma/README.rst new file mode 100644 index 00000000000..1719f57f589 --- /dev/null +++ b/Examples/Physics_applications/uniform_plasma/README.rst @@ -0,0 +1,73 @@ +.. _examples-uniform-plasma: + +Uniform Plasma +============== + +This example evolves a uniformly distributed, hot plasma over time. + + +Run +--- + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: 3D + + .. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created following the ``inputs_3d`` file. + + .. literalinclude:: PICMI_inputs_3d.py + :language: python3 + :caption: You can copy this file from ``usage/examples/lwfa/PICMI_inputs_3d.py``. + + .. tab-item:: Executable: Input File + + This example can be run **either** as WarpX **executable** using an input file: ``warpx.3d inputs_3d`` + + .. literalinclude:: inputs_3d + :language: ini + :caption: You can copy this file from ``usage/examples/lwfa/inputs_3d``. + + .. tab-item:: 2D + + .. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created following the ``inputs_2d`` file. + + .. literalinclude:: PICMI_inputs_2d.py + :language: python3 + :caption: You can copy this file from ``usage/examples/lwfa/PICMI_inputs_2d.py``. + + .. tab-item:: Executable: Input File + + This example can be run **either** as WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``usage/examples/lwfa/inputs_2d``. + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Tests/gaussian_beam/README.rst b/Examples/Tests/gaussian_beam/README.rst new file mode 100644 index 00000000000..bfca2bb2398 --- /dev/null +++ b/Examples/Tests/gaussian_beam/README.rst @@ -0,0 +1,47 @@ +.. _examples-gaussian-beam: + +Gaussian Beam +============= + +This example initializes a Gaussian beam distribution. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: ``python3 PICMI_inputs_gaussian_beam.py`` or +* WarpX **executable** using an input file: (*TODO*) + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. literalinclude:: PICMI_inputs_gaussian_beam.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py``. + + .. tab-item:: Executable: Input File + + .. note:: + + TODO: This input file should be created following the ``PICMI_inputs_gaussian_beam.py`` file. + + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Tests/langmuir/README.md b/Examples/Tests/langmuir/README.md deleted file mode 100644 index febf280d490..00000000000 --- a/Examples/Tests/langmuir/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Examples of Langmuir oscillations in a uniform plasma in 1D, 2D, 3D, and RZ - -In each case, a uniform plasma is setup with a sinusoidal perturbation in the -electron momentum along each axis. The plasma is followed for a short period -of time, long enough so that E fields develop. The resulting fields can be -compared to the analytic solutions. - -# Input files (for C++ version) - - inputs_1d - inputs_2d - inputs_3d - inputs_rz - -# Input files (for Python version) - - PICMI_inputs_2d.py - PICMI_inputs_3d.py - PICMI_inputs_rz.py - -# Analysis scripts to check the results - - analysis_1d.py - analysis_2d.py - analysis_3d.py - analysis_rz.py diff --git a/Examples/Tests/langmuir/README.rst b/Examples/Tests/langmuir/README.rst new file mode 100644 index 00000000000..60c0018744c --- /dev/null +++ b/Examples/Tests/langmuir/README.rst @@ -0,0 +1,146 @@ +.. _examples-langmuir: + +Langmuir Waves +============== + +These are examples of Plasma oscillations (`Langmuir waves `__) in a uniform plasma in 1D, 2D, 3D, and RZ. + +In each case, a uniform plasma is setup with a sinusoidal perturbation in the electron momentum along each axis. +The plasma is followed for a short period of time, long enough so that E fields develop. +The resulting fields can be compared to the analytic solutions. + + +Run +--- + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: 3D + + .. tab-set:: + + .. tab-item:: Python: Script + + This example can be run as a **Python** script: ``python3 PICMI_inputs_3d.py``. + + .. literalinclude:: PICMI_inputs_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_3d.py``. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.3d inputs_3d`` + + .. literalinclude:: inputs_3d + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_3d``. + + .. tab-item:: 2D + + .. tab-set:: + + .. tab-item:: Python: Script + + This example can be run as a **Python** script: ``python3 PICMI_inputs_2d.py``. + + .. literalinclude:: PICMI_inputs_2d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_2d.py``. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_2d``. + + + .. tab-item:: RZ + + .. tab-set:: + + .. tab-item:: Python: Script + + This example can be run as a **Python** script: ``python3 PICMI_inputs_rz.py``. + + .. literalinclude:: PICMI_inputs_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_rz.py``. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.rz inputs_rz`` + + .. literalinclude:: inputs_rz + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_rz``. + + + .. tab-item:: 1D + + .. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created, like the ``inputs_1d`` file. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.1d inputs_1d`` + + .. literalinclude:: inputs_1d + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_1d``. + + +Analyze +------- + +We run the following script to analyze correctness: + +.. tab-set:: + + .. tab-item:: 3D + + .. dropdown:: Script ``analysis_3d.py`` + + .. literalinclude:: analysis_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_3d.py``. + + .. tab-item:: 2D + + .. dropdown:: Script ``analysis_2d.py`` + + .. literalinclude:: analysis_2d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_2d.py``. + + .. tab-item:: RZ + + .. dropdown:: Script ``analysis_rz.py`` + + .. literalinclude:: analysis_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_rz.py``. + + .. tab-item:: 1D + + .. dropdown:: Script ``analysis_1d.py`` + + .. literalinclude:: analysis_1d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_1d.py``. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Tests/ohm_solver_EM_modes/README.rst b/Examples/Tests/ohm_solver_EM_modes/README.rst new file mode 100644 index 00000000000..a45cecb3c88 --- /dev/null +++ b/Examples/Tests/ohm_solver_EM_modes/README.rst @@ -0,0 +1,45 @@ +.. _examples-ohm-solver-em-modes: + +Ohm Solver: Electromagnetic modes +================================= + +In this example a simulation is seeded with a thermal plasma while an initial magnetic field is applied in either the +:math:`z` or :math:`x` direction. The simulation is progressed for a large number of steps and the resulting fields are +analyzed for mode excitations. + +Right and left circularly polarized electromagnetic waves are supported through the cyclotron motion of the ions, except +in a region of thermal resonances as indicated on the plot below. + +.. figure:: https://user-images.githubusercontent.com/40245517/216207688-9c39374a-9e69-45b8-a588-35b087b83d27.png + :alt: Parallel EM modes in thermal ion plasma + :width: 70% + +Perpendicularly propagating modes are also supported, commonly referred to as ion-Bernstein modes. + +.. figure:: https://user-images.githubusercontent.com/40245517/231217944-7d12b8d4-af4b-44f8-a1b9-a2b59ce3a1c2.png + :alt: Perpendicular EM modes in thermal ion plasma + :width: 50% + +The input file for these examples and the corresponding analysis can be found at: + +* :download:`EM modes input ` +* :download:`Analysis script ` + +The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the parallel propagating or +ion-Bernstein modes as indicated below. + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --bdir {x/y/z} + +A RZ-geometry example case for normal modes propagating along an applied magnetic field in a cylinder is also available. +The analytical solution for these modes are described in :cite:t:`ex-Stix1992` Chapter 6, Sec. 2. + +.. figure:: https://user-images.githubusercontent.com/40245517/259251824-33e78375-81d8-410d-a147-3fa0498c66be.png + :alt: Normal EM modes in a metallic cylinder + :width: 90% + +The input file for this example and corresponding analysis can be found at: + +* :download:`Cylindrical modes input ` +* :download:`Analysis script ` diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst new file mode 100644 index 00000000000..7e1fc21d935 --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst @@ -0,0 +1,25 @@ +.. _examples-ohm-solver-ion-landau-damping: + +Ohm Solver: Ion Landau Damping +============================== + +Landau damping is a well known process in which electrostatic (acoustic) waves are damped by transferring energy to particles satisfying a resonance condition. +The process can be simulated by seeding a plasma with a specific acoustic mode (density perturbation) and tracking the strength of the mode as a function of time. +The figure below shows a set of such simulations with parameters matching those described in section 4.5 of :cite:t:`ex-MUNOZ2018`. +The straight lines show the theoretical damping rate for the given temperature ratios. + +.. figure:: https://user-images.githubusercontent.com/40245517/230523935-3c8d63bd-ee69-4639-b111-f06dad5587f6.png + :alt: Ion Landau damping + :width: 70% + +The input file for these examples and the corresponding analysis can be found at: + +* :download:`Ion Landau damping input ` +* :download:`Analysis script ` + +The same input script can be used for 1d, 2d or 3d simulations and to sweep different +temperature ratios. + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --temp_ratio {value} diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/README.rst b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst new file mode 100644 index 00000000000..ae98787b799 --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst @@ -0,0 +1,36 @@ +.. _examples-ohm-solver-ion-beam-instability: + +Ohm Solver: Ion Beam R Instability +================================== + +In this example a low density ion beam interacts with a "core" plasma population which induces an instability. +Based on the relative density between the beam and the core plasma a resonant or non-resonant condition can +be accessed. The figures below show the evolution of the y-component of the magnetic field as the beam and +core plasma interact. + +.. figure:: https://user-images.githubusercontent.com/40245517/217923933-6bdb65cb-7d26-40d8-8687-7dd75274bd48.png + :alt: Resonant ion beam R instability + :width: 70% + +.. figure:: https://user-images.githubusercontent.com/40245517/217925983-b91d6482-69bc-43c1-8c7d-23ebe7c69d49.png + :alt: Non-resonant ion beam R instability + :width: 70% + +The growth rates of the strongest growing modes for the resonant case are compared +to theory (dashed lines) in the figure below. + +.. figure:: https://github.com/ECP-WarpX/WarpX/assets/40245517/a94bb6e5-30e9-4d8f-9e6b-844dc8f51d17 + :alt: Resonant ion beam R instability growth rates + :width: 50% + +The input file for these examples and the corresponding analysis can be found at: + +* :download:`Ion beam R instability input ` +* :download:`Analysis script ` + +The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the resonant or non-resonant +condition as indicated below. + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --resonant diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst new file mode 100644 index 00000000000..c31f9915abd --- /dev/null +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst @@ -0,0 +1,44 @@ +.. _examples-ohm-solver-magnetic-reconnection: + +Ohm Solver: Magnetic Reconnection +================================= + +Hybrid-PIC codes are often used to simulate magnetic reconnection in space plasmas. +An example of magnetic reconnection from a force-free sheet is provided, based on the simulation described in :cite:t:`ex-Le2016`. + +Run +--- + +This example can be run as a **Python** script: ``python3 PICMI_inputs.py``. + +.. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py``. + + +Analyze +------- + +We run the following script to analyze correctness: + +.. note:: + + This section is TODO. + + +Visualize +--------- + +You can run the following script to visualize the B-field evolution over time: + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py``. + +.. figure:: https://user-images.githubusercontent.com/40245517/229639784-b5d3b596-3550-4570-8761-8d9a67aa4b3b.gif + :alt: Magnetic reconnection. + :width: 70% + + Magnetic reconnection. diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 191eab3bca2..fed2fcae5a6 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -2162,7 +2162,7 @@ doVis = 0 [LaserIonAcc2d] buildDir = . -inputFile = Examples/Physics_applications/laser_ion/inputs +inputFile = Examples/Physics_applications/laser_ion/inputs_2d runtime_params = amr.n_cell=384 512 max_step=100 dim = 2 addToCompileString = USE_OPENPMD=TRUE From c138a02998919561f120df87fbc21df988d2a197 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 4 Dec 2023 22:51:32 -0800 Subject: [PATCH 017/176] Docs: Fix Many Small Errors & Warnings Mostly file names that changed or were not yet created. --- Docs/source/dataanalysis/ascent.rst | 2 +- Docs/source/developers/fields.rst | 4 +-- Docs/source/developers/particles.rst | 2 +- Docs/source/index.rst | 1 - Docs/source/install/hpc/lassen.rst | 8 +++--- Docs/source/install/hpc/quartz.rst | 4 +-- Docs/source/usage/examples.rst | 1 + .../laser_acceleration/README.rst | 8 ------ .../plasma_acceleration/README.rst | 12 ++------ .../plasma_mirror/README.rst | 6 +--- .../uniform_plasma/README.rst | 8 ------ Python/pywarpx/picmi.py | 28 +++++++++---------- 12 files changed, 28 insertions(+), 56 deletions(-) diff --git a/Docs/source/dataanalysis/ascent.rst b/Docs/source/dataanalysis/ascent.rst index 1fbeeb043e2..6748b111a2f 100644 --- a/Docs/source/dataanalysis/ascent.rst +++ b/Docs/source/dataanalysis/ascent.rst @@ -274,7 +274,7 @@ Example Actions A visualization of the electric field component :math:`E_x` (variable: ``Ex``) with a contour plot and with added particles can be obtained with the following Ascent Action. This action can be used both in replay as well as in situ runs. -.. literalinclude:: examples/Physics_applications/laser_acceleration/ascent_actions.yaml +.. literalinclude:: ../../../Examples/Physics_applications/laser_acceleration/ascent_actions.yaml :language: yaml There are more `Ascent Actions examples available `_ for you to play. diff --git a/Docs/source/developers/fields.rst b/Docs/source/developers/fields.rst index 9ed790e4562..d0af160afef 100644 --- a/Docs/source/developers/fields.rst +++ b/Docs/source/developers/fields.rst @@ -112,9 +112,9 @@ Bilinear filter The multi-pass bilinear filter (applied on the current density) is implemented in ``Source/Filter/``, and class ``WarpX`` holds an instance of this class in member variable ``WarpX::bilinear_filter``. For performance reasons (to avoid creating too many guard cells), this filter is directly applied in communication routines, see ``WarpX::AddCurrentFromFineLevelandSumBoundary`` above and -.. doxygenfunction:: WarpX::ApplyFilterJ(const amrex::Vector, 3>> ¤t, const int lev, const int idim) +.. doxygenfunction:: WarpX::ApplyFilterJ(const amrex::Vector, 3>> ¤t, int lev, int idim) -.. doxygenfunction:: WarpX::SumBoundaryJ(const amrex::Vector, 3>> ¤t, const int lev, const int idim, const amrex::Periodicity &period) +.. doxygenfunction:: WarpX::SumBoundaryJ(const amrex::Vector, 3>> ¤t, int lev, int idim, const amrex::Periodicity &period) Godfrey's anti-NCI filter for FDTD simulations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Docs/source/developers/particles.rst b/Docs/source/developers/particles.rst index e9ad7754771..c1a711e0a01 100644 --- a/Docs/source/developers/particles.rst +++ b/Docs/source/developers/particles.rst @@ -87,7 +87,7 @@ Main functions .. doxygenfunction:: PhysicalParticleContainer::PushPX -.. doxygenfunction:: WarpXParticleContainer::DepositCurrent(amrex::Vector, 3>> &J, const amrex::Real dt, const amrex::Real relative_time) +.. doxygenfunction:: WarpXParticleContainer::DepositCurrent(amrex::Vector, 3>> &J, amrex::Real dt, amrex::Real relative_time) .. note:: The current deposition is used both by ``PhysicalParticleContainer`` and ``LaserParticleContainer``, so it is in the parent class ``WarpXParticleContainer``. diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 7fc2a1c5833..ffceeccad2b 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -79,7 +79,6 @@ Usage usage/parameters usage/python usage/examples - usage/pwfa usage/workflows usage/faq diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index ac15844a5ff..7285bc217f2 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -263,15 +263,15 @@ The batch script below can be used to run a WarpX simulation on 2 nodes on the s Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. Note that the only option so far is to run with one MPI rank per GPU. -.. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen.bsub +.. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100.bsub :language: bash - :caption: You can copy this file from ``Tools/machines/lassen-llnl/lassen.bsub``. + :caption: You can copy this file from ``Tools/machines/lassen-llnl/lassen_v100.bsub``. -To run a simulation, copy the lines above to a file ``lassen.bsub`` and run +To run a simulation, copy the lines above to a file ``lassen_v100.bsub`` and run .. code-block:: bash - bsub lassen.bsub + bsub lassen_v100.bsub to submit the job. diff --git a/Docs/source/install/hpc/quartz.rst b/Docs/source/install/hpc/quartz.rst index de7c5ace848..03860687971 100644 --- a/Docs/source/install/hpc/quartz.rst +++ b/Docs/source/install/hpc/quartz.rst @@ -37,14 +37,14 @@ Create it now: .. code-block:: bash - cp $HOME/src/warpx/Tools/machines/quartz-llnl/quartz/quartz_warpx.profile.example $HOME/quartz_warpx.profile + cp $HOME/src/warpx/Tools/machines/quartz-llnl/quartz_warpx.profile.example $HOME/quartz_warpx.profile .. dropdown:: Script Details :color: light :icon: info :animate: fade-in-slide-down - .. literalinclude:: ../../../../Tools/machines/quartz-llnl/quartz/quartz_warpx.profile.example + .. literalinclude:: ../../../../Tools/machines/quartz-llnl/quartz_warpx.profile.example :language: bash Edit the 2nd line of this script, which sets the ``export proj=""`` variable. diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index dc0a076aaaa..eeee6ddf092 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -23,6 +23,7 @@ Plasma-Based Acceleration examples/lwfa/README.rst examples/pwfa/README.rst + pwfa.rst Coming soon: diff --git a/Examples/Physics_applications/laser_acceleration/README.rst b/Examples/Physics_applications/laser_acceleration/README.rst index 5b5ef4e7e43..737cb1fe677 100644 --- a/Examples/Physics_applications/laser_acceleration/README.rst +++ b/Examples/Physics_applications/laser_acceleration/README.rst @@ -68,14 +68,6 @@ Analyze This section is TODO. -We run the following script to analyze correctness: - -.. dropdown:: Script ``analysis_3d.py`` - - .. literalinclude:: analysis_3d.py - :language: python3 - :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/analysis_3d.py``. - Visualize --------- diff --git a/Examples/Physics_applications/plasma_acceleration/README.rst b/Examples/Physics_applications/plasma_acceleration/README.rst index 508df75c6e7..b6a2b9952c6 100644 --- a/Examples/Physics_applications/plasma_acceleration/README.rst +++ b/Examples/Physics_applications/plasma_acceleration/README.rst @@ -36,13 +36,13 @@ For `MPI-parallel `__ runs, prefix these lines with ` TODO: This input file should use the boosted frame method, like the ``inputs_3d_boost`` file. - .. literalinclude:: PICMI_inputs_3d.py + .. literalinclude:: PICMI_inputs_plasma_acceleration.py :language: python3 :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py``. .. tab-item:: Executable: Input File - .. literalinclude:: inputs_3d + .. literalinclude:: inputs_3d_boost :language: ini :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/inputs_3d_boost``. @@ -53,14 +53,6 @@ Analyze This section is TODO. -We run the following script to analyze correctness: - -.. dropdown:: Script ``analysis_3d.py`` - - .. literalinclude:: analysis_3d.py - :language: python3 - :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/analysis_3d.py``. - Visualize --------- diff --git a/Examples/Physics_applications/plasma_mirror/README.rst b/Examples/Physics_applications/plasma_mirror/README.rst index ca3d6627c42..b115d6fa586 100644 --- a/Examples/Physics_applications/plasma_mirror/README.rst +++ b/Examples/Physics_applications/plasma_mirror/README.rst @@ -29,11 +29,7 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. note:: - TODO: This input file should be created, like the ``inputs`` file. - - .. literalinclude:: PICMI_plasma_mirror_2d.py - :language: python3 - :caption: You can copy this file from ``Examples/Physics_applications/plasma_mirror/PICMI_plasma_mirror_2d.py``. + TODO: This input file should be created following the ``inputs_2d`` file. .. tab-item:: Executable: Input File diff --git a/Examples/Physics_applications/uniform_plasma/README.rst b/Examples/Physics_applications/uniform_plasma/README.rst index 1719f57f589..50d132712c6 100644 --- a/Examples/Physics_applications/uniform_plasma/README.rst +++ b/Examples/Physics_applications/uniform_plasma/README.rst @@ -23,10 +23,6 @@ For `MPI-parallel `__ runs, prefix these lines with ` TODO: This input file should be created following the ``inputs_3d`` file. - .. literalinclude:: PICMI_inputs_3d.py - :language: python3 - :caption: You can copy this file from ``usage/examples/lwfa/PICMI_inputs_3d.py``. - .. tab-item:: Executable: Input File This example can be run **either** as WarpX **executable** using an input file: ``warpx.3d inputs_3d`` @@ -45,10 +41,6 @@ For `MPI-parallel `__ runs, prefix these lines with ` TODO: This input file should be created following the ``inputs_2d`` file. - .. literalinclude:: PICMI_inputs_2d.py - :language: python3 - :caption: You can copy this file from ``usage/examples/lwfa/PICMI_inputs_2d.py``. - .. tab-item:: Executable: Input File This example can be run **either** as WarpX **executable** using an input file: ``warpx.2d inputs_2d`` diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 140bde7efd6..2998cd6747e 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -45,7 +45,7 @@ class constants: class Species(picmistandard.PICMI_Species): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -558,7 +558,7 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): """ This assumes that WarpX was compiled with USE_RZ = TRUE - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters --------- @@ -678,7 +678,7 @@ def initialize_inputs(self): class Cartesian1DGrid(picmistandard.PICMI_Cartesian1DGrid): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters --------- @@ -767,7 +767,7 @@ def initialize_inputs(self): class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters --------- @@ -876,7 +876,7 @@ def initialize_inputs(self): class Cartesian3DGrid(picmistandard.PICMI_Cartesian3DGrid): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters --------- @@ -1004,7 +1004,7 @@ def initialize_inputs(self): class ElectromagneticSolver(picmistandard.PICMI_ElectromagneticSolver): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -1188,7 +1188,7 @@ def initialize_inputs(self): class ElectrostaticSolver(picmistandard.PICMI_ElectrostaticSolver): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -1683,7 +1683,7 @@ def initialize_inputs(self): class Simulation(picmistandard.PICMI_Simulation): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -2110,7 +2110,7 @@ class ParticleFieldDiagnostic: class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -2271,7 +2271,7 @@ class Checkpoint(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): """ Sets up checkpointing of the simulation, allowing for later restarts - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -2310,7 +2310,7 @@ def initialize_inputs(self): class ParticleDiagnostic(picmistandard.PICMI_ParticleDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -2429,7 +2429,7 @@ def initialize_inputs(self): class LabFrameFieldDiagnostic(picmistandard.PICMI_LabFrameFieldDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ + See `Input Parameters `__ for more information. Parameters @@ -2538,7 +2538,7 @@ def initialize_inputs(self): class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ + See `Input Parameters `__ for more information. Parameters @@ -2644,7 +2644,7 @@ class ReducedDiagnostic(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): """ Sets up a reduced diagnostic in the simulation. - See `Input Parameters `_ + See `Input Parameters `__ for more information. Parameters From 2566eaf6726c43bbabdcffd13114def6075faa5c Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 4 Dec 2023 23:03:49 -0800 Subject: [PATCH 018/176] Docs: Fix Formatting in `parameters.rst` More formatting issues in a list. --- Docs/source/usage/parameters.rst | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 4bc8fdfc7cc..9109324620c 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1303,14 +1303,16 @@ Laser initialization ``.e_max`` (i.e. in most cases the maximum of abs(E(x,y,t)) should be 1, so that the maximum field intensity can be set straightforwardly with ``.e_max``). The binary file has to respect the following format: - * flag to indicate the grid is uniform(1 byte, 0 means non-uniform, !=0 means uniform) - only uniform is supported - * nt, number of timesteps (uint32_t, must be >=2) - * nx, number of points along x (uint32_t, must be >=2) - * ny, number of points along y (uint32_t, must be 1 for 2D simulations and >=2 for 3D simulations) - * timesteps (double[2]=[t_min,t_max]) - * x_coords (double[2]=[x_min,x_max]) - * y_coords (double[1] in 2D, double[2]=[y_min,y_max] in 3D) - * field_data (double[nt * nx * ny], with nt being the slowest coordinate). + + * ``flag`` to indicate the grid is uniform (1 byte, 0 means non-uniform, !=0 means uniform) - only uniform is supported + * ``nt``, number of timesteps (``uint32_t``, must be >=2) + * ``nx``, number of points along x (``uint32_t``, must be >=2) + * ``ny``, number of points along y (``uint32_t``, must be 1 for 2D simulations and >=2 for 3D simulations) + * ``timesteps`` (``double[2]=[t_min,t_max]``) + * ``x_coords`` (``double[2]=[x_min,x_max]``) + * ``y_coords`` (``double[1]`` in 2D, ``double[2]=[y_min,y_max]`` in 3D) + * ``field_data`` (``double[nt x nx * ny]``, with ``nt`` being the slowest coordinate). + A binary file can be generated from Python, see an example at ``Examples/Tests/laser_injection_from_file`` * ``.profile_t_peak`` (`float`; in seconds) @@ -1724,7 +1726,7 @@ WarpX provides several particle collision models, using varying degrees of appro See `Higginson et al. (JCP 388, 439-453, 2019) `__ for more details. The default value of ``fusion_multiplier`` is 1. -* ``.fusion_probability_threshold``(`float`) optional. +* ``.fusion_probability_threshold`` (`float`) optional. Only for ``nuclearfusion``. If the fusion multiplier is too high and results in a fusion probability that approaches 1 (for a given collision between two macroparticles), then @@ -2262,8 +2264,8 @@ Additional parameters ``idx_type = {0, 0, 0}``: Sort particles to a cell centered grid ``idx_type = {1, 1, 1}``: Sort particles to a node centered grid ``idx_type = {2, 2, 2}``: Compromise between a cell and node centered grid. - In 2D (XZ and RZ), only the first two elements are read. - In 1D, only the first element is read. + In 2D (XZ and RZ), only the first two elements are read. + In 1D, only the first element is read. * ``warpx.sort_bin_size`` (list of `int`) optional (default ``1 1 1``) If ``sort_intervals`` is activated and ``sort_particles_for_deposition`` is ``false``, particles are sorted in bins of ``sort_bin_size`` cells. From ef4f922f1b33cdd2b2ad0d103e0aef461d7f618c Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Tue, 5 Dec 2023 00:32:51 -0800 Subject: [PATCH 019/176] macOS CI: export CCACHE_DEPEND=1 (#4475) Without it, the hit rate of ccache is very low. --- .github/workflows/macos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index ec1b43a1a30..32a31ea66f8 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -55,6 +55,7 @@ jobs: export CCACHE_COMPRESS=1 export CCACHE_COMPRESSLEVEL=10 export CCACHE_MAXSIZE=100M + export CCACHE_DEPEND=1 ccache -z source py-venv/bin/activate From 431bd9456136b3a09c5aff891bb206b16e015991 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 5 Dec 2023 08:42:52 -0800 Subject: [PATCH 020/176] Doc: Streamline Python Input (#4472) Start to streamline the PICMI Python input. --- Docs/source/index.rst | 5 +- Docs/source/usage/parameters.rst | 19 +- Docs/source/usage/pwfa.rst | 2 + Docs/source/usage/python.rst | 251 +++++------------- Docs/source/usage/workflows.rst | 1 + .../{ => workflows}/domain_decomposition.rst | 2 +- Python/pywarpx/callbacks.py | 85 +++--- Python/pywarpx/particle_containers.py | 9 +- Python/pywarpx/picmi.py | 16 +- 9 files changed, 153 insertions(+), 237 deletions(-) rename Docs/source/usage/{ => workflows}/domain_decomposition.rst (98%) diff --git a/Docs/source/index.rst b/Docs/source/index.rst index ffceeccad2b..4c5f791ca6b 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -75,9 +75,8 @@ Usage :hidden: usage/how_to_run - usage/domain_decomposition - usage/parameters usage/python + usage/parameters usage/examples usage/workflows usage/faq @@ -125,10 +124,10 @@ Development :hidden: developers/contributing - developers/workflows developers/developers developers/doxygen developers/gnumake + developers/workflows developers/faq .. good to have in the future: .. developers/repostructure diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 9109324620c..16502a5ed65 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1,7 +1,11 @@ .. _running-cpp-parameters: -Input Parameters -================ +Parameters: Inputs File +======================= + +This documents on how to use WarpX with an inputs file (e.g., ``warpx.3d input_3d``). + +Complete example input files can be found in :ref:`the examples section `. .. note:: @@ -1084,11 +1088,12 @@ Particle initialization the above mentioned function. .. note:: - When accessing the data via Python, the scraped particle buffer relies on the user - to clear the buffer after processing the data. The - buffer will grow unbounded as particles are scraped and therefore could - lead to memory issues if not periodically cleared. To clear the buffer - call ``warpx_clearParticleBoundaryBuffer()``. + + When accessing the data via Python, the scraped particle buffer relies on the user + to clear the buffer after processing the data. The + buffer will grow unbounded as particles are scraped and therefore could + lead to memory issues if not periodically cleared. To clear the buffer + call ``clear_buffer()``. * ``.do_field_ionization`` (`0` or `1`) optional (default `0`) Do field ionization for this species (using the ADK theory). diff --git a/Docs/source/usage/pwfa.rst b/Docs/source/usage/pwfa.rst index 44d62384699..fc26f5db31a 100644 --- a/Docs/source/usage/pwfa.rst +++ b/Docs/source/usage/pwfa.rst @@ -1,3 +1,5 @@ +.. _examples-pwfa-boost: + In-Depth: PWFA ============== diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index a32aa02aa42..c9d846b41e7 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -1,256 +1,178 @@ .. _usage-picmi: +.. _usage-picmi-run: + +Parameters: Python (PICMI) +========================== -Python (PICMI) -============== +This documents on how to use WarpX as a Python script (e.g., ``python3 PICMI_script.py``). WarpX uses the `PICMI standard `__ for its Python input files. -Python version 3.8 or newer is required. +Complete example input files can be found in :ref:`the examples section `. -Example input files can be found in :ref:`the examples section `. In the input file, instances of classes are created defining the various aspects of the simulation. -The `Simulation` object is the central object, where the instances are passed, -defining the simulation time, field solver, registered species, etc. +A variable of type :py:class:`pywarpx.picmi.Simulation` is the central object to which all other options are passed, defining the simulation time, field solver, registered species, etc. -.. _usage-picmi-parameters: +Once the simulation is fully configured, it can be used in one of two modes. +**Interactive** use is the most common and can be :ref:`extended with custom runtime functionality `: -Classes -------- +.. tab-set:: -Simulation and grid setup -^^^^^^^^^^^^^^^^^^^^^^^^^ + .. tab-item:: Interactive -Simulation -"""""""""" -.. autoclass:: pywarpx.picmi.Simulation - :members: step, add_species, add_laser, write_input_file + :py:meth:`~pywarpx.picmi.Simulation.step`: run directly from Python -Constants -""""""""" -For convenience, the PICMI interface defines the following constants, -which can be used directly inside any PICMI script. The values are in SI units. + .. tab-item:: Preprocessor -- ``picmi.constants.c``: The speed of light in vacuum. -- ``picmi.constants.ep0``: The vacuum permittivity :math:`\epsilon_0` -- ``picmi.constants.mu0``: The vacuum permeability :math:`\mu_0` -- ``picmi.constants.q_e``: The elementary charge (absolute value of the charge of an electron). -- ``picmi.constants.m_e``: The electron mass -- ``picmi.constants.m_p``: The proton mass + :py:meth:`~pywarpx.picmi.Simulation.write_input_file`: create an :ref:`inputs file for a WarpX executable ` -Field solvers define the updates of electric and magnetic fields. -ElectromagneticSolver -""""""""""""""""""""" -.. autoclass:: pywarpx.picmi.ElectromagneticSolver +.. _usage-picmi-parameters: -ElectrostaticSolver -""""""""""""""""""" -.. autoclass:: pywarpx.picmi.ElectrostaticSolver +Simulation and Grid Setup +------------------------- + +.. autoclass:: pywarpx.picmi.Simulation + :members: step, add_species, add_laser, write_input_file -Cartesian3DGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.Cartesian3DGrid -Cartesian2DGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.Cartesian2DGrid -Cartesian1DGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.Cartesian1DGrid -CylindricalGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.CylindricalGrid -EmbeddedBoundary -"""""""""""""""" .. autoclass:: pywarpx.picmi.EmbeddedBoundary +Field solvers define the updates of electric and magnetic fields. + +.. autoclass:: pywarpx.picmi.ElectromagneticSolver + +.. autoclass:: pywarpx.picmi.ElectrostaticSolver + +Constants +--------- + +For convenience, the PICMI interface defines the following constants, +which can be used directly inside any PICMI script. The values are in SI units. + +- ``picmi.constants.c``: The speed of light in vacuum. +- ``picmi.constants.ep0``: The vacuum permittivity :math:`\epsilon_0` +- ``picmi.constants.mu0``: The vacuum permeability :math:`\mu_0` +- ``picmi.constants.q_e``: The elementary charge (absolute value of the charge of an electron). +- ``picmi.constants.m_e``: The electron mass +- ``picmi.constants.m_p``: The proton mass + Applied fields -^^^^^^^^^^^^^^ +-------------- -AnalyticInitialField -"""""""""""""""""""" .. autoclass:: pywarpx.picmi.AnalyticInitialField -ConstantAppliedField -"""""""""""""""""""" .. autoclass:: pywarpx.picmi.ConstantAppliedField -AnalyticAppliedField -"""""""""""""""""""" .. autoclass:: pywarpx.picmi.AnalyticAppliedField -PlasmaLens -"""""""""" .. autoclass:: pywarpx.picmi.PlasmaLens -Mirror -"""""" .. autoclass:: pywarpx.picmi.Mirror Diagnostics -^^^^^^^^^^^ +----------- -ParticleDiagnostic -"""""""""""""""""" .. autoclass:: pywarpx.picmi.ParticleDiagnostic -FieldDiagnostic -""""""""""""""" .. autoclass:: pywarpx.picmi.FieldDiagnostic -ElectrostaticFieldDiagnostic -"""""""""""""""""""""""""""" .. autoclass:: pywarpx.picmi.ElectrostaticFieldDiagnostic +.. autoclass:: pywarpx.picmi.Checkpoint + Lab-frame diagnostics diagnostics are used when running boosted-frame simulations. -LabFrameFieldDiagnostic -""""""""""""""""""""""" .. autoclass:: pywarpx.picmi.LabFrameFieldDiagnostic -Checkpoint -"""""""""" -.. autoclass:: pywarpx.picmi.Checkpoint - Particles -^^^^^^^^^ +--------- Species objects are a collection of particles with similar properties. For instance, background plasma electrons, background plasma ions and an externally injected beam could each be their own particle species. -Species -""""""" .. autoclass:: pywarpx.picmi.Species -MultiSpecies -"""""""""""" .. autoclass:: pywarpx.picmi.MultiSpecies Particle distributions can be used for to initialize particles in a particle species. -GaussianBunchDistribution -""""""""""""""""""""""""" .. autoclass:: pywarpx.picmi.GaussianBunchDistribution -UniformDistribution -""""""""""""""""""" .. autoclass:: pywarpx.picmi.UniformDistribution -AnalyticDistribution -"""""""""""""""""""" .. autoclass:: pywarpx.picmi.AnalyticDistribution -ParticleListDistribution -"""""""""""""""""""""""" .. autoclass:: pywarpx.picmi.ParticleListDistribution Particle layouts determine how to microscopically place macro particles in a grid cell. -GriddedLayout -""""""""""""" .. autoclass:: pywarpx.picmi.GriddedLayout -PseudoRandomLayout -"""""""""""""""""" .. autoclass:: pywarpx.picmi.PseudoRandomLayout -Other operations related to particles +Other operations related to particles: -CoulombCollisions -""""""""""""""""" .. autoclass:: pywarpx.picmi.CoulombCollisions -MCCCollisions -""""""""""""" .. autoclass:: pywarpx.picmi.MCCCollisions -FieldIonization -""""""""""""""" .. autoclass:: pywarpx.picmi.FieldIonization -Lasers -^^^^^^ +Laser Pulses +------------ Laser profiles can be used to initialize laser pulses in the simulation. -GaussianLaser -""""""""""""" .. autoclass:: pywarpx.picmi.GaussianLaser -AnalyticLaser -""""""""""""" .. autoclass:: pywarpx.picmi.AnalyticLaser Laser injectors control where to initialize laser pulses on the simulation grid. -LaserAntenna -"""""""""""" .. autoclass:: pywarpx.picmi.LaserAntenna -.. _usage-picmi-run: - -Running -------- - -WarpX can be run in one of two modes. It can run as a preprocessor, using the -Python input file to generate an input file to be used by the C++ version, or -it can be run directly from Python. -The examples support running in both modes by commenting and uncommenting the appropriate lines. - -In either mode, if using a `virtual environment `__, be sure to activate it before compiling and running WarpX. - - -Running WarpX directly from Python -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -For this, a full Python installation of WarpX is required, as described in :ref:`the install documentation ` (:ref:`developers `). - -In order to run a new simulation: - -* Create a **new directory**, where the simulation will be run. +.. _usage-picmi-extend: -* Add a **Python script** in the directory. +Extending a Simulation from Python +---------------------------------- -The input file should have the line ``sim.step()`` which runs the simulation. +When running WarpX directly from Python it is possible to interact with the simulation. -* **Run** the script with Python: +For instance, with the :py:meth:`~pywarpx.picmi.Simulation.step` method of the simulation class, one could run ``sim.step(nsteps=1)`` in a loop: -.. code-block:: bash +.. code-block:: python3 - mpirun -np python + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... -where ```` is the number of MPI ranks used, and ```` -is the name of the script. + steps = 1000 + for _ in range(steps): + sim.step(nsteps=1) + # do something custom with the sim object -.. _usage-picmi-extend: - -Extending a Simulation from Python -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When running WarpX directly from Python it is possible to interact with the simulation -by installing ``CallbackFunctions``, which will execute a given Python function at a +As a more flexible alternative, one can install `callback functions `__, which will execute a given Python function at a specific location in the WarpX simulation loop. -.. autoclass:: pywarpx.callbacks.CallbackFunctions +.. automodule:: pywarpx.callbacks + :members: installcallback, uninstallcallback, isinstalled -Places in the WarpX loop where callbacks are available include: -``afterinit``, ``beforecollisions``, ``aftercollisions``, ``beforeEsolve``, ``afterEsolve``, -``beforeInitEsolve``, ``afterInitEsolve``, ``beforedeposition``, ``afterdeposition``, -``beforestep``, ``afterstep``, ``afterdiagnostics``,``afterrestart`` and ``oncheckpointsignal``. -See the examples in ``Examples/Tests/ParticleDataPython`` for references on how to use -``callbacks``. +Data Access +^^^^^^^^^^^ -There are several "hooks" available via the ``libwarpx`` shared library to access and manipulate -simulation objects (particles, fields and memory buffers) as well as general properties -(such as processor number). These "hooks" are accessible through the `Simulation.extension` object. +While the simulation is running, callbacks can access the WarpX simulation data *in situ*. -An important object is ``Simulation.extension.warpx``, which is available during simulation run. -This object is the Python equivalent to the central ``WarpX`` simulation class and provides access to -field ``MultiFab`` and ``ParticleContainer`` data. +An important object for data access is ``Simulation.extension.warpx``, which is available only during the simulation run. +This object is the Python equivalent to the C++ ``WarpX`` simulation class and provides access to field ``MultiFab`` and ``ParticleContainer`` data. .. py:function:: pywarpx.picmi.Simulation.extension.warpx.getistep() @@ -282,8 +204,7 @@ After the simulation is initialized, pyAMReX can be accessed via .. py:function:: amr.ParallelDescriptor.IOProcessorNumber() -Particles can be added to the simulation at specific positions and with specific -attribute values: +Particles can be added to the simulation at specific positions and with specific attribute values: .. code-block:: python @@ -295,8 +216,7 @@ attribute values: .. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.add_particles -Properties of the particles already in the simulation can be obtained with various -functions. +Properties of the particles already in the simulation can be obtained with various functions. .. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_particle_count @@ -314,10 +234,8 @@ New components can be added via Python. .. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.add_real_comp Various diagnostics are also accessible from Python. -This includes getting the deposited or total charge density from a given species -as well as accessing the scraped particle buffer. See the example in -``Examples/Tests/ParticleBoundaryScrape`` for a reference on how to interact -with scraped particle data. +This includes getting the deposited or total charge density from a given species as well as accessing the scraped particle buffer. +See the example in ``Examples/Tests/ParticleBoundaryScrape`` for a reference on how to interact with scraped particle data. .. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_species_charge_sum @@ -329,31 +247,8 @@ with scraped particle data. .. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.get_particle_boundary_buffer -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.clearParticleBoundaryBuffer +.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.clear_buffer The embedded boundary conditions can be modified when using the electrostatic solver. .. py:function:: pywarpx.picmi.Simulation.extension.warpx.set_potential_on_eb() - -Using Python Input as a Preprocessor -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In this case, only the pure Python version needs to be installed, as described :ref:`here `. - -In order to run a new simulation: - -* Create a **new directory**, where the simulation will be run. - -* Add a **Python script** in the directory. - -The input file should have the line like ``sim.write_input_file(file_name = 'inputs_from_PICMI')`` -which runs the preprocessor, generating the AMReX inputs file. - -* **Run** the script with Python: - -.. code-block:: bash - - python - -where ```` is the name of the script. -This creates the WarpX input file that you can run as normal with the WarpX executable. diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index 6fefcd9a243..4fe1b9e155a 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -8,6 +8,7 @@ This section collects typical user workflows and best practices for WarpX. .. toctree:: :maxdepth: 2 + workflows/domain_decomposition workflows/debugging workflows/libensemble workflows/plot_timestep_duration diff --git a/Docs/source/usage/domain_decomposition.rst b/Docs/source/usage/workflows/domain_decomposition.rst similarity index 98% rename from Docs/source/usage/domain_decomposition.rst rename to Docs/source/usage/workflows/domain_decomposition.rst index d3df5b78f5c..1340ecc10d9 100644 --- a/Docs/source/usage/domain_decomposition.rst +++ b/Docs/source/usage/workflows/domain_decomposition.rst @@ -66,7 +66,7 @@ and the details of the on-node parallelization and computer architecture used fo Because these parameters put additional constraints on the domain size for a simulation, it can be cumbersome to calculate the number of cells and the physical size of the computational domain for a given resolution. This -:download:`Python script <../../../Tools/DevUtils/compute_domain.py>` does it +:download:`Python script <../../../../Tools/DevUtils/compute_domain.py>` does it automatically. When using the RZ spectral solver, the values of ``amr.max_grid_size`` and ``amr.blocking_factor`` are constrained since the solver diff --git a/Python/pywarpx/callbacks.py b/Python/pywarpx/callbacks.py index b8824d0ef16..11ec7a279de 100644 --- a/Python/pywarpx/callbacks.py +++ b/Python/pywarpx/callbacks.py @@ -1,65 +1,71 @@ -# Copyright 2017-2022 The WarpX Community +# Copyright 2017-2023 The WarpX Community # # This file is part of WarpX. # -# Authors: David Grote, Roelof Groenewald +# Authors: David Grote, Roelof Groenewald, Axel Huebl # # License: BSD-3-Clause-LBNL -"""call back operations -===================== +"""Callback Locations +================== These are the functions which allow installing user created functions so that they are called at various places along the time step. The following three functions allow the user to install, uninstall and verify the different call back types. - - installcallback: Installs a function to be called at that specified time - - uninstallcallback: Uninstalls the function (so it won't be called anymore) - - isinstalled: Checks if the function is installed + +* :py:func:`installcallback`: Installs a function to be called at that specified time +* :py:func:`uninstallcallback`: Uninstalls the function (so it won't be called anymore) +* :py:func:`isinstalled`: Checks if the function is installed These functions all take a callback location name (string) and function or instance method as an argument. Note that if an instance method is used, an extra reference to the method's object is saved. -The install can be done using a decorator, which has the prefix "callfrom". See +The install can be done using a decorator, which has the prefix ``callfrom``. See example below. Functions can be called at the following times: - - beforeInitEsolve: before the initial solve for the E fields (i.e. before the PIC loop starts) - - afterinit: immediately after the init is complete - - beforeEsolve: before the solve for E fields - - poissonsolver: In place of the computePhi call but only in an electrostatic simulation - - afterEsolve: after the solve for E fields - - beforedeposition: before the particle deposition (for charge and/or current) - - afterdeposition: after particle deposition (for charge and/or current) - - beforestep: before the time step - - afterstep: after the time step - - afterdiagnostics: after diagnostic output - - oncheckpointsignal: on a checkpoint signal - - onbreaksignal: on a break signal. These callbacks will be the last ones executed before the simulation ends. - - particlescraper: just after the particle boundary conditions are applied - but before lost particles are processed - - particleloader: at the time that the standard particle loader is called - - particleinjection: called when particle injection happens, after the position - advance and before deposition is called, allowing a user - defined particle distribution to be injected each time step - - appliedfields: allows directly specifying any fields to be applied to the particles - during the advance + +* ``beforeInitEsolve``: before the initial solve for the E fields (i.e. before the PIC loop starts) +* ``afterinit``: immediately after the init is complete +* ``beforeEsolve``: before the solve for E fields +* ``poissonsolver``: In place of the computePhi call but only in an electrostatic simulation +* ``afterEsolve``: after the solve for E fields +* ``beforedeposition``: before the particle deposition (for charge and/or current) +* ``afterdeposition``: after particle deposition (for charge and/or current) +* ``beforestep``: before the time step +* ``afterstep``: after the time step +* ``afterdiagnostics``: after diagnostic output +* ``oncheckpointsignal``: on a checkpoint signal +* ``onbreaksignal``: on a break signal. These callbacks will be the last ones executed before the simulation ends. +* ``particlescraper``: just after the particle boundary conditions are applied + but before lost particles are processed +* ``particleloader``: at the time that the standard particle loader is called +* ``particleinjection``: called when particle injection happens, after the position + advance and before deposition is called, allowing a user + defined particle distribution to be injected each time step +* ``appliedfields``: allows directly specifying any fields to be applied to the particles + during the advance To use a decorator, the syntax is as follows. This will install the function ``myplots`` to be called after each step. -@callfromafterstep -def myplots(): - ppzx() +.. code-block:: python3 + + @callfromafterstep + def myplots(): + ppzx() This is equivalent to the following: -def myplots(): - ppzx() -installcallback('afterstep', myplots) +.. code-block:: python3 + + def myplots(): + ppzx() + installcallback('afterstep', myplots) """ import copy import sys @@ -282,15 +288,20 @@ def callfuncsinlist(self,*args,**kw): callback_instances[key] = CallbackFunctions(name=key, **val) def installcallback(name, f): - "Adds a function to the list of functions called by this callback" + """Installs a function to be called at that specified time. + + Adds a function to the list of functions called by this callback. + """ callback_instances[name].installfuncinlist(f) def uninstallcallback(name, f): - "Removes the function from the list of functions called by this callback" + """Uninstalls the function (so it won't be called anymore). + + Removes the function from the list of functions called by this callback.""" callback_instances[name].uninstallfuncinlist(f) def isinstalled(name, f): - "Checks if the function is called by this callback" + """Checks if a function is installed for this callback.""" return callback_instances[name].isinstalledfuncinlist(f) def clear_all(): diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index cece7968079..84f78a029ad 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -200,7 +200,7 @@ def get_particle_structs(self, level, copy_to_host=False): Note that cupy does not support structs: https://github.com/cupy/cupy/issues/2031 - and will return arrays of binary blobs for the AoS (DP: "|V24"). If copied + and will return arrays of binary blobs for the AoS (DP: ``"|V24"``). If copied to host via copy_to_host, we correct for the right numpy AoS type. Parameters @@ -652,9 +652,10 @@ def getbz(self): def deposit_charge_density(self, level, clear_rho=True, sync_rho=True): - ''' + """ Deposit this species' charge density in rho_fp in order to access that data via pywarpx.fields.RhoFPWrapper(). + Parameters ---------- species_name : str @@ -665,7 +666,7 @@ def deposit_charge_density(self, level, clear_rho=True, sync_rho=True): If True, zero out rho_fp before deposition. sync_rho : bool If True, perform MPI exchange and properly set boundary cells for rho_fp. - ''' + """ rho_fp = libwarpx.warpx.multifab(f'rho_fp[level={level}]') if rho_fp is None: @@ -734,7 +735,7 @@ def get_particle_boundary_buffer_structs( Note that cupy does not support structs: https://github.com/cupy/cupy/issues/2031 - and will return arrays of binary blobs for the AoS (DP: "|V24"). If copied + and will return arrays of binary blobs for the AoS (DP: ``"|V24"``). If copied to host via copy_to_host, we correct for the right numpy AoS type. Parameters diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 2998cd6747e..08f96238703 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -561,7 +561,7 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -681,7 +681,7 @@ class Cartesian1DGrid(picmistandard.PICMI_Cartesian1DGrid): See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -770,7 +770,7 @@ class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -879,7 +879,7 @@ class Cartesian3DGrid(picmistandard.PICMI_Cartesian3DGrid): See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -1828,9 +1828,11 @@ class Simulation(picmistandard.PICMI_Simulation): warpx_sort_idx_type: list of int, optional (default: 0 0 0) This controls the type of grid used to sort the particles when sort_particles_for_deposition is true. Possible values are: - idx_type = {0, 0, 0}: Sort particles to a cell centered grid, - idx_type = {1, 1, 1}: Sort particles to a node centered grid, - idx_type = {2, 2, 2}: Compromise between a cell and node centered grid. + + * idx_type = {0, 0, 0}: Sort particles to a cell centered grid, + * idx_type = {1, 1, 1}: Sort particles to a node centered grid, + * idx_type = {2, 2, 2}: Compromise between a cell and node centered grid. + In 2D (XZ and RZ), only the first two elements are read. In 1D, only the first element is read. warpx_sort_bin_size: list of int, optional (default 1 1 1) From 60516d545689be72859f5608efd03cbad81ea690 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 5 Dec 2023 08:44:18 -0800 Subject: [PATCH 021/176] Rename Depose -> Deposit (#4474) --- Source/Evolve/WarpXEvolve.cpp | 62 +++--- .../Particles/Deposition/ChargeDeposition.H | 8 +- .../Particles/Deposition/CurrentDeposition.H | 32 +-- Source/Particles/PhotonParticleContainer.H | 4 +- Source/Particles/PhotonParticleContainer.cpp | 4 +- Source/Particles/WarpXParticleContainer.H | 4 +- Source/Particles/WarpXParticleContainer.cpp | 188 +++++++++--------- Source/WarpX.H | 4 +- Source/ablastr/particles/DepositCharge.H | 22 +- 9 files changed, 164 insertions(+), 164 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 1515a1b3ff0..8d936d3d4c0 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -188,7 +188,7 @@ WarpX::Evolve (int numsteps) electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) { const bool skip_deposition = true; - PushParticlesandDepose(cur_time, skip_deposition); + PushParticlesandDeposit(cur_time, skip_deposition); } // Electromagnetic case: multi-J algorithm else if (do_multi_J) @@ -425,7 +425,7 @@ WarpX::OneStep_nosub (Real cur_time) ExecutePythonCallback("particlescraper"); ExecutePythonCallback("beforedeposition"); - PushParticlesandDepose(cur_time); + PushParticlesandDeposit(cur_time); ExecutePythonCallback("afterdeposition"); @@ -597,7 +597,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Push particle from x^{n} to x^{n+1} // from p^{n-1/2} to p^{n+1/2} const bool skip_deposition = true; - PushParticlesandDepose(cur_time, skip_deposition); + PushParticlesandDeposit(cur_time, skip_deposition); // Initialize multi-J loop: @@ -639,29 +639,29 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) } // Number of depositions for multi-J scheme - const int n_depose = WarpX::do_multi_J_n_depositions; + const int n_deposit = WarpX::do_multi_J_n_depositions; // Time sub-step for each multi-J deposition - const amrex::Real sub_dt = dt[0] / static_cast(n_depose); + const amrex::Real sub_dt = dt[0] / static_cast(n_deposit); // Whether to perform multi-J depositions on a time interval that spans // one or two full time steps (from n*dt to (n+1)*dt, or from n*dt to (n+2)*dt) - const int n_loop = (WarpX::fft_do_time_averaging) ? 2*n_depose : n_depose; + const int n_loop = (WarpX::fft_do_time_averaging) ? 2*n_deposit : n_deposit; // Loop over multi-J depositions - for (int i_depose = 0; i_depose < n_loop; i_depose++) + for (int i_deposit = 0; i_deposit < n_loop; i_deposit++) { // Move J from new to old if J is linear in time if (J_in_time == JInTime::Linear) PSATDMoveJNewToJOld(); - const amrex::Real t_depose_current = (J_in_time == JInTime::Linear) ? - (i_depose-n_depose+1)*sub_dt : (i_depose-n_depose+0.5_rt)*sub_dt; + const amrex::Real t_deposit_current = (J_in_time == JInTime::Linear) ? + (i_deposit-n_deposit+1)*sub_dt : (i_deposit-n_deposit+0.5_rt)*sub_dt; - const amrex::Real t_depose_charge = (rho_in_time == RhoInTime::Linear) ? - (i_depose-n_depose+1)*sub_dt : (i_depose-n_depose+0.5_rt)*sub_dt; + const amrex::Real t_deposit_charge = (rho_in_time == RhoInTime::Linear) ? + (i_deposit-n_deposit+1)*sub_dt : (i_deposit-n_deposit+0.5_rt)*sub_dt; - // Deposit new J at relative time t_depose_current with time step dt + // Deposit new J at relative time t_deposit_current with time step dt // (dt[0] denotes the time step on mesh refinement level 0) auto& current = (WarpX::do_current_centering) ? current_fp_nodal : current_fp; - mypc->DepositCurrent(current, dt[0], t_depose_current); + mypc->DepositCurrent(current, dt[0], t_deposit_current); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering @@ -678,8 +678,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Move rho from new to old if rho is linear in time if (rho_in_time == RhoInTime::Linear) PSATDMoveRhoNewToRhoOld(); - // Deposit rho at relative time t_depose_charge - mypc->DepositCharge(rho_fp, t_depose_charge); + // Deposit rho at relative time t_deposit_charge + mypc->DepositCharge(rho_fp, t_deposit_charge); // Filter, exchange boundary, and interpolate across levels SyncRho(rho_fp, rho_cp, charge_buf); // Forward FFT of rho @@ -696,9 +696,9 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Advance E,B,F,G fields in time and update the average fields PSATDPushSpectralFields(); - // Transform non-average fields E,B,F,G after n_depose pushes + // Transform non-average fields E,B,F,G after n_deposit pushes // (the relative time reached here coincides with an integer full time step) - if (i_depose == n_depose-1) + if (i_deposit == n_deposit-1) { PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); if (WarpX::do_dive_cleaning) PSATDBackwardTransformF(); @@ -778,7 +778,7 @@ WarpX::OneStep_sub1 (Real curtime) const int coarse_lev = 0; // i) Push particles and fields on the fine patch (first fine step) - PushParticlesandDepose(fine_lev, curtime, DtType::FirstHalf); + PushParticlesandDeposit(fine_lev, curtime, DtType::FirstHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); if (use_filter) ApplyFilterJ(current_fp, fine_lev); @@ -809,7 +809,7 @@ WarpX::OneStep_sub1 (Real curtime) // ii) Push particles on the coarse patch and mother grid. // Push the fields on the coarse patch and mother grid // by only half a coarse step (first half) - PushParticlesandDepose(coarse_lev, curtime, DtType::Full); + PushParticlesandDeposit(coarse_lev, curtime, DtType::Full); StoreCurrent(coarse_lev); AddCurrentFromFineLevelandSumBoundary(current_fp, current_cp, current_buf, coarse_lev); AddRhoFromFineLevelandSumBoundary(rho_fp, rho_cp, charge_buf, coarse_lev, 0, ncomps); @@ -839,7 +839,7 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryAux(guard_cells.ng_UpdateAux); // iv) Push particles and fields on the fine patch (second fine step) - PushParticlesandDepose(fine_lev, curtime+dt[fine_lev], DtType::SecondHalf); + PushParticlesandDeposit(fine_lev, curtime + dt[fine_lev], DtType::SecondHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); if (use_filter) ApplyFilterJ(current_fp, fine_lev); @@ -954,17 +954,17 @@ WarpX::doQEDEvents (int lev) #endif void -WarpX::PushParticlesandDepose (amrex::Real cur_time, bool skip_deposition) +WarpX::PushParticlesandDeposit (amrex::Real cur_time, bool skip_current) { // Evolve particles to p^{n+1/2} and x^{n+1} - // Depose current, j^{n+1/2} + // Deposit current, j^{n+1/2} for (int lev = 0; lev <= finest_level; ++lev) { - PushParticlesandDepose(lev, cur_time, DtType::Full, skip_deposition); + PushParticlesandDeposit(lev, cur_time, DtType::Full, skip_current); } } void -WarpX::PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_deposition) +WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_current) { amrex::MultiFab* current_x = nullptr; amrex::MultiFab* current_y = nullptr; @@ -991,15 +991,15 @@ WarpX::PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type, } mypc->Evolve(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2], + *Efield_aux[lev][0], *Efield_aux[lev][1], *Efield_aux[lev][2], + *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], *current_x, *current_y, *current_z, current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), rho_fp[lev].get(), charge_buf[lev].get(), Efield_cax[lev][0].get(), Efield_cax[lev][1].get(), Efield_cax[lev][2].get(), Bfield_cax[lev][0].get(), Bfield_cax[lev][1].get(), Bfield_cax[lev][2].get(), - cur_time, dt[lev], a_dt_type, skip_deposition); - if (! skip_deposition) { + cur_time, dt[lev], a_dt_type, skip_current); + if (! skip_current) { #ifdef WARPX_DIM_RZ // This is called after all particles have deposited their current and charge. ApplyInverseVolumeScalingToCurrentDensity(current_fp[lev][0].get(), current_fp[lev][1].get(), current_fp[lev][2].get(), lev); @@ -1022,9 +1022,9 @@ WarpX::PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type, #endif if (do_fluid_species) { myfl->Evolve(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2], - rho_fp[lev].get(),*current_x, *current_y, *current_z, cur_time, skip_deposition); + *Efield_aux[lev][0], *Efield_aux[lev][1], *Efield_aux[lev][2], + *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], + rho_fp[lev].get(), *current_x, *current_y, *current_z, cur_time, skip_current); } } } diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index df7097f7600..50ed74439e5 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -27,7 +27,7 @@ since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param rho_fab FArrayBox of charge density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param dx 3D cell size * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. @@ -41,7 +41,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, const int* ion_lev, amrex::FArrayBox& rho_fab, - long np_to_depose, + long np_to_deposit, const std::array& dx, const std::array xyzmin, amrex::Dim3 lo, @@ -95,8 +95,8 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, } #endif amrex::ParallelFor( - np_to_depose, - [=] AMREX_GPU_DEVICE (long ip) { + np_to_deposit, + [=] AMREX_GPU_DEVICE (long ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 2c4e4bd448d..8735366e018 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -37,7 +37,7 @@ since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match @@ -60,7 +60,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jx_fab, amrex::FArrayBox& jy_fab, amrex::FArrayBox& jz_fab, - long np_to_depose, + long np_to_deposit, amrex::Real relative_time, const std::array& dx, const std::array& xyzmin, @@ -126,8 +126,8 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, } #endif amrex::ParallelFor( - np_to_depose, - [=] AMREX_GPU_DEVICE (long ip) { + np_to_deposit, + [=] AMREX_GPU_DEVICE (long ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), @@ -350,7 +350,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level * \param relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, @@ -374,7 +374,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jx_fab, amrex::FArrayBox& jy_fab, amrex::FArrayBox& jz_fab, - long np_to_depose, + long np_to_deposit, const amrex::Real relative_time, const std::array& dx, const std::array& xyzmin, @@ -437,7 +437,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, WARPX_ALWAYS_ASSERT_WITH_MESSAGE(shared_mem_bytes <= max_shared_mem_bytes, "Tile size too big for GPU shared memory current deposition"); - amrex::ignore_unused(np_to_depose); + amrex::ignore_unused(np_to_deposit); // Launch one thread-block per bin amrex::launch( nblocks, threads_per_block, shared_mem_bytes, amrex::Gpu::gpuStream(), @@ -545,7 +545,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, // Note, you should never reach this part of the code. This funcion cannot be called unless // using HIP/CUDA, and those things are checked prior //don't use any args - ignore_unused( GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size); + ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size); WARPX_ABORT_WITH_MESSAGE("Shared memory only implemented for HIP/CUDA"); #endif } @@ -562,7 +562,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level * \param[in] relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, @@ -586,7 +586,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::Array4& Jx_arr, const amrex::Array4& Jy_arr, const amrex::Array4& Jz_arr, - long np_to_depose, + long np_to_deposit, amrex::Real dt, amrex::Real relative_time, const std::array& dx, @@ -656,7 +656,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, } #endif amrex::ParallelFor( - np_to_depose, + np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( @@ -941,7 +941,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, required to have the charge of each macroparticle since \c q is a scalar. For non-ionizable species, \c ion_lev is \c null * \param[in,out] Dx_fab,Dy_fab,Dz_fab FArrayBox of Vay current density, either full array or tile - * \param[in] np_to_depose Number of particles for which current is deposited + * \param[in] np_to_deposit Number of particles for which current is deposited * \param[in] dt Time step for particle level * \param[in] relative_time Time at which to deposit D, relative to the time of the * current positions of the particles. When different than 0, @@ -966,7 +966,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& Dx_fab, amrex::FArrayBox& Dy_fab, amrex::FArrayBox& Dz_fab, - long np_to_depose, + long np_to_deposit, amrex::Real dt, amrex::Real relative_time, const std::array& dx, @@ -982,14 +982,14 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, #if defined(WARPX_DIM_RZ) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_depose, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); + np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in RZ geometry"); #endif #if defined(WARPX_DIM_1D_Z) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_depose, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); + np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in cartesian 1D geometry"); #endif @@ -1054,7 +1054,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, *cost_real = 0._rt; } #endif - amrex::ParallelFor(np_to_depose, [=] AMREX_GPU_DEVICE (long ip) + amrex::ParallelFor(np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index d4498785e49..5a26dadaaa8 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -103,7 +103,7 @@ public: amrex::MultiFab* /*rho*/, int /*icomp*/, const long /*offset*/, - const long /*np_to_depose*/, + const long /*np_to_deposit*/, int /*thread_num*/, int /*lev*/, int /*depos_lev*/) override {} @@ -119,7 +119,7 @@ public: amrex::MultiFab * const /*jy*/, amrex::MultiFab * const /*jz*/, long const /*offset*/, - long const /*np_to_depose*/, + long const /*np_to_deposit*/, int const /*thread_num*/, int const /*lev*/, int const /*depos_lev*/, diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index 96091b9067f..28dfcd8e280 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -242,8 +242,8 @@ PhotonParticleContainer::Evolve (int lev, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, Real t, Real dt, DtType a_dt_type, bool skip_deposition) { - // This does gather, push and depose. - // Push and depose have been re-written for photons + // This does gather, push and deposit. + // Push and deposit have been re-written for photons PhysicalParticleContainer::Evolve (lev, Ex, Ey, Ez, Bx, By, Bz, diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 82dcd720179..5011623bdf8 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -230,7 +230,7 @@ public: amrex::MultiFab* rho, int icomp, long offset, - long np_to_depose, + long np_to_deposit, int thread_num, int lev, int depos_lev); @@ -245,7 +245,7 @@ public: amrex::MultiFab* jy, amrex::MultiFab* jz, long offset, - long np_to_depose, + long np_to_deposit, int thread_num, int lev, int depos_lev, diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 98868d1cf14..d44bc8ad9c3 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -295,22 +295,22 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, } /* \brief Current Deposition for thread thread_num - * \param pti : Particle iterator - * \param wp : Array of particle weights - * \param uxp uyp uzp : Array of particle momenta - * \param ion_lev : Pointer to array of particle ionization level. This is - required to have the charge of each macroparticle - since q is a scalar. For non-ionizable species, - ion_lev is a null pointer. - * \param jx jy jz : Full array of current density - * \param offset : Index of first particle for which current is deposited - * \param np_to_depose: Number of particles for which current is deposited. - Particles [offset,offset+np_tp_depose] deposit current - * \param thread_num : Thread number (if tiling) - * \param lev : Level of box that contains particles - * \param depos_lev : Level on which particles deposit (if buffers are used) - * \param dt : Time step for particle level - * \param relative_time: Time at which to deposit J, relative to the time of the + * \param pti Particle iterator + * \param wp Array of particle weights + * \param uxp uyp uzp Array of particle momenta + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param jx jy jz Full array of current density + * \param offset Index of first particle for which current is deposited + * \param np_to_deposit Number of particles for which current is deposited. + Particles [offset,offset+np_to_deposit] deposit current + * \param thread_num Thread number (if tiling) + * \param lev Level of box that contains particles + * \param depos_lev Level on which particles deposit (if buffers are used) + * \param dt Time step for particle level + * \param relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. @@ -321,7 +321,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, RealVector const & uyp, RealVector const & uzp, int const * const ion_lev, amrex::MultiFab * const jx, amrex::MultiFab * const jy, amrex::MultiFab * const jz, - long const offset, long const np_to_depose, + long const offset, long const np_to_deposit, int const thread_num, const int lev, int const depos_lev, amrex::Real const dt, amrex::Real const relative_time) { @@ -330,7 +330,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, "Deposition buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_depose == 0) return; + if (np_to_deposit == 0) return; // If user decides not to deposit if (do_not_deposit) return; @@ -512,25 +512,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, WARPX_PROFILE_VAR_START(direct_current_dep_kernel); if (WarpX::nox == 1){ doDepositionSharedShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 2){ doDepositionSharedShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 3){ doDepositionSharedShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } WARPX_PROFILE_VAR_STOP(direct_current_dep_kernel); } @@ -540,71 +540,71 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { if (WarpX::nox == 1){ doEsirkepovDepositionShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { if (WarpX::nox == 1){ doVayDepositionShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } else if (WarpX::nox == 2){ doVayDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } else if (WarpX::nox == 3){ doVayDepositionShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } } else { // Direct deposition if (WarpX::nox == 1){ doDepositionShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } else if (WarpX::nox == 2){ doDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } else if (WarpX::nox == 3){ doDepositionShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } } } @@ -662,28 +662,28 @@ WarpXParticleContainer::DepositCurrent ( } /* \brief Charge Deposition for thread thread_num - * \param pti : Particle iterator - * \param wp : Array of particle weights - * \param ion_lev : Pointer to array of particle ionization level. This is - required to have the charge of each macroparticle - since q is a scalar. For non-ionizable species, - ion_lev is a null pointer. - * \param rho : Full array of charge density - * \param icomp : Component of rho into which charge is deposited. - 0: old value (before particle push). - 1: new value (after particle push). - * \param offset : Index of first particle for which charge is deposited - * \param np_to_depose: Number of particles for which charge is deposited. - Particles [offset,offset+np_tp_depose) deposit charge - * \param thread_num : Thread number (if tiling) - * \param lev : Level of box that contains particles - * \param depos_lev : Level on which particles deposit (if buffers are used) + * \param pti Particle iterator + * \param wp Array of particle weights + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param rho Full array of charge density + * \param icomp Component of rho into which charge is deposited. + 0: old value (before particle push). + 1: new value (after particle push). + * \param offset Index of first particle for which charge is deposited + * \param np_to_deposit Number of particles for which charge is deposited. + Particles [offset,offset+np_to_deposit) deposit charge + * \param thread_num Thread number (if tiling) + * \param lev Level of box that contains particles + * \param depos_lev Level on which particles deposit (if buffers are used) */ void WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const int * const ion_lev, amrex::MultiFab* rho, const int icomp, - const long offset, const long np_to_depose, + const long offset, const long np_to_deposit, const int thread_num, const int lev, const int depos_lev) { if (WarpX::do_shared_mem_charge_deposition) @@ -693,7 +693,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, "Deposition buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_depose == 0) return; + if (np_to_deposit == 0) return; // Number of guard cells for local deposition of rho const WarpX& warpx = WarpX::GetInstance(); @@ -888,19 +888,19 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, if (WarpX::nox == 1){ doChargeDepositionSharedShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_depose, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 2){ doChargeDepositionSharedShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_depose, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 3){ doChargeDepositionSharedShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_depose, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); @@ -954,13 +954,13 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noz); ablastr::particles::deposit_charge( - pti, wp, this->charge, ion_lev, - rho, local_rho[thread_num], - WarpX::noz, dx, xyzmin, WarpX::n_rz_azimuthal_modes, - ng_rho, depos_lev, ref_ratio, - offset, np_to_depose, - icomp, nc, - cost, WarpX::load_balance_costs_update_algo, WarpX::do_device_synchronize + pti, wp, this->charge, ion_lev, + rho, local_rho[thread_num], + WarpX::noz, dx, xyzmin, WarpX::n_rz_azimuthal_modes, + ng_rho, depos_lev, ref_ratio, + offset, np_to_deposit, + icomp, nc, + cost, WarpX::load_balance_costs_update_algo, WarpX::do_device_synchronize ); } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 04327752de8..0b116a20d9f 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -761,8 +761,8 @@ public: void doQEDEvents (int lev); #endif - void PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false); - void PushParticlesandDepose ( amrex::Real cur_time, bool skip_current=false); + void PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false); + void PushParticlesandDeposit (amrex::Real cur_time, bool skip_current=false); // This function does aux(lev) = fp(lev) + I(aux(lev-1)-cp(lev)). // Caller must make sure fp and cp have ghost cells filled. diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index eecc56f9acc..821a3e0d9a9 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -40,8 +40,8 @@ namespace ablastr::particles * \param num_rho_deposition_guards number of ghost cells to use for rho (default: rho.nGrowVect()) * \param depos_lev the level to deposit the particles to (default: lev) * \param rel_ref_ratio mesh refinement ratio between lev and depos_lev (default: 1) - * \param offset index to start at when looping over particles to depose (default: 0) - * \param np_to_depose number of particles to depose (default: pti.numParticles()) + * \param offset index to start at when looping over particles to deposit (default: 0) + * \param np_to_deposit number of particles to deposit (default: pti.numParticles()) * \param icomp component in MultiFab to start depositing to * \param nc number of components to deposit * \param cost pointer to (load balancing) cost corresponding to box where present @@ -65,7 +65,7 @@ deposit_charge (typename T_PC::ParIterType& pti, std::optional depos_lev = std::nullopt, std::optional rel_ref_ratio = std::nullopt, long const offset = 0, - std::optional np_to_depose = std::nullopt, + std::optional np_to_deposit = std::nullopt, int const icomp = 0, int const nc = 1, amrex::Real * const AMREX_RESTRICT cost = nullptr, long const load_balance_costs_update_algo = 0, @@ -82,10 +82,10 @@ deposit_charge (typename T_PC::ParIterType& pti, // used for MR when we want to deposit for a subset of the particles on the level in the // current box; with offset, we start at a later particle index - if (!np_to_depose.has_value()) - np_to_depose = pti.numParticles(); - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(np_to_depose.value() + offset <= pti.numParticles(), - "np_to_depose + offset are out-of-bounds for particle iterator"); + if (!np_to_deposit.has_value()) + np_to_deposit = pti.numParticles(); + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(np_to_deposit.value() + offset <= pti.numParticles(), + "np_to_deposit + offset are out-of-bounds for particle iterator"); int const lev = pti.GetLevel(); if (!depos_lev.has_value()) @@ -100,7 +100,7 @@ deposit_charge (typename T_PC::ParIterType& pti, } // If no particles, do not do anything - if (np_to_depose == 0) return; + if (np_to_deposit == 0) return; // Extract deposition order and check that particles shape fits within the guard cells. // NOTE: In specific situations where the staggering of rho and the charge deposition algorithm @@ -179,17 +179,17 @@ deposit_charge (typename T_PC::ParIterType& pti, if (nox == 1){ doChargeDepositionShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_depose.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo); } else if (nox == 2){ doChargeDepositionShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_depose.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo); } else if (nox == 3){ doChargeDepositionShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_depose.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo); } From 0a39c8ae5e75aae75f3570708e6bf9734cab34a1 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 5 Dec 2023 11:31:37 -0800 Subject: [PATCH 022/176] Correct on-axis field boundary for RZ (#4464) * Correct on-axis fields for RZ * Update checksum file * Clean commented-out code --- .../Tests/ohm_solver_EM_modes/analysis_rz.py | 2 +- ...uir_multi_rz_psatd_current_correction.json | 43 ++++--- .../benchmarks_json/LaserAccelerationRZ.json | 110 +++++++++--------- .../Python_LaserAccelerationRZ.json | 110 +++++++++--------- ...ilean_rz_psatd_current_correction_psb.json | 44 +++---- .../WarpXFieldBoundaries.cpp | 107 +++++++++++++++++ Source/BoundaryConditions/WarpX_PEC.H | 18 +++ Source/WarpX.H | 5 + 8 files changed, 284 insertions(+), 155 deletions(-) diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py index 9eb747dd6e8..49746a73a77 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py @@ -154,7 +154,7 @@ def process(it): amps = np.abs(F_kw[2, 1, len(kz)//2-2:len(kz)//2+2]) print("Amplitude sample: ", amps) assert np.allclose( - amps, np.array([61.41633903, 19.39353485, 101.08693342, 11.09248295]) + amps, np.array([61.4170941, 19.39380715, 101.08640009, 11.09261815]) ) if sim.test: diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json index 4f8c01586cd..12f1f37dfd7 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json @@ -1,30 +1,29 @@ { - "electrons": { - "particle_momentum_x": 3.059978628132309e-36, - "particle_momentum_y": 2.003371642665396e-20, - "particle_momentum_z": 2.785324114261674e-20, - "particle_position_x": 0.5290000114309512, - "particle_position_y": 0.5888000690527484, - "particle_theta": 92488.48772168353, - "particle_weight": 81147583679.15044 + "lev=0": { + "Bt": 6.633814882712189, + "Er": 478973170490.2528, + "Ez": 662610386678.0768, + "divE": 4.354742261016806e+17, + "jr": 893973344247781.6, + "jz": 1251505325555575.2, + "rho": 3855770.585538005 }, "ions": { - "particle_momentum_x": 1.4925140714935799e-36, - "particle_momentum_y": 1.4765381278752215e-21, - "particle_momentum_z": 1.9505526940463635e-21, - "particle_position_x": 0.5290000097195765, - "particle_position_y": 0.5887999999975628, + "particle_momentum_x": 9.303284812749536e-37, + "particle_momentum_y": 1.475435436402189e-21, + "particle_momentum_z": 1.9507023826482256e-21, + "particle_position_x": 0.5290000097194892, + "particle_position_y": 0.5887999999995832, "particle_theta": 92488.48772168353, "particle_weight": 81147583679.15044 }, - "lev=0": { - "Bt": 6.632261581203373, - "Er": 478937338832.87415, - "Ez": 662607529111.8489, - "divE": 4.355808435163123e+17, - "jr": 893968704248372.6, - "jz": 1251505252231323.8, - "rho": 3856714.5961512737 + "electrons": { + "particle_momentum_x": 2.5301631422412464e-36, + "particle_momentum_y": 2.0033141618079855e-20, + "particle_momentum_z": 2.785324108949854e-20, + "particle_position_x": 0.5290000111160095, + "particle_position_y": 0.5888000013680398, + "particle_theta": 92488.48772168353, + "particle_weight": 81147583679.15044 } } - diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json index 976a3bdee69..0c6f5242b87 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json @@ -1,64 +1,64 @@ { - "beam": { - "particle_momentum_x": 3.880115499392648e-20, - "particle_momentum_y": 5.07820548292446e-20, - "particle_momentum_z": 1.3503614921828295e-17, - "particle_position_x": 6.242131246165356e-05, - "particle_position_y": 0.0026764363296957524, - "particle_theta": 151.40798444325364, - "particle_weight": 6241509.074460764 + "lev=0": { + "Br": 104965.61239547668, + "Br_0_real": 0.27473110662919303, + "Br_1_imag": 104.10451752777047, + "Br_1_real": 104965.62478916143, + "Bt": 1296.2723279668432, + "Btheta_0_real": 1297.3296772773876, + "Btheta_1_imag": 105725.25398122355, + "Btheta_1_real": 141.255481019691, + "Bz": 5076.744762384442, + "Bz_0_real": 0.4603819957305941, + "Bz_1_imag": 1.007360812967319, + "Bz_1_real": 5076.632351216658, + "Er": 273570494222.5625, + "Er_0_real": 271974091013.7082, + "Er_1_imag": 39530787242966.72, + "Er_1_real": 42616886403.53066, + "Et": 39016542462571.72, + "Etheta_0_real": 112249482.93967965, + "Etheta_1_imag": 33602799006.8748, + "Etheta_1_real": 39016517454881.91, + "Ez": 511653064866.5733, + "Ez_0_real": 496845125310.57947, + "Ez_1_imag": 1245709520854.4875, + "Ez_1_real": 24849976994.356285, + "Jr_0_real": 1264417193501.7915, + "Jr_1_imag": 2.3356633334993878e+17, + "Jr_1_real": 2272922066062.2944, + "Jtheta_0_real": 475304056513.7001, + "Jtheta_1_imag": 1028929701334.7467, + "Jtheta_1_real": 2.1766379427377802e+17, + "Jz_0_real": 1832468408628522.8, + "Jz_1_imag": 556484600934089.9, + "Jz_1_real": 602703622358902.4, + "jr": 1749994884866.3667, + "jt": 2.176637940885753e+17, + "jz": 1954078685186198.5, + "rho": 39314210.88334631, + "rho_0_real": 38889615.839967206, + "rho_1_imag": 21546499.619336687, + "rho_1_real": 2012888.5658883716 }, "electrons": { - "particle_momentum_x": 1.886388623495423e-24, - "particle_momentum_y": 3.989021920324841e-22, - "particle_momentum_z": 1.2499427438675522e-23, + "particle_momentum_x": 1.3203417227476259e-24, + "particle_momentum_y": 4.0070625287284664e-22, + "particle_momentum_z": 1.2493391415020164e-23, "particle_orig_x": 0.026508328457558912, "particle_orig_z": 0.04789125000000001, - "particle_position_x": 0.041602500066712574, - "particle_position_y": 0.047891250456211995, - "particle_theta": 7325.121562757762, + "particle_position_x": 0.04160250006680718, + "particle_position_y": 0.04789125046517409, + "particle_theta": 7325.121266775259, "particle_weight": 813672305.532158 }, - "lev=0": { - "Br_0_real": 0.27473108145012964, - "Br_1_imag": 104.10424416504374, - "Br_1_real": 104965.62622212195, - "Btheta_0_real": 1297.3299824026299, - "Btheta_1_imag": 105725.25637121125, - "Btheta_1_real": 141.25524413452112, - "Br": 104965.6138283532, - "Bt": 1296.2727613183374, - "Bz": 5076.743764997268, - "Bz_0_real": 0.46038147543152824, - "Bz_1_imag": 1.007452397747621, - "Bz_1_real": 5076.631353934757, - "Er_0_real": 271974182110.8858, - "Er_1_imag": 39530787290253.98, - "Er_1_real": 42616765306.284, - "Etheta_0_real": 112249661.08828562, - "Etheta_1_imag": 33602739100.133934, - "Etheta_1_real": 39016517445019.95, - "Er": 273570655229.63535, - "Et": 39016542452492.57, - "Ez": 511653044133.8529, - "Ez_0_real": 496845145101.94775, - "Ez_1_imag": 1245709559726.1033, - "Ez_1_real": 24849961919.57713, - "Jr_0_real": 1264766566288.467, - "Jr_1_imag": 2.335663089152921e+17, - "Jr_1_real": 2273346021690.177, - "Jtheta_0_real": 475301266327.4687, - "Jtheta_1_imag": 1028946515774.2778, - "Jtheta_1_real": 2.176637922654668e+17, - "Jz_0_real": 1832469154489130.8, - "Jz_1_imag": 556484676782123.3, - "Jz_1_real": 602703174081284.9, - "jr": 1750423139263.8418, - "jt": 2.176637920803457e+17, - "jz": 1954078914516329.2, - "rho": 39314213.383921936, - "rho_0_real": 38889619.16177814, - "rho_1_imag": 21546507.958223887, - "rho_1_real": 2012888.0198535046 + "beam": { + "particle_momentum_x": 3.8798910618915136e-20, + "particle_momentum_y": 5.078287631744985e-20, + "particle_momentum_z": 1.3503610556163801e-17, + "particle_position_x": 6.242134237822313e-05, + "particle_position_y": 0.0026764363296840257, + "particle_theta": 151.40797316586125, + "particle_weight": 6241509.074460764 } } diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json index 1c75b69dc9b..d4c8d2b0c4e 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json @@ -1,62 +1,62 @@ { + "lev=0": { + "Br": 142258.01161290862, + "Br_0_real": 0.36356680791862006, + "Br_1_imag": 115.41915541351702, + "Br_1_real": 142258.0180774376, + "Bt": 1301.5691011565148, + "Btheta_0_real": 1299.8813681794945, + "Btheta_1_imag": 143318.04184449295, + "Btheta_1_real": 155.37799428725944, + "Bz": 5993.6411733668865, + "Bz_0_real": 0.47374186122724626, + "Bz_1_imag": 1.140891753831192, + "Bz_1_real": 5993.529100973487, + "Er": 278531478924.55994, + "Er_0_real": 276179480674.09875, + "Er_1_imag": 47911368149173.87, + "Er_1_real": 46900731766.21943, + "Et": 47328876349791.84, + "Etheta_0_real": 135868018.29309198, + "Etheta_1_imag": 36802947213.299355, + "Etheta_1_real": 47328835574237.43, + "Ez": 514006688163.9632, + "Ez_0_real": 499008057521.7594, + "Ez_1_imag": 1565161964565.9727, + "Ez_1_real": 28898941539.846138, + "Jr_0_real": 1458783813352.3413, + "Jr_1_imag": 2.335663323286376e+17, + "Jr_1_real": 2725720908169.8604, + "Jtheta_0_real": 499386783042.54987, + "Jtheta_1_imag": 1179203035377.7158, + "Jtheta_1_real": 2.1766371791828432e+17, + "Jz_0_real": 1832469778455967.8, + "Jz_1_imag": 621923937700173.0, + "Jz_1_real": 660910016274555.4, + "jr": 2108653492279.1746, + "jt": 2.1766371088503258e+17, + "jz": 1954236547127886.2, + "rho": 39480728.028152466, + "rho_0_real": 39055923.162689775, + "rho_1_imag": 21660762.696674928, + "rho_1_real": 2131498.588887921 + }, "beam": { - "particle_momentum_x": 3.880109055649298e-20, - "particle_momentum_y": 5.0781930103830196e-20, - "particle_momentum_z": 1.3503608494680855e-17, - "particle_position_x": 6.242131236443886e-05, - "particle_position_y": 0.0026764363296979446, - "particle_theta": 151.4079870868123, + "particle_momentum_x": 3.8798818966213747e-20, + "particle_momentum_y": 5.078274625117952e-20, + "particle_momentum_z": 1.3503604116563051e-17, + "particle_position_x": 6.242134249270767e-05, + "particle_position_y": 0.0026764363296859625, + "particle_theta": 151.40797595010355, "particle_weight": 6241509.074460764 }, "electrons": { - "particle_momentum_x": 1.787201778017949e-24, - "particle_momentum_y": 3.9234822345143987e-22, - "particle_momentum_z": 1.0100062552925791e-23, - "particle_position_x": 0.041602500069929174, - "particle_position_y": 0.047891250477036906, - "particle_theta": 7325.1193688944695, + "particle_momentum_x": 1.2377776380899228e-24, + "particle_momentum_y": 3.9465600403988994e-22, + "particle_momentum_z": 1.009794959365435e-23, + "particle_position_x": 0.04160250006989259, + "particle_position_y": 0.047891250488567585, + "particle_theta": 7325.119014611929, "particle_weight": 813672305.532158 - }, - "lev=0": { - "Br_0_real": 0.36356649135193925, - "Br_1_imag": 115.41886748920795, - "Br_1_real": 142258.01965536995, - "Btheta_0_real": 1299.8816721733124, - "Btheta_1_imag": 143318.04456658955, - "Btheta_1_real": 155.37774833024366, - "Br": 142258.01319076555, - "Bt": 1301.5695263567557, - "Bz": 5993.640969075834, - "Bz_0_real": 0.4737412745527051, - "Bz_1_imag": 1.1409956493384725, - "Bz_1_real": 5993.528898267216, - "Er_0_real": 276179575540.0639, - "Er_1_imag": 47911367858371.875, - "Er_1_real": 46900598536.03668, - "Etheta_0_real": 135868121.7945822, - "Etheta_1_imag": 36802874200.20133, - "Etheta_1_real": 47328835452079.97, - "Er": 278531658643.55005, - "Et": 47328876227481.125, - "Ez": 514006664374.9789, - "Ez_0_real": 499008075334.7451, - "Ez_1_imag": 1565161989236.6174, - "Ez_1_real": 28898922272.75169, - "Jr_0_real": 1459118139844.9216, - "Jr_1_imag": 2.3356630589200717e+17, - "Jr_1_real": 2726204346551.3022, - "Jtheta_0_real": 499384029970.2145, - "Jtheta_1_imag": 1179215927404.282, - "Jtheta_1_real": 2.17663715880068e+17, - "Jz_0_real": 1832470462501306.8, - "Jz_1_imag": 621924149855721.0, - "Jz_1_real": 660909646259030.1, - "jr": 2109207014985.481, - "jt": 2.1766370884715638e+17, - "jz": 1954236712029783.0, - "rho": 39480730.556067616, - "rho_0_real": 39055926.50167212, - "rho_1_imag": 21660770.34248945, - "rho_1_real": 2131498.060778751 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json b/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json index eddc22348f0..c412bebafb0 100644 --- a/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json +++ b/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bt": 0.0001546688348134161, + "Er": 46301.81902926975, + "Et": 198512.5412869523, + "Ez": 4269.5944463710985, + "divE": 56474.76102581718, + "jr": 5.550914333033371, + "jz": 149.7445144478365, + "rho": 5.000381408055163e-07 + }, "electrons": { - "particle_momentum_x": 7.188314912300234e-22, - "particle_momentum_y": 1.0394436737288987e-23, - "particle_momentum_z": 8.903837852634527e-17, - "particle_position_x": 633733.136187464, - "particle_position_y": 7862152.0078902915, - "particle_theta": 51471.82060104632, + "particle_momentum_x": 7.188316348440827e-22, + "particle_momentum_y": 1.0394552237458006e-23, + "particle_momentum_z": 8.903837851755449e-17, + "particle_position_x": 633733.136190813, + "particle_position_y": 7862152.007883845, + "particle_theta": 51471.82060123148, "particle_weight": 1.0261080645329302e+20 }, "ions": { - "particle_momentum_x": 1.3139399186198985e-18, - "particle_momentum_y": 1.0393368162404926e-23, - "particle_momentum_z": 1.6348806630609593e-13, - "particle_position_x": 633733.1365361367, - "particle_position_y": 7862152.007286081, - "particle_theta": 51471.609916929985, + "particle_momentum_x": 1.3139399189530324e-18, + "particle_momentum_y": 1.0393490021212813e-23, + "particle_momentum_z": 1.6348806630610471e-13, + "particle_position_x": 633733.1365361346, + "particle_position_y": 7862152.007286085, + "particle_theta": 51471.609916929934, "particle_weight": 1.0261080645329302e+20 - }, - "lev=0": { - "Bt": 0.00015467138145729178, - "Er": 46302.55104771345, - "Et": 198512.61435989104, - "Ez": 4269.500411629442, - "divE": 56473.85900924703, - "jr": 5.550691097640847, - "jz": 149.74203244538995, - "rho": 5.000301541815611e-07 } } diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index fc4b563915d..f0a1d406f95 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -41,6 +41,18 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) } } } + +#ifdef WARPX_DIM_RZ + if (patch_type == PatchType::fine) { + ApplyFieldBoundaryOnAxis(get_pointer_Efield_fp(lev, 0), + get_pointer_Efield_fp(lev, 1), + get_pointer_Efield_fp(lev, 2), lev); + } else { + ApplyFieldBoundaryOnAxis(get_pointer_Efield_cp(lev, 0), + get_pointer_Efield_cp(lev, 1), + get_pointer_Efield_cp(lev, 2), lev); + } +#endif } void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_dt_type) @@ -76,6 +88,18 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d WarpX::field_boundary_hi); } } + +#ifdef WARPX_DIM_RZ + if (patch_type == PatchType::fine) { + ApplyFieldBoundaryOnAxis(get_pointer_Bfield_fp(lev, 0), + get_pointer_Bfield_fp(lev, 1), + get_pointer_Bfield_fp(lev, 2), lev); + } else { + ApplyFieldBoundaryOnAxis(get_pointer_Bfield_cp(lev, 0), + get_pointer_Bfield_cp(lev, 1), + get_pointer_Bfield_cp(lev, 2), lev); + } +#endif } void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, @@ -91,6 +115,89 @@ void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, if (PEC::isAnyBoundaryPEC()) PEC::ApplyPECtoJfield(Jx, Jy, Jz, lev, patch_type); } +#ifdef WARPX_DIM_RZ +// Applies the boundary conditions that are specific to the axis when in RZ. +void +WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev) +{ + const amrex::IntVect ngE = get_ng_fieldgather(); + + constexpr int NODE = amrex::IndexType::NODE; + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*Er, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + + amrex::Box const & tilebox = mfi.tilebox(); + + // Lower corner of tile box physical domain + // Note that this is done before the tilebox.grow so that + // these do not include the guard cells. + const std::array& xyzmin = LowerCorner(tilebox, lev, 0._rt); + const amrex::Real rmin = xyzmin[0]; + + // Skip blocks that don't touch the axis + if (rmin > 0._rt) continue; + + amrex::Array4 const& Er_arr = Er->array(mfi); + amrex::Array4 const& Et_arr = Et->array(mfi); + amrex::Array4 const& Ez_arr = Ez->array(mfi); + + amrex::Box tbr = amrex::convert( tilebox, Er->ixType().toIntVect() ); + amrex::Box tbt = amrex::convert( tilebox, Et->ixType().toIntVect() ); + amrex::Box tbz = amrex::convert( tilebox, Ez->ixType().toIntVect() ); + + // For ishift, 1 means cell centered, 0 means node centered + int const ishift_r = (tbr.type(0) != NODE); + int const ishift_t = (tbt.type(0) != NODE); + int const ishift_z = (tbz.type(0) != NODE); + + // Set tileboxes to only include the axis guard cells + // (including the corners in z). + tbr.setRange(0, -ngE[0], ngE[0]); + tbt.setRange(0, -ngE[0], ngE[0]); + tbz.setRange(0, -ngE[0], ngE[0]); + tbr.grow(1, ngE[1]); + tbt.grow(1, ngE[1]); + tbz.grow(1, ngE[1]); + + const int nmodes = n_rz_azimuthal_modes; + + amrex::ParallelFor(tbr, tbt, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/) + { + Er_arr(i,j,0,0) = -Er_arr(-i-ishift_r,j,0,0); + + for (int imode=1 ; imode < nmodes ; imode++) { + Er_arr(i,j,0,2*imode-1) = std::pow(-1._rt, imode+1._rt)*Er_arr(-i-ishift_r,j,0,2*imode-1); + Er_arr(i,j,0,2*imode) = std::pow(-1._rt, imode+1._rt)*Er_arr(-i-ishift_r,j,0,2*imode); + } + }, + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/) + { + Et_arr(i,j,0,0) = -Et_arr(-i-ishift_t,j,0,0); + + for (int imode=1 ; imode < nmodes ; imode++) { + Et_arr(i,j,0,2*imode-1) = std::pow(-1._rt, imode+1._rt)*Et_arr(-i-ishift_t,j,0,2*imode-1); + Et_arr(i,j,0,2*imode) = std::pow(-1._rt, imode+1._rt)*Et_arr(-i-ishift_t,j,0,2*imode); + } + }, + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/) + { + Ez_arr(i,j,0,0) = Ez_arr(-i-ishift_z,j,0,0); + + for (int imode=1 ; imode < nmodes ; imode++) { + Ez_arr(i,j,0,2*imode-1) = -std::pow(-1._rt, imode+1._rt)*Ez_arr(-i-ishift_z,j,0,2*imode-1); + Ez_arr(i,j,0,2*imode) = -std::pow(-1._rt, imode+1._rt)*Ez_arr(-i-ishift_z,j,0,2*imode); + } + + }); + } +} +#endif + void WarpX::ApplyElectronPressureBoundary (const int lev, PatchType patch_type) { if (PEC::isAnyBoundaryPEC()) { diff --git a/Source/BoundaryConditions/WarpX_PEC.H b/Source/BoundaryConditions/WarpX_PEC.H index 3d3e4729d45..79ac8cd7c37 100644 --- a/Source/BoundaryConditions/WarpX_PEC.H +++ b/Source/BoundaryConditions/WarpX_PEC.H @@ -192,6 +192,16 @@ using namespace amrex; GuardCell = true; // tangential components are inverted across PEC boundary if (is_tangent_to_PEC) sign *= -1._rt; +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drEr/dr = 0. + // This only works for the first guard cell and with + // Er cell centered in r. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif } } // is PEC boundary } // loop over iside @@ -320,6 +330,14 @@ using namespace amrex; GuardCell = true; // Sign of the normal component in guard cell is inverted if (is_normal_to_PEC) sign *= -1._rt; +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drBr/dr = 0. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif } } // if PEC Boundary } // loop over sides diff --git a/Source/WarpX.H b/Source/WarpX.H index 0b116a20d9f..3e6078d4f43 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -718,6 +718,11 @@ public: void ApplyEfieldBoundary (int lev, PatchType patch_type); void ApplyBfieldBoundary (int lev, PatchType patch_type, DtType dt_type); +#ifdef WARPX_DIM_RZ + // Applies the boundary conditions that are specific to the axis when in RZ. + void ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev); +#endif + /** * \brief When the Ohm's law solver is used, the electron pressure values * on PEC boundaries are set to enforce a zero derivative Neumann condition, From fa32500a31b89bfc67cf462713337409bed1dd31 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Tue, 5 Dec 2023 15:51:31 -0600 Subject: [PATCH 023/176] Fix typos in comments and docs (#4471) * Fix typos and spelling * More typos * Typos 2023 Nov 27 * 2023 Nov 30 Update * Docs typos * Fix "arbitrary" * 2023 Dec 01 typos * Particles * ablastr - FieldSolver * Filter * Fluids * Docs + Tools * Change back to "lamda" * Fix typo Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * Fix cylindrical --------- Co-authored-by: Axel Huebl Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- CONTRIBUTING.rst | 2 +- Docs/source/developers/checksum.rst | 2 +- Docs/source/developers/profiling.rst | 2 +- Docs/source/developers/python.rst | 4 +-- Docs/source/developers/repo_organization.rst | 2 +- Docs/source/developers/testing.rst | 2 +- Docs/source/developers/warning_logger.rst | 6 ++-- Docs/source/glossary.rst | 2 +- Docs/source/install/hpc/fugaku.rst | 2 +- Docs/source/install/hpc/karolina.rst | 4 +-- Docs/source/latex_theory/AMR/AMR.tex | 2 +- Docs/source/theory/cold_fluid_model.rst | 2 +- .../theory/kinetic_fluid_hybrid_model.rst | 2 +- Docs/source/usage/how_to_run.rst | 2 +- Docs/source/usage/parameters.rst | 12 +++---- .../laser_acceleration/analysis_1d_fluids.py | 2 +- .../analysis_1d_fluids_boosted.py | 2 +- .../analysis_refined_injection.py | 2 +- .../Tests/field_probe/analysis_field_probe.py | 6 ++-- .../analysis_flux_injection_3d.py | 2 +- .../ion_stopping/analysis_ion_stopping.py | 2 +- .../analysis_proton_boron_fusion.py | 2 +- .../PICMI_inputs.py | 2 +- .../PICMI_inputs.py | 2 +- .../analysis.py | 2 +- Examples/Tests/pec/analysis_pec.py | 2 +- Examples/Tests/pec/analysis_pec_mr.py | 2 +- .../analysis_bilinear_filter.py | 2 +- Python/pywarpx/callbacks.py | 2 +- Python/pywarpx/picmi.py | 16 ++++----- Regression/prepare_file_ci.py | 2 +- Source/BoundaryConditions/PMLComponent.H | 2 +- Source/BoundaryConditions/WarpX_PEC.H | 2 +- Source/Diagnostics/BTDiagnostics.cpp | 4 +-- .../BackTransformFunctor.H | 2 +- .../BackTransformParticleFunctor.H | 2 +- .../ComputeParticleDiagFunctor.H | 2 +- Source/Diagnostics/Diagnostics.H | 6 ++-- Source/Diagnostics/FieldIO.H | 6 ++-- .../Diagnostics/ReducedDiags/BeamRelevant.H | 2 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 2 +- .../FieldProbeParticleContainer.cpp | 2 +- .../ReducedDiags/ParticleExtrema.H | 2 +- Source/Diagnostics/SliceDiagnostic.cpp | 6 ++-- Source/Diagnostics/WarpXOpenPMD.cpp | 2 +- Source/FieldSolver/ElectrostaticSolver.cpp | 2 +- .../CylindricalYeeAlgorithm.H | 2 +- .../HybridPICModel/HybridPICModel.H | 2 +- .../MacroscopicProperties.H | 4 +-- .../MacroscopicProperties.cpp | 3 +- .../MagnetostaticSolver.cpp | 8 ++--- .../SpectralSolver/SpectralFieldData.cpp | 2 +- .../SpectralSolver/SpectralFieldDataRZ.cpp | 2 +- .../HankelTransform.cpp | 4 +-- .../SpectralSolver/SpectralKSpace.cpp | 2 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 2 +- Source/FieldSolver/WarpX_QED_K.H | 34 +++++++++---------- Source/Filter/NCIGodfreyFilter.H | 2 +- Source/Filter/NCIGodfreyFilter.cpp | 2 +- Source/Fluids/WarpXFluidContainer.H | 4 +-- Source/Fluids/WarpXFluidContainer.cpp | 16 ++++----- Source/Initialization/InjectorMomentum.H | 12 +++---- Source/Initialization/InjectorPosition.H | 2 +- Source/Initialization/WarpXInitData.cpp | 2 +- Source/Parallelization/GuardCellManager.H | 2 +- Source/Parallelization/WarpXComm.cpp | 6 ++-- .../BinaryCollision/ParticleCreationFunc.cpp | 4 +-- .../Particles/Deposition/CurrentDeposition.H | 2 +- .../Deposition/SharedDepositionUtils.H | 2 +- .../BreitWheelerEngineWrapper.cpp | 2 +- .../QEDInternals/QedWrapperCommons.H | 2 +- .../QEDInternals/QuantumSyncEngineWrapper.cpp | 2 +- Source/Particles/Filter/FilterFunctors.H | 2 +- Source/Particles/MultiParticleContainer.H | 6 ++-- Source/Particles/PhotonParticleContainer.H | 2 +- .../Particles/PhysicalParticleContainer.cpp | 4 +-- Source/Particles/Pusher/PushSelector.H | 2 +- .../Pusher/UpdateMomentumHigueraCary.H | 2 +- .../RigidInjectedParticleContainer.H | 2 +- .../RigidInjectedParticleContainer.cpp | 2 +- Source/Particles/WarpXParticleContainer.H | 2 +- Source/Utils/WarpXAlgorithmSelection.cpp | 2 +- Source/Utils/WarpXMovingWindow.cpp | 2 +- Source/WarpX.H | 2 +- Tools/Algorithms/psatd_pml.ipynb | 2 +- Tools/DevUtils/compute_domain.py | 2 +- .../update_benchmarks_from_azure_output.py | 2 +- Tools/LibEnsemble/run_libensemble_on_warpx.py | 2 +- Tools/PerformanceTests/run_alltests.py | 2 +- Tools/PerformanceTests/run_alltests_1node.py | 2 +- Tools/PerformanceTests/run_automated.py | 2 +- .../plot_distribution_mapping.py | 2 +- Tools/PostProcessing/plot_parallel.py | 2 +- 93 files changed, 159 insertions(+), 158 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index bb3a25e0daf..a3a6a3046a9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -117,7 +117,7 @@ A typical format is: This is a short, 40-character title - After a newline, you can write arbitray paragraphs. You + After a newline, you can write arbitrary paragraphs. You usually limit the lines to 70 characters, but if you don't, then nothing bad will happen. diff --git a/Docs/source/developers/checksum.rst b/Docs/source/developers/checksum.rst index 62a367a4866..4023bc06d50 100644 --- a/Docs/source/developers/checksum.rst +++ b/Docs/source/developers/checksum.rst @@ -9,7 +9,7 @@ The checksum module is located in ``Regression/Checksum/``, and the benchmarks a For more details on the implementation, the Python files in ``Regression/Checksum/`` should be well documented. -From a user point of view, you should only need to use ``checksumAPI.py``. It contains Python functions that can be imported and used from an analysis Python script. It can also be executed directly as a Python script. Here are recipies for the main tasks related to checksum regression tests in WarpX CI. +From a user point of view, you should only need to use ``checksumAPI.py``. It contains Python functions that can be imported and used from an analysis Python script. It can also be executed directly as a Python script. Here are recipes for the main tasks related to checksum regression tests in WarpX CI. Include a checksum regression test in an analysis Python script --------------------------------------------------------------- diff --git a/Docs/source/developers/profiling.rst b/Docs/source/developers/profiling.rst index 8fe68fac817..a1c610eed4a 100644 --- a/Docs/source/developers/profiling.rst +++ b/Docs/source/developers/profiling.rst @@ -247,7 +247,7 @@ node, or without a workload management system. .. note:: To collect full statistics, Nsight-Compute reruns kernels, - temporarilly saving device memory in host memory. This makes it + temporarily saving device memory in host memory. This makes it slower than Nsight-Systems, so the provided script profiles only a single step of a single process. This is generally enough to extract relevant information. diff --git a/Docs/source/developers/python.rst b/Docs/source/developers/python.rst index aa58b0a398a..108f957c2f3 100644 --- a/Docs/source/developers/python.rst +++ b/Docs/source/developers/python.rst @@ -56,10 +56,10 @@ This is part of the standard - each of the PICMI classes call the method ``handl The main purpose is to process application specific keyword arguments (those that start with ``warpx_`` for example). These are then passed into the ``init`` methods. In the WarpX implementation, in the ``init``, each of the WarpX specific arguments are saved as attributes of the implementation -class instancles. +class instances. It is in the second method, ``initialize_inputs``, where the PICMI input parameters are translated into WarpX input parameters. -This method is called later during the intialization. +This method is called later during the initialization. The prefix instances described above are all accessible in the implementation classes (via the ``pywarpx`` module). For each PICMI input quantity, the appropriate WarpX input parameters are set in the prefix classes. As needed, for example in the ``Species`` class, the dynamic prefix instances are created and the attributes set. diff --git a/Docs/source/developers/repo_organization.rst b/Docs/source/developers/repo_organization.rst index 5eaed5fcce5..cb450cc4c95 100644 --- a/Docs/source/developers/repo_organization.rst +++ b/Docs/source/developers/repo_organization.rst @@ -41,7 +41,7 @@ All WarpX header files need to be specified relative to the ``Source/`` director By default, in a ``MyName.cpp`` source file we do not include headers already included in ``MyName.H``. Besides this exception, if a function or a class is used in a source file, the header file containing its declaration must be included, unless the inclusion of a facade header is more appropriate. This is sometimes the case for AMReX headers. For instance ``AMReX_GpuLaunch.H`` is a façade header for ``AMReX_GpuLaunchFunctsC.H`` and ``AMReX_GpuLaunchFunctsG.H``, which -contain respectively the CPU and the GPU implemetation of some methods, and which should not be included directly. +contain respectively the CPU and the GPU implementation of some methods, and which should not be included directly. Whenever possible, forward declarations headers are included instead of the actual headers, in order to save compilation time (see dedicated section below). In WarpX forward declaration headers have the suffix ``*_fwd.H``, while in AMReX they have the suffix ``*Fwd.H``. The include order (see `PR #874 `__ and `PR #1947 `__) and `proper quotation marks `__ are: diff --git a/Docs/source/developers/testing.rst b/Docs/source/developers/testing.rst index 227c2fd97b0..2bd74e862f6 100644 --- a/Docs/source/developers/testing.rst +++ b/Docs/source/developers/testing.rst @@ -25,7 +25,7 @@ For example, if you like to change the compiler to compilation to build on Nvidi branch = development cmakeSetupOpts = -DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON -DWarpX_COMPUTE=CUDA -We also support changing compilation options via the usual :ref:`build enviroment variables `. +We also support changing compilation options via the usual :ref:`build environment variables `. For instance, compiling with ``clang++ -Werror`` would be: .. code-block:: sh diff --git a/Docs/source/developers/warning_logger.rst b/Docs/source/developers/warning_logger.rst index 057aa21bb8e..d878b042502 100644 --- a/Docs/source/developers/warning_logger.rst +++ b/Docs/source/developers/warning_logger.rst @@ -47,15 +47,15 @@ Each entry of warning list respects the following format: .. code-block:: sh * --> [PRIORITY] [TOPIC] [raised COUNTER] - * MULTILINE MESSAGGE - * MULTILINE MESSAGGE + * MULTILINE MESSAGE + * MULTILINE MESSAGE * @ Raised by: WHICH_RANKS where: * ``[PRIORITY]`` can be ``[! ]`` (low priority), ``[!! ]`` (medium priority) or ``[!!!]`` (high priority). It indicates the importance of the warning. * ``[TOPIC]`` indicates which part of the code is concerned by the warning (e.g., particles, laser, parallelization...) -* ``MULTILINE MESSAGGE`` is an arbitrary text message. It can span multiple-lines. Text is wrapped automatically. +* ``MULTILINE MESSAGE`` is an arbitrary text message. It can span multiple-lines. Text is wrapped automatically. * ``COUNTER`` indicates the number of times the warning was raised **across all the MPI ranks**. This means that if we run WarpX with 2048 MPI ranks and each rank raises the same warning once, the displayed message will be ``[raised 2048 times]``. Possible values are ``once``, ``twice``, ``XX times`` * ``WHICH_RANKS`` can be either ``ALL`` or a sequence of rank IDs. It is the list of the MPI ranks which have raised the warning message. diff --git a/Docs/source/glossary.rst b/Docs/source/glossary.rst index b78d2446de8..000b8354e15 100644 --- a/Docs/source/glossary.rst +++ b/Docs/source/glossary.rst @@ -17,7 +17,7 @@ Abbreviations * **AMR:** adaptive mesh-refinement * **BC:** boundary condition (of a simulation) * **BCK:** `Benkler-Chavannes-Kuster `__ method, a stabilization technique for small cells in the electromagnetic solver -* **BTD:** backtransformed diagnosics, a method to collect data for analysis from a *boosted frame* simulation +* **BTD:** backtransformed diagnostics, a method to collect data for analysis from a *boosted frame* simulation * **CEX:** charge-exchange collisions * **CFL:** the Courant-Friedrichs-Lewy condition, a numerical parameter for the numerical convergence of PDE solvers * **CI:** continuous integration, automated tests that we perform before a proposed code-change is accepted; see PR diff --git a/Docs/source/install/hpc/fugaku.rst b/Docs/source/install/hpc/fugaku.rst index 43eb414de09..e70aeefa504 100644 --- a/Docs/source/install/hpc/fugaku.rst +++ b/Docs/source/install/hpc/fugaku.rst @@ -24,7 +24,7 @@ Use the following commands to download the WarpX source code and switch to the c git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx -Compiling WarpX on Fugaku is more pratical on a compute node. Use the following commands to acquire a compute node for one hour: +Compiling WarpX on Fugaku is more practical on a compute node. Use the following commands to acquire a compute node for one hour: .. code-block:: bash diff --git a/Docs/source/install/hpc/karolina.rst b/Docs/source/install/hpc/karolina.rst index 117cc32a0df..7a8bf561986 100644 --- a/Docs/source/install/hpc/karolina.rst +++ b/Docs/source/install/hpc/karolina.rst @@ -17,7 +17,7 @@ If you are new to this system, **please see the following resources**: * `Filesystems `__: * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (25GB default quota) - * ``/scatch/``: `production directory `__; very fast for parallel jobs (20TB default) + * ``/scratch/``: `production directory `__; very fast for parallel jobs (20TB default) .. _building-karolina-preparation: @@ -143,7 +143,7 @@ Use the following :ref:`cmake commands ` to compile the applicat Now, you can :ref:`submit Karolina compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Karolina jobs (:ref:`example inputs `). -For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scatch/``. +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch/``. .. _building-karolina-update: diff --git a/Docs/source/latex_theory/AMR/AMR.tex b/Docs/source/latex_theory/AMR/AMR.tex index 948921eca5e..0e9eace8390 100644 --- a/Docs/source/latex_theory/AMR/AMR.tex +++ b/Docs/source/latex_theory/AMR/AMR.tex @@ -53,7 +53,7 @@ \subsection{Electrostatic} % \caption{Snapshot from a 3D self-consistent simulation of the injector in the High Current Experiment shows the beam emerging from the source at low energy (blue) and being accelerated (green-yellow-orange) and transported in a four quadrupole front end. The automatic layout of the mesh refinement patches from a 2D axisymmetric simulation of the source area shows 2 levels of refinement, concentrating the finer meshes around the emitter (white curve surface) and the beam edge (dark blue).} % \label{fig:ESHCX} %\end{figure} -%Automatic remeshing has been implemented in WarpX following the procedure described in \cite{Vaynim2005}, refining on criteria based on measures of local charge density magnitude and gradients. AMR WarpX simulations were applied to the modeling of the front end injector of the High Current Experiment (HCX) \cite{Prostprstab2005}, and provided the first numerically converged estimates of phase space beam distorsions, which directly affects beam quality \cite{Vaypop04}. Fig.~\ref{fig:ESHCX} shows snapshots from 2D axisymmetric simulation of the souce area illustrating the automatic placement of refined patches, and 3D simulation of the full injector showing the beam generation, acceleration and transport. +%Automatic remeshing has been implemented in WarpX following the procedure described in \cite{Vaynim2005}, refining on criteria based on measures of local charge density magnitude and gradients. AMR WarpX simulations were applied to the modeling of the front end injector of the High Current Experiment (HCX) \cite{Prostprstab2005}, and provided the first numerically converged estimates of phase space beam distorsions, which directly affects beam quality \cite{Vaypop04}. Fig.~\ref{fig:ESHCX} shows snapshots from 2D axisymmetric simulation of the source area illustrating the automatic placement of refined patches, and 3D simulation of the full injector showing the beam generation, acceleration and transport. \subsection{Electromagnetic} The method that is used for electrostatic mesh refinement is not directly applicable to electromagnetic calculations. As was shown in section 3.4 of \cite{Vayjcp01}, refinement schemes relying solely on interpolation between coarse and fine patches lead to the reflection with amplification of the short wavelength modes that fall below the cutoff of the Nyquist frequency of the coarse grid. Unless these modes are damped heavily or prevented from occurring at their source, they may affect particle motion and their effect can escalate if trapped within a patch, via multiple successive reflections with amplification. diff --git a/Docs/source/theory/cold_fluid_model.rst b/Docs/source/theory/cold_fluid_model.rst index 971ebdba062..7e69516270b 100644 --- a/Docs/source/theory/cold_fluid_model.rst +++ b/Docs/source/theory/cold_fluid_model.rst @@ -95,7 +95,7 @@ Step 5: **Current and Charge Deposition** Mesh refinement is not supported for the fluids. - The implemented MUSCL scheme has a simplifed slope averaging, see the extended writeup for details. + The implemented MUSCL scheme has a simplified slope averaging, see the extended writeup for details. More details on the precise implementation are available here, `WarpX_Cold_Rel_Fluids.pdf`_. .. _WarpX_Cold_Rel_Fluids.pdf: https://github.com/ECP-WarpX/WarpX/files/12886437/WarpX_Cold_Rel_Fluids.pdf diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index d70e30a9821..69797352bee 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -134,7 +134,7 @@ Extrapolation step Obtaining the E-field at timestep :math:`t=t_{n+1}` is a well documented issue for the hybrid model. Currently the approach in WarpX is to simply extrapolate -:math:`\vec{J}_i` foward in time, using +:math:`\vec{J}_i` forward in time, using .. math:: diff --git a/Docs/source/usage/how_to_run.rst b/Docs/source/usage/how_to_run.rst index 26c56cb6007..ab5cc6e79bd 100644 --- a/Docs/source/usage/how_to_run.rst +++ b/Docs/source/usage/how_to_run.rst @@ -25,7 +25,7 @@ Where ```` by the actual path to the run directory. ------------- If you installed warpX with a :ref:`package manager `, a ``warpx``-prefixed executable will be available as a regular system command to you. -Depending on the choosen build options, the name is suffixed with more details. +Depending on the chosen build options, the name is suffixed with more details. Try it like this: .. code-block:: bash diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 16502a5ed65..18c1e53ada2 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -155,7 +155,7 @@ Overall simulation parameters This only applies when warpx.do_electrostatic = labframe. * ``warpx.self_fields_verbosity`` (`integer`, default: 2) - The vebosity used for MLMG solver for space-charge fields calculation. Currently + The verbosity used for MLMG solver for space-charge fields calculation. Currently MLMG solver looks for verbosity levels from 0-5. A higher number results in more verbose output. @@ -369,7 +369,7 @@ Domain Boundary Conditions * ``Absorbing``: Particles leaving the boundary will be deleted. - * ``Periodic``: Particles leaving the boundary will re-enter from the opposite boundary. The field boundary condition must be consistenly set to periodic and both lower and upper boundaries must be periodic. + * ``Periodic``: Particles leaving the boundary will re-enter from the opposite boundary. The field boundary condition must be consistently set to periodic and both lower and upper boundaries must be periodic. * ``Reflecting``: Particles leaving the boundary are reflected from the boundary back into the domain. When ``boundary.reflect_all_velocities`` is false, the sign of only the normal velocity is changed, otherwise the sign of all velocities are changed. @@ -861,7 +861,7 @@ Particle initialization ``.ux``, ``.uy`` and ``.uz``, the normalized momenta in the x, y and z direction respectively, which are all ``0.`` by default. - * ``uniform``: uniform probability distribution between a minumum and a maximum value. + * ``uniform``: uniform probability distribution between a minimum and a maximum value. The x, y and z directions are sampled independently and the final momentum space is a cuboid. The parameters that control the minimum and maximum domain of the distribution are ``.u_min`` and ``.u_max`` in each @@ -1190,7 +1190,7 @@ Cold Relativistic Fluid initialization * ``fluids.species_names`` (`strings`, separated by spaces) Defines the names of each fluid species. It is a required input to create and evolve fluid species using the cold relativistic fluid equations. Most of the parameters described in the section "Particle initialization" can also be used to initialize fluid properties (e.g. initial density distribution). - For fluid-specific inputs we use `` as a placeholder. Also see external fields + For fluid-specific inputs we use `` as a placeholder. Also see external fields for how to specify these for fluids as the function names differ. .. _running-cpp-parameters-laser: @@ -1726,7 +1726,7 @@ WarpX provides several particle collision models, using varying degrees of appro total number of physical particle remains the same). This can improve the statistics of the simulation, in the case where fusion reactions are very rare. More specifically, in a fusion reaction between two macroparticles with weight ``w_1`` and ``w_2``, - the weight of the product macroparticles will be ``min(w_1,w_2)/fusion_multipler``. + the weight of the product macroparticles will be ``min(w_1,w_2)/fusion_multiplier``. (And the weights of the reactant macroparticles are reduced correspondingly after the reaction.) See `Higginson et al. (JCP 388, 439-453, 2019) `__ for more details. The default value of ``fusion_multiplier`` is 1. @@ -2299,7 +2299,7 @@ Additional parameters * ``warpx.shared_tilesize`` (list of `int`) optional (default `6 6 8` in 3D; `14 14` in 2D; `1s` otherwise) Used to tune performance when ``do_shared_mem_current_deposition`` or - ``do_shared_mem_charge_depostion`` is enabled. ``shared_tilesize`` is the + ``do_shared_mem_charge_deposition`` is enabled. ``shared_tilesize`` is the size of the temporary buffer allocated in shared memory for a threadblock. A larger tilesize requires more shared memory, but gives more work to each threadblock, which can lead to higher occupancy, and allows for more diff --git a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py index a376ed872a9..a33b82ebc02 100755 --- a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py +++ b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py @@ -90,7 +90,7 @@ def odefcn(phi, xi, kp, a0, c, tau, xi_0, lambda_laser): # Remove the ions rho_th = rho_th - e*n0 -# Dicate which region to compare solutions over +# Dictate which region to compare solutions over # (Currently this is the full domain) min_i = 0 max_i = 10240 diff --git a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py index 4f4ae2a812a..30301996921 100755 --- a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py +++ b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py @@ -90,7 +90,7 @@ def odefcn(phi, xi, kp, a0, c, tau, xi_0, lambda_laser): # Remove the ions rho_th = rho_th - e*n0 -# Dicate which region to compare solutions over (cuttoff 0's from BTD extra) +# Dictate which region to compare solutions over (cuttoff 0's from BTD extra) min_i = 200 max_i = 4864 diff --git a/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py b/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py index e7d26d2cb30..a66c838fe9d 100755 --- a/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py +++ b/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py @@ -43,7 +43,7 @@ n_move = 192 # ref ratio = 2 1 -# Refined only transversly. Longitudinal spacing between particles in each stream is the same in both coarse and fine regions +# Refined only transversely. Longitudinal spacing between particles in each stream is the same in both coarse and fine regions rr_longitudinal = 1 np_expected = (n_coarse + n_fine*rr_longitudinal)*(n_0 + n_move) diff --git a/Examples/Tests/field_probe/analysis_field_probe.py b/Examples/Tests/field_probe/analysis_field_probe.py index e038bfd0f48..e167942d77c 100755 --- a/Examples/Tests/field_probe/analysis_field_probe.py +++ b/Examples/Tests/field_probe/analysis_field_probe.py @@ -9,9 +9,9 @@ """ This script tests the accuracy of the FieldProbe diagnostic by observing a plane wave undergoing single slit diffraction. The input file inputs_2d is used. This -file defines the simulation box, laser pulse, embeded boundary with single slit, +file defines the simulation box, laser pulse, embedded boundary with single slit, and line of detector points. The plane wave initializes near the negative Z end -of the simulation box. The wave interacts with the embeded boundary at Z=0. The +of the simulation box. The wave interacts with the embedded boundary at Z=0. The wave undergoes diffraction at the slit. The electromagnetic flux is calculated at the line detector which is placed perpendicular to Z beyond the slit. This test will check if the detected EM flux matches expected values, @@ -39,7 +39,7 @@ def I_envelope (x, lam = 0.2e-6, a = 0.3e-6, D = 1.7e-6): arg = np.pi * a / lam * np.sin(np.arctan(x / D)) return np.sinc( arg / np.pi )**2 -# Count non-outlyer values away from simulation boundaries +# Count non-outlier values away from simulation boundaries counter = np.arange(60, 140, 2) # Count average error from expected values diff --git a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py index 0998de25a7b..470cbfe6065 100755 --- a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py +++ b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py @@ -17,7 +17,7 @@ After the particles are emitted with flux injection, this script produces histograms of the velocity distribution and compares it with the expected -velocity distibution (Gaussian or Gaussian-flux depending on the direction +velocity distribution (Gaussian or Gaussian-flux depending on the direction of space) """ import os diff --git a/Examples/Tests/ion_stopping/analysis_ion_stopping.py b/Examples/Tests/ion_stopping/analysis_ion_stopping.py index dd9ee0f8744..b090014c0cf 100755 --- a/Examples/Tests/ion_stopping/analysis_ion_stopping.py +++ b/Examples/Tests/ion_stopping/analysis_ion_stopping.py @@ -73,7 +73,7 @@ def stopping_from_ions(dt, ni, Ti, mi, Zi, Zb, ion_mass, ion_energy): ion_energy = (f1)**(2./3.)/e return ion_energy -# Fetch background parameters and inital particle data +# Fetch background parameters and initial particle data ds0 = yt.load(f'{prefix}{len(last_it)*"0"}') ad0 = ds0.all_data() diff --git a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py index 8550b8767c6..eef371589f5 100755 --- a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py +++ b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -521,7 +521,7 @@ def check_initial_energy2(data): # Loop over all slices (i.e. cells in the z direction) for slice_number in range(1, size_z): - ## For simplicity, all the calculations in this functino are done nonrelativistically + ## For simplicity, all the calculations in this function are done nonrelativistically ## Proton kinetic energy in the lab frame before fusion E_proton_nonrelativistic = Energy_step*slice_number**2 ## Corresponding square norm of proton momentum diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py index bd28f46b087..d18c2025e45 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py @@ -2,7 +2,7 @@ # # --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are # --- treated as kinetic particles and electrons as an isothermal, inertialess -# --- background fluid. The script simulates ion Landau damping as descibed +# --- background fluid. The script simulates ion Landau damping as described # --- in section 4.5 of Munoz et al. (2018). import argparse diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py index 79f268f6e3b..7a3b976b092 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py @@ -378,7 +378,7 @@ def text_diag(self): self.prev_step = step def energy_diagnostic(self): - """Diangostic to get the total, magnetic and kinetic energies in the + """Diagnostic to get the total, magnetic and kinetic energies in the simulation.""" step = simulation.extension.warpx.getistep(lev=0) - 1 diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py index b541a83283a..8c5a819a9e0 100755 --- a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py @@ -85,7 +85,7 @@ plt.plot(t_grid, np.abs(field_kt[:, 5] / sim.B0), 'b', label=f'm = 5, $kl_i={k[5]:.2f}$') plt.plot(t_grid, np.abs(field_kt[:, 6] / sim.B0), 'k', label=f'm = 6, $kl_i={k[6]:.2f}$') - # The theoretical growth rates for the 4th, 5th and 6th Foruier modes of + # The theoretical growth rates for the 4th, 5th and 6th Fourier modes of # the By-field was obtained from Fig. 12a of Munoz et al. # Note the rates here are gamma / w_ci gamma4 = 0.1915611861780133 diff --git a/Examples/Tests/pec/analysis_pec.py b/Examples/Tests/pec/analysis_pec.py index 0414420ef91..841cc06ae22 100755 --- a/Examples/Tests/pec/analysis_pec.py +++ b/Examples/Tests/pec/analysis_pec.py @@ -7,7 +7,7 @@ # License: BSD-3-Clause-LBNL # # This is a script that analyses the simulation results from -# the script `inputs_field_PEC_3d`. This simulates a sinusoindal wave. +# the script `inputs_field_PEC_3d`. This simulates a sinusoidal wave. # The electric field (Ey) is a standing wave due to the PEC boundary condition, # and as a result, the minimum and maximum value after reflection would be two times the value at initialization due to constructive interference. # Additionally, the value of Ey at the boundary must be equal to zero. diff --git a/Examples/Tests/pec/analysis_pec_mr.py b/Examples/Tests/pec/analysis_pec_mr.py index 1f9e5276c51..e8aab4dcd6f 100755 --- a/Examples/Tests/pec/analysis_pec_mr.py +++ b/Examples/Tests/pec/analysis_pec_mr.py @@ -7,7 +7,7 @@ # License: BSD-3-Clause-LBNL # # This is a script that analyses the simulation results from -# the script `inputs_field_PEC_mr_3d`. This simulates a sinusoindal wave. +# the script `inputs_field_PEC_mr_3d`. This simulates a sinusoidal wave. # The electric field (Ey) is a standing wave due to the PEC boundary condition, # and as a result, the minimum and maximum value after reflection would be two times the value at initialization due to constructive interference. # Additionally, the value of Ey at the boundary must be equal to zero. diff --git a/Examples/Tests/single_particle/analysis_bilinear_filter.py b/Examples/Tests/single_particle/analysis_bilinear_filter.py index 95ee0761943..db7250dc3bb 100755 --- a/Examples/Tests/single_particle/analysis_bilinear_filter.py +++ b/Examples/Tests/single_particle/analysis_bilinear_filter.py @@ -42,7 +42,7 @@ my_order[1] -= 1 my_filter = my_filterx[:,None]*my_filtery -# Apply filter. my_F_filetered is the theoretical value for filtered field +# Apply filter. my_F_filtered is the theoretical value for filtered field my_F_filtered = signal.convolve2d(my_F_nofilter, my_filter, boundary='symm', mode='same') # Get simulation result for F_filtered diff --git a/Python/pywarpx/callbacks.py b/Python/pywarpx/callbacks.py index 11ec7a279de..1c191e607f4 100644 --- a/Python/pywarpx/callbacks.py +++ b/Python/pywarpx/callbacks.py @@ -87,7 +87,7 @@ class CallbackFunctions(object): original reference. This is good since the user does not need to keep the reference to the object (for example it can be created using a local variable in a function). It may be bad if the user thinks an object was - deleted, but it actually isn't since it had (unkown to the user) + deleted, but it actually isn't since it had (unknown to the user) installed a method in one of the call back lists. """ diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 08f96238703..168ad8508f6 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -79,7 +79,7 @@ class Species(picmistandard.PICMI_Species): Whether or not to push this species warpx_do_not_gather: bool, default=False - Whether or not to gahter the fields from grids for this species + Whether or not to gather the fields from grids for this species warpx_random_theta: bool, default=True Whether or not to add random angle to the particles in theta @@ -1196,7 +1196,7 @@ class ElectrostaticSolver(picmistandard.PICMI_ElectrostaticSolver): Whether to use the relativistic solver or lab frame solver warpx_absolute_tolerance: float, default=0. - Absolute tolerance on the lab fram solver + Absolute tolerance on the lab frame solver warpx_self_fields_verbosity: integer, default=2 Level of verbosity for the lab frame solver @@ -1422,7 +1422,7 @@ def initialize_inputs(self): class CoulombCollisions(picmistandard.base._ClassWithInit): """ - Custom class to handle setup of binary Coulmb collisions in WarpX. If + Custom class to handle setup of binary Coulomb collisions in WarpX. If collision initialization is added to picmistandard this can be changed to inherit that functionality. @@ -1649,13 +1649,13 @@ class PlasmaLens(picmistandard.base._ClassWithInit): The field that is applied depends on the transverse position of the particle, (x,y) - - Ex = x*stengths_E + - Ex = x*strengths_E - - Ey = y*stengths_E + - Ey = y*strengths_E - - Bx = +y*stengths_B + - Bx = +y*strengths_B - - By = -x*stengths_B + - By = -x*strengths_B """ def __init__(self, period, starts, lengths, strengths_E=None, strengths_B=None, **kw): @@ -2702,7 +2702,7 @@ class ReducedDiagnostic(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): reduction_type: {'Maximum', 'Minimum', or 'Integral'} For diagnostic type 'FieldReduction', the type of reduction - probe_geometry: {'Point', 'Line', 'Plane'}, defaut='Point' + probe_geometry: {'Point', 'Line', 'Plane'}, default='Point' For diagnostic type 'FieldProbe', the geometry of the probe integrate: bool, default=false diff --git a/Regression/prepare_file_ci.py b/Regression/prepare_file_ci.py index 1fa6b4c9a0a..d52d05b1139 100644 --- a/Regression/prepare_file_ci.py +++ b/Regression/prepare_file_ci.py @@ -7,7 +7,7 @@ import os # This script modifies `WarpX-test.ini` (which is used for nightly builds) -# and creates the file `ci-test.ini` (which is used for continous +# and creates the file `ci-test.ini` (which is used for continuous # integration) # The subtests that are selected are controlled by WARPX_TEST_DIM # The architecture (CPU/GPU) is selected by WARPX_TEST_ARCH diff --git a/Source/BoundaryConditions/PMLComponent.H b/Source/BoundaryConditions/PMLComponent.H index 6fa4f8af784..83dca1e8b9a 100644 --- a/Source/BoundaryConditions/PMLComponent.H +++ b/Source/BoundaryConditions/PMLComponent.H @@ -9,7 +9,7 @@ /* In WarpX, the split fields of the PML (e.g. Eyx, Eyz) are stored as * components of a MultiFab (e.g. component 0 and 1 of the MultiFab for Ey) - * The correspondance between the component index (0,1) and its meaning + * The correspondence between the component index (0,1) and its meaning * (yx, yz, etc.) is defined in the present file */ struct PMLComp { diff --git a/Source/BoundaryConditions/WarpX_PEC.H b/Source/BoundaryConditions/WarpX_PEC.H index 79ac8cd7c37..ef3d3dc2ad5 100644 --- a/Source/BoundaryConditions/WarpX_PEC.H +++ b/Source/BoundaryConditions/WarpX_PEC.H @@ -147,7 +147,7 @@ using namespace amrex; amrex::GpuArray const& fbndry_lo, amrex::GpuArray const& fbndry_hi ) { - // Tangential Efield componentes in guard cells set equal and opposite to cells + // Tangential Efield components in guard cells set equal and opposite to cells // in the mirror locations across the PEC boundary, whereas normal E-field // components are set equal to values in the mirror locations across the PEC // boundary. Here we just initialize it. diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 52d3656a425..aed62f8e02c 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -348,7 +348,7 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) amrex::RealBox diag_dom; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim ) { // Setting lo-coordinate for the diag domain by taking the max of user-defined - // lo-cordinate and lo-coordinat of the simulation domain at level, lev + // lo-cordinate and lo-coordinate of the simulation domain at level, lev diag_dom.setLo(idim, std::max(m_lo[idim],warpx.Geom(lev).ProbLo(idim)) ); // Setting hi-coordinate for the diag domain by taking the max of user-defined // hi-cordinate and hi-coordinate of the simulation domain at level, lev @@ -787,7 +787,7 @@ BTDiagnostics::PrepareFieldDataForOutput () auto & warpx = WarpX::GetInstance(); // In this function, we will get cell-centered data for every level, lev, - // using the cell-center functors and their respective opeators() + // using the cell-center functors and their respective operators() // Call m_cell_center_functors->operator for (int lev = 0; lev < nmax_lev; ++lev) { int icomp_dst = 0; diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H index ef61d96f0aa..bef40ae1ce0 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H @@ -58,7 +58,7 @@ public: * The data is then lorentz-transformed in-place using LorenzTransformZ (). * The user-requested fields are then copied to mf_dst. * - * \param[out] mf_dst output MuliFab where the back-transformed data is written + * \param[out] mf_dst output MultiFab where the back-transformed data is written * \param[in] dcomp first component of mf_dst in which the back-transformed * lab-frame data for the user-request fields is written. * \param[in] i_buffer buffer index for which the data is transformed. diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H index ed867520375..e88b522f148 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H @@ -202,7 +202,7 @@ public: /** Computes the Lorentz transform of source particles to obtain lab-frame data in pc_dst*/ void operator () (PinnedMemoryParticleContainer& pc_dst, int &TotalParticleCounter, int i_buffer) const override; void InitData() override; - /** \brief Prepare data required to back-transform particle attribtutes for lab-frame snapshot, i_buffer + /** \brief Prepare data required to back-transform particle attributes for lab-frame snapshot, i_buffer * * \param[in] i_buffer index of the snapshot * \param[in] z_slice_in_domain if the z-slice at current_z_boost is within the bounds of diff --git a/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H index e5f8c22b95c..6856c4c90a2 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H @@ -29,7 +29,7 @@ public: ComputeParticleDiagFunctor(ComputeParticleDiagFunctor&&) = default; ComputeParticleDiagFunctor& operator=(ComputeParticleDiagFunctor&&) = default; - /** \brief Prepare data required to back-transform particle attribtutes for + /** \brief Prepare data required to back-transform particle attributes for * lab-frame snapshot, with index i_buffer. * Note that this function has parameters that are specific to * back-transformed diagnostics, that are unused for regular diagnostics. diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 5d5a300dea9..21d5ff699d0 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -75,7 +75,7 @@ public: * * Derived classes MUST implement this function, and it must allocate m_all_field_functors * and fill it with ComputeDiagFunctor objects. - * The functions is called at intiailization and when the domain is decomposed + * The functions is called at initialization and when the domain is decomposed * during the simulation to load-balance. * \param[in] lev level on which the vector of unique_ptrs to field functors is initialized. */ @@ -239,13 +239,13 @@ protected: amrex::Vector< std::string > m_pfield_varnames; /** Names of species for which to output particle field diagnostics */ std::vector< std::string > m_pfield_species; - /** Whether to do averaging for each of the particle fied diagnostics */ + /** Whether to do averaging for each of the particle field diagnostics */ std::vector< bool > m_pfield_do_average; /** Species indices corresponding to elements of m_pfield_varnames */ std::vector< int > m_pfield_species_index; /** List of the parser strings for the particle field diagnostics */ std::vector< std::string > m_pfield_strings; - /** Whether to use a filter function on particles before calculating particle fied diagnostics */ + /** Whether to use a filter function on particles before calculating particle field diagnostics */ std::vector< bool> m_pfield_dofilter; /** List of parser strings for pre-average filtering for the particle field diagnostics */ std::vector< std::string > m_pfield_filter_strings; diff --git a/Source/Diagnostics/FieldIO.H b/Source/Diagnostics/FieldIO.H index 1f8bfe5346a..138a1317e08 100644 --- a/Source/Diagnostics/FieldIO.H +++ b/Source/Diagnostics/FieldIO.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_FielIO_H_ -#define WARPX_FielIO_H_ +#ifndef WARPX_FieldIO_H_ +#define WARPX_FieldIO_H_ #include @@ -53,4 +53,4 @@ getReversedVec( const amrex::IntVect& v ); std::vector getReversedVec( const amrex::Real* v ); -#endif // WARPX_FielIO_H_ +#endif // WARPX_FieldIO_H_ diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.H b/Source/Diagnostics/ReducedDiags/BeamRelevant.H index f5037e37bc0..018ddef5463 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.H +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.H @@ -29,7 +29,7 @@ public: std::string m_beam_name; /** - * This function computes beam relevant quantites. + * This function computes beam relevant quantities. * * @param[in] step current time step */ diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 9d35c1896b3..c041a70aaef 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -600,7 +600,7 @@ void FieldProbe::ComputeDiags (int step) // IO processor sums values from length_array to get size of total output array. /* displs records the size of each m_data as well as previous displs. This array - * tells Gatherv where in the m_data_out array allocation to write incomming data. */ + * tells Gatherv where in the m_data_out array allocation to write incoming data. */ long total_data_size = 0; amrex::Vector displs_vector; if (amrex::ParallelDescriptor::IOProcessor()) { diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp index 9a943759542..1fd741ddc47 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp @@ -142,7 +142,7 @@ FieldProbeParticleContainer::AddNParticles (int lev, /* * Redistributes particles to their appropriate tiles if the box - * structure of the simulation changes to accomodate data more + * structure of the simulation changes to accommodate data more * efficiently. */ Redistribute(); diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H index 51bc1a600f9..ed5cce1d7e0 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H @@ -31,7 +31,7 @@ public: std::string m_species_name; /** - * This funciton computes the particle extrema + * This function computes the particle extrema * * @param[in] step current time step */ diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index ddadc70d343..70471eb3910 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -44,7 +44,7 @@ using namespace amrex; /* \brief * The functions creates the slice for diagnostics based on the user-input. - * The slice can be 1D, 2D, or 3D and it inherts the index type of the underlying data. + * The slice can be 1D, 2D, or 3D and it inherits the index type of the underlying data. * The implementation assumes that the slice is aligned with the coordinate axes. * The input parameters are modified if the user-input does not comply with requirements of coarsenability or if the slice extent is not contained within the simulation domain. * First a slice multifab (smf) with cell size equal to that of the simulation grid is created such that it extends from slice.dim_lo to slice.dim_hi and shares the same index space as the source multifab (mf) @@ -151,7 +151,7 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, const amrex::IntVect nghost_vect(AMREX_D_DECL(nghost, nghost, nghost)); ablastr::utils::communication::ParallelCopy(*smf, mf, 0, 0, ncomp, nghost_vect, nghost_vect, WarpX::do_single_precision_comms); - // inteprolate if required on refined slice // + // interpolate if required on refined slice // if (interpolate == 1 ) { InterpolateSliceValues( *smf, interp_lo, slice_cc_nd_box, dom_geom, ncomp, nghost, slice_lo, slice_hi, SliceType, real_box); @@ -349,7 +349,7 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, } else { - // moving realbox.lo and reabox.hi to nearest coarsenable grid point // + // moving realbox.lo and realbox.hi to nearest coarsenable grid point // auto index_lo = static_cast(floor(((slice_realbox.lo(idim) + very_small_number - (real_box.lo(idim))) / dom_geom[0].CellSize(idim))) ); auto index_hi = static_cast(ceil(((slice_realbox.hi(idim) - very_small_number diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index a91511dcced..a375266b1d3 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -943,7 +943,7 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, auto const numParticleOnTile64 = static_cast( numParticleOnTile ); auto const& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile auto const& soa = pti.GetStructOfArrays(); - // first we concatinate the AoS into contiguous arrays + // first we concatenate the AoS into contiguous arrays { // note: WarpX does not yet use extra AoS Real attributes for( auto idx=0; idx const& Pe, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H index 610a17b6998..205b009b2eb 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H @@ -101,7 +101,7 @@ private: /** * \brief - * This struct contains only static functions to compute the co-efficients for the + * This struct contains only static functions to compute the coefficients for the * Lax-Wendroff scheme of macroscopic Maxwells equations using * macroscopic properties, namely, conductivity (sigma), permittivity (epsilon). * Permeability of the material, mu, is used as (beta/mu) for the E-update @@ -133,7 +133,7 @@ struct LaxWendroffAlgo { /** * \brief - * This struct contains only static functions to compute the co-efficients for the + * This struct contains only static functions to compute the coefficients for the * BackwardEuler scheme of macroscopic Maxwells equations using * macroscopic properties, namely, conductivity (sigma) and permittivity (epsilon). * Permeability of the material, mu, is used as (beta/mu) for the E-update diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index 32c07e0b9fe..965db68a558 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -66,6 +66,7 @@ MacroscopicProperties::ReadParameters () utils::parser::makeParser(m_str_sigma_function,{"x","y","z"})); } + // Query input for material permittivity, epsilon. bool epsilon_specified = false; if (utils::parser::queryWithParser(pp_macroscopic, "epsilon", m_epsilon)) { m_epsilon_s = "constant"; @@ -91,7 +92,7 @@ MacroscopicProperties::ReadParameters () utils::parser::makeParser(m_str_epsilon_function,{"x","y","z"})); } - // Query input for material permittivity, epsilon. + // Query input for material permeability, mu. bool mu_specified = false; if (utils::parser::queryWithParser(pp_macroscopic, "mu", m_mu)) { m_mu_s = "constant"; diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 6b77a559b8c..bc51d65b536 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -266,7 +266,7 @@ void MagnetostaticSolver::VectorPoissonBoundaryHandler::defineVectorPotentialBCs dirichlet_flag[adim][0] = false; dim_start = 1; - // handle the r_max boundary explicity + // handle the r_max boundary explicitly if (WarpX::field_boundary_hi[0] == FieldBoundaryType::PEC) { if (adim == 0) { hibc[adim][0] = LinOpBCType::Neumann; @@ -317,7 +317,7 @@ void MagnetostaticSolver::VectorPoissonBoundaryHandler::defineVectorPotentialBCs } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, - "Field boundary conditions have to be either periodic, PEC, or neumann " + "Field boundary conditions have to be either periodic, PEC, or Neumann " "when using the magnetostatic solver" ); } @@ -337,8 +337,8 @@ void MagnetostaticSolver::VectorPoissonBoundaryHandler::defineVectorPotentialBCs } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, - "Field boundary conditions have to be either periodic, PEC, or neumann " - "when using the electrostatic solver" + "Field boundary conditions have to be either periodic, PEC, or Neumann " + "when using the magnetostatic solver" ); } } diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp index f256c4d2148..8bc7403316d 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp @@ -140,7 +140,7 @@ SpectralFieldData::SpectralFieldData( const int lev, tmpSpectralField = SpectralField(spectralspace_ba, dm, 1, 0); // By default, we assume the FFT is done from/to a nodal grid in real space - // It the FFT is performed from/to a cell-centered grid in real space, + // If the FFT is performed from/to a cell-centered grid in real space, // a correcting "shift" factor must be applied in spectral space. xshift_FFTfromCell = k_space.getSpectralShiftFactor(dm, 0, ShiftType::TransformFromCellCentered); diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp index 236fe3fbb29..1268d277e63 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp @@ -52,7 +52,7 @@ SpectralFieldDataRZ::SpectralFieldDataRZ (const int lev, tmpSpectralField = SpectralField(spectralspace_ba, dm, n_rz_azimuthal_modes, 0); // By default, we assume the z FFT is done from/to a nodal grid in real space. - // It the FFT is performed from/to a cell-centered grid in real space, + // If the FFT is performed from/to a cell-centered grid in real space, // a correcting "shift" factor must be applied in spectral space. zshift_FFTfromCell = k_space.getSpectralShiftFactor(dm, 1, ShiftType::TransformFromCellCentered); diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp index aff5ad25fcc..a8930e1d48b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp @@ -122,7 +122,7 @@ HankelTransform::HankelTransform (int const hankel_order, // Calculate the matrix M by inverting invM if (azimuthal_mode !=0 && hankel_order != azimuthal_mode-1) { // In this case, invM is singular, thus we calculate the pseudo-inverse. - // The Moore-Penrose psuedo-inverse is calculated using the SVD method. + // The Moore-Penrose pseudo-inverse is calculated using the SVD method. M.resize(m_nk*m_nr, 0.); amrex::Vector invMcopy(invM); @@ -132,7 +132,7 @@ HankelTransform::HankelTransform (int const hankel_order, amrex::Vector sp((m_nr)*(m_nk-1), 0.); amrex::Vector temp((m_nr)*(m_nk-1), 0.); - // Calculate the singlular-value-decomposition of invM (leaving out the first row). + // Calculate the singular-value-decomposition of invM (leaving out the first row). // invM = u*sdiag*vt // Note that invMcopy.dataPtr()+1 is passed in so that the first ik row is skipped // A copy is passed in since the matrix is destroyed diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp index 6d35234f0ed..91fe2953668 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp @@ -137,7 +137,7 @@ SpectralKSpace::getKComponent( const DistributionMapping& dm, * specified by `i_dim`. * * (By default, we assume the FFT is done from/to a collocated grid in real space - * It the FFT is performed from/to a cell-centered grid in real space, + * If the FFT is performed from/to a cell-centered grid in real space, * a correcting "shift" factor must be applied in spectral space.) */ SpectralShiftFactor diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index af1c5224df0..fa9d1860053 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -1207,7 +1207,7 @@ void WarpX::DampFieldsInGuards(const int lev, std::unique_ptr& } #ifdef WARPX_DIM_RZ -// This scales the current by the inverse volume and wraps around the depostion at negative radius. +// This scales the current by the inverse volume and wraps around the deposition at negative radius. // It is faster to apply this on the grid than to do it particle by particle. // It is put here since there isn't another nice place for it. void diff --git a/Source/FieldSolver/WarpX_QED_K.H b/Source/FieldSolver/WarpX_QED_K.H index 94903f3cb35..6c86bba67b6 100644 --- a/Source/FieldSolver/WarpX_QED_K.H +++ b/Source/FieldSolver/WarpX_QED_K.H @@ -18,12 +18,12 @@ /** * calc_M calculates the Magnetization field of the vacuum at a specific point and returns it as a three component vector * \param[in] arr This is teh empty array that will be filled with the components of the M-field - * \param[in] ex The x-component of the E-field at the point at whicht the M-field is to be calculated - * \param[in] ey The y-component of the E-field at the point at whicht the M-field is to be calculated - * \param[in] ez The z-component of the E-field at the point at whicht the M-field is to be calculated - * \param[in] bx The x-component of the B-field at the point at whicht the M-field is to be calculated - * \param[in] by The y-component of the B-field at the point at whicht the M-field is to be calculated - * \param[in] bz The z-component of the B-field at the point at whicht the M-field is to be calculated + * \param[in] ex The x-component of the E-field at the point at which the M-field is to be calculated + * \param[in] ey The y-component of the E-field at the point at which the M-field is to be calculated + * \param[in] ez The z-component of the E-field at the point at which the M-field is to be calculated + * \param[in] bx The x-component of the B-field at the point at which the M-field is to be calculated + * \param[in] by The y-component of the B-field at the point at which the M-field is to be calculated + * \param[in] bz The z-component of the B-field at the point at which the M-field is to be calculated * \param[in] xi_c2 The quantum parameter * c2 being used for the simulation * \param[in] c2 the speed of light squared */ @@ -72,7 +72,7 @@ void calc_M(amrex::Real arr [], amrex::Real ex, amrex::Real ey, amrex::Real ez, * \param[in] Jz the current field in z * \param[in] dx The x spatial step, used for calculating curls * \param[in] dy The y spatial step, used for calculating curls - * \param[in] dz The z spatial step, used for calulating curls + * \param[in] dz The z spatial step, used for calculating curls * \param[in] dt The temporal step, used for the half push/correction to the E-fields at the end of the function * \param[in] xi_c2 Quantum parameter * c**2 */ @@ -109,7 +109,7 @@ const amrex::Real dyi = 1._rt/dy; amrex::Real Mpz [3] = {0._rt,0._rt,0._rt}; amrex::Real Mnz [3] = {0._rt,0._rt,0._rt}; - // Calcualting the M-field at the chosen stencil points + // Calculating the M-field at the chosen stencil points calc_M(Mpx, tmpEx(j+1,k,l), tmpEy(j+1,k,l), tmpEz(j+1,k,l), Bx(j+1,k,l), By(j+1,k,l), Bz(j+1,k,l), xi_c2, c2); @@ -182,7 +182,7 @@ const amrex::Real dyi = 1._rt/dy; + 7._rt*c2*bz*(BVxB + Bmu0J) ) }; - // Calcualting matrix values for the QED correction algorithm + // Calculating matrix values for the QED correction algorithm const amrex::Real a00 = beta + xi_c2*( 8._rt*c2i*ex*ex + 14._rt*bx*bx ); @@ -198,7 +198,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real detA = a00*( a11*a22 - a12*a12 ) - a01*( a01*a22 - a02*a12 )+a02*( a01*a12 - a02*a11 ); - // Calcualting the rows of the inverse matrix using the general 3x3 inverse form + // Calculating the rows of the inverse matrix using the general 3x3 inverse form const amrex::Real invAx[3] = {a22*a11 - a12*a12, a12*a02 - a22*a01, a12*a01 - a11*a02}; @@ -206,7 +206,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real invAz[3] = {a12*a01 - a02*a11, a02*a01 - a12*a00, a11*a00 - a01*a01}; - // Calcualting the final QED corrections by mutliplying the Omega vector with the inverse matrix + // Calculating the final QED corrections by mutliplying the Omega vector with the inverse matrix const amrex::Real dEx = (-1._rt/detA)*(invAx[0]*Omega[0] + invAx[1]*Omega[1] + @@ -220,7 +220,7 @@ const amrex::Real dyi = 1._rt/dy; invAz[1]*Omega[1] + invAz[2]*Omega[2]); - // Adding the QED corrections to the origional fields + // Adding the QED corrections to the original fields Ex(j,k,l) = Ex(j,k,l) + 0.5_rt*dt*dEx; @@ -239,7 +239,7 @@ const amrex::Real dyi = 1._rt/dy; amrex::Real Mpz [3] = {0._rt,0._rt,0._rt}; amrex::Real Mnz [3] = {0._rt,0._rt,0._rt}; - // Calcualting the M-field at the chosen stencil points + // Calculating the M-field at the chosen stencil points calc_M(Mpx, tmpEx(j+1,k,0), tmpEy(j+1,k,0), tmpEz(j+1,k,0), Bx(j+1,k,0), By(j+1,k,0), Bz(j+1,k,0), xi_c2, c2); @@ -308,7 +308,7 @@ const amrex::Real dyi = 1._rt/dy; + 7._rt*c2*bz*(BVxB + Bmu0J) ) }; - // Calcualting matrix values for the QED correction algorithm + // Calculating matrix values for the QED correction algorithm const amrex::Real a00 = beta + xi_c2*( 8._rt*c2i*ex*ex + 14._rt*bx*bx ); @@ -324,7 +324,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real detA = a00*( a11*a22 - a12*a12 ) - a01*( a01*a22 - a02*a12 ) + a02*( a01*a12 - a02*a11 ); - // Calcualting inverse matrix values using the general 3x3 form + // Calculating inverse matrix values using the general 3x3 form const amrex::Real invAx[3] = {a22*a11 - a12*a12, a12*a02 - a22*a01, a12*a01 - a11*a02}; @@ -332,7 +332,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real invAz[3] = {a12*a01 - a02*a11, a02*a01 - a12*a00, a11*a00 - a01*a01}; - // Calcualting the final QED corrections by mutliplying the Omega vector with the inverse matrix + // Calculating the final QED corrections by mutliplying the Omega vector with the inverse matrix const amrex::Real dEx = (-1._rt/detA)*(invAx[0]*Omega[0] + invAx[1]*Omega[1] + @@ -346,7 +346,7 @@ const amrex::Real dyi = 1._rt/dy; invAz[1]*Omega[1] + invAz[2]*Omega[2]); - // Adding the QED corrections to the origional fields + // Adding the QED corrections to the original fields Ex(j,k,0) = Ex(j,k,0) + 0.5_rt*dt*dEx; diff --git a/Source/Filter/NCIGodfreyFilter.H b/Source/Filter/NCIGodfreyFilter.H index 86446495869..91479cecd0d 100644 --- a/Source/Filter/NCIGodfreyFilter.H +++ b/Source/Filter/NCIGodfreyFilter.H @@ -16,7 +16,7 @@ enum class godfrey_coeff_set { Ex_Ey_Bz=0, Bx_By_Ez=1 }; /** - * \brief Class for Godrey's filter to suppress Numerical Cherenkov Instability + * \brief Class for Godfrey's filter to suppress Numerical Cherenkov Instability * * It derives from the base class Filter. * The filter stencil is initialized in method ComputeStencils. Computing the diff --git a/Source/Filter/NCIGodfreyFilter.cpp b/Source/Filter/NCIGodfreyFilter.cpp index 1cfd42aed03..9567bdf1bb2 100644 --- a/Source/Filter/NCIGodfreyFilter.cpp +++ b/Source/Filter/NCIGodfreyFilter.cpp @@ -45,7 +45,7 @@ void NCIGodfreyFilter::ComputeStencils() { using namespace warpx::nci_godfrey; - // Sanity checks: filter length shoulz be 5 in z + // Sanity checks: filter length should be 5 in z # if defined(WARPX_DIM_3D) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( slen.z==5,"ERROR: NCI filter requires 5 points in z"); diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H index 172ef4c4209..17d541f180f 100644 --- a/Source/Fluids/WarpXFluidContainer.H +++ b/Source/Fluids/WarpXFluidContainer.H @@ -23,7 +23,7 @@ * * WarpXFluidContainer contains the main functions for initialization, * interaction with the grid (field gather and current deposition), fluid - * source and push, advective update and updates for non-intertial terms. + * source and push, advective update and updates for non-inertial terms. */ class WarpXFluidContainer { @@ -45,7 +45,7 @@ public: void ReadParameters (); /** - * Evolve updates a single timestep (dt) of the cold relativistic fluid eqautions + * Evolve updates a single timestep (dt) of the cold relativistic fluid equations */ void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index f27de6be1cc..32edcad23e7 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -332,7 +332,7 @@ void WarpXFluidContainer::ApplyBcFluidsAndComms (int lev) [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // If the cell is is first gaurd cell & the dimension is non + // If the cell is is first guard cell & the dimension is non // periodic, then copy Q_{i+1} = Q_{i-1}. // Don't check r-dir in Z: #if defined(WARPX_DIM_3D) @@ -495,7 +495,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) #endif //N and NU are always defined at the nodes, the tmp_Q_* are defined - //inbetween the nodes (i.e. on the staggered Yee grid) and store the + //in between the nodes (i.e. on the staggered Yee grid) and store the //values of N and U at these points. //(i.e. the 4 components correspond to N + the 3 components of U) // Extract the temporary arrays for edge values @@ -594,7 +594,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) // Select the specific implementation depending on dimensionality #if defined(WARPX_DIM_3D) - // Compute U ([ N, U]) at the halfsteps (U_tidle) using the slopes (dU) + // Compute U ([ N, U]) at the halfsteps (U_tilde) using the slopes (dU) amrex::Real JdU0x = J00x*dU0x + J01x*dU1x + J02x*dU2x + J03x*dU3x; amrex::Real JdU1x = J11x*dU1x ; amrex::Real JdU2x = J22x*dU2x ; @@ -636,7 +636,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - // Have no RZ-intertial source for primative vars if in XZ + // Have no RZ-inertial source for primitive vars if in XZ amrex::Real N_source = 0.0; #if defined(WARPX_DIM_RZ) @@ -652,7 +652,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) // NUz -> -NUz (NUz_arr(i-1,j,k) -> NUz_arr(i+1,j,k)) dU0x = ave( -UpDx_N(N_arr,i,j,k) , UpDx_N(N_arr,i,j,k) ); // First term in the ave is: U_{x,y} + U_{x,y}_p, - // which can be writen as 2*U_{x,y} + UpDx_U(U_{x,y}) + // which can be written as 2*U_{x,y} + UpDx_U(U_{x,y}) dU1x = ave( 2.0_rt*Ux + UpDx_U(N_arr,NUx_arr,Ux,i,j,k) , UpDx_U(N_arr,NUx_arr,Ux,i,j,k) ); dU2x = ave( 2.0_rt*Uy + UpDx_U(N_arr,NUy_arr,Uy,i,j,k) , UpDx_U(N_arr,NUy_arr,Uy,i,j,k) ); dU3x = ave( -UpDx_U(N_arr,NUz_arr,Uz,i,j,k) , UpDx_U(N_arr,NUz_arr,Uz,i,j,k) ); @@ -669,7 +669,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) } #endif - // Compute U ([ N, U]) at the halfsteps (U_tidle) using the slopes (dU) + // Compute U ([ N, U]) at the halfsteps (U_tilde) using the slopes (dU) amrex::Real JdU0x = J00x*dU0x + J01x*dU1x + J02x*dU2x + J03x*dU3x; amrex::Real JdU1x = J11x*dU1x; amrex::Real JdU2x = J22x*dU2x; @@ -699,7 +699,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) #else - // Compute U ([ N, U]) at the halfsteps (U_tidle) using the slopes (dU) + // Compute U ([ N, U]) at the halfsteps (U_tilde) using the slopes (dU) amrex::Real JdU0z = J00z*dU0z + J01z*dU1z + J02z*dU2z + J03z*dU3z; amrex::Real JdU1z = J11z*dU1z; amrex::Real JdU2z = J22z*dU2z; @@ -734,7 +734,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) ); } - // Given the values of `U_minus` and `U_plus`, compute fluxes inbetween nodes, and update N, NU accordingly + // Given the values of `U_minus` and `U_plus`, compute fluxes in between nodes, and update N, NU accordingly #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index b0e71b52fa7..d5f19214e3c 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -309,7 +309,7 @@ struct InjectorMomentumBoltzmann // The following condition is equation 32 in Zenitani 2015 // (Phys. Plasmas 22, 042116) , called the flipping method. It - // transforms the intergral: d3x' -> d3x where d3x' is the volume + // transforms the integral: d3x' -> d3x where d3x' is the volume // element for positions in the boosted frame. The particle positions // and densities can be initialized in the simulation frame. // The flipping method can transform any symmetric distribution from one @@ -351,7 +351,7 @@ private: GetTemperature temperature; }; -// struct whose getMomentum returns momentum for 1 particle with relativistc +// struct whose getMomentum returns momentum for 1 particle with relativistic // drift velocity beta, from the Maxwell-Juttner distribution. Method is from // Zenitani 2015 (Phys. Plasmas 22, 042116). struct InjectorMomentumJuttner @@ -409,8 +409,8 @@ struct InjectorMomentumJuttner // The value of dir is the boost direction to be transformed. u[dir] = u[dir]*(2._rt*x1-1._rt); x1 = amrex::Random(engine); - // The following condition is equtaion 32 in Zenitani, called - // The flipping method. It transforms the intergral: d3x' -> d3x + // The following condition is equation 32 in Zenitani, called + // The flipping method. It transforms the integral: d3x' -> d3x // where d3x' is the volume element for positions in the boosted frame. // The particle positions and densities can be initialized in the // simulation frame with this method. @@ -488,7 +488,7 @@ private: amrex::Real u_over_r; }; -// struct whose getMomentumm returns local momentum computed from parser. +// struct whose getMomentum returns local momentum computed from parser. struct InjectorMomentumParser { InjectorMomentumParser (amrex::ParserExecutor<3> const& a_ux_parser, @@ -517,7 +517,7 @@ struct InjectorMomentumParser amrex::ParserExecutor<3> m_ux_parser, m_uy_parser, m_uz_parser; }; -// struct whose getMomentumm returns local momentum and thermal spread computed from parser. +// struct whose getMomentum returns local momentum and thermal spread computed from parser. struct InjectorMomentumGaussianParser { InjectorMomentumGaussianParser (amrex::ParserExecutor<3> const& a_ux_m_parser, diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index 99aba7bc88b..64cc1a2a620 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -221,7 +221,7 @@ struct InjectorPosition z <= zmax and z >= zmin); } - // bool: whether the region defined by lo and hi overaps with the plasma region + // bool: whether the region defined by lo and hi overlaps with the plasma region [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 93cf38f7c75..c1abf74cba5 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -119,7 +119,7 @@ WarpX::PostProcessBaseGrids (BoxArray& ba0) const #if defined(WARPX_DIM_3D) for (int k = 0; k < numprocs[2]; ++k) { // The first extra[2] blocks get one extra cell with a total of - // sz[2]+1. The rest get sz[2] cells. The docomposition in y + // sz[2]+1. The rest get sz[2] cells. The decomposition in y // and x directions are similar. int klo = (k < extra[2]) ? k*(sz[2]+1) : (k*sz[2]+extra[2]); int khi = (k < extra[2]) ? klo+(sz[2]+1)-1 : klo+sz[2]-1; diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index ee8fd044abb..38cef54921b 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -100,7 +100,7 @@ public: amrex::IntVect ng_UpdateAux = amrex::IntVect::TheZeroVector(); // Number of guard cells of all MultiFabs that must exchanged before moving window amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); - // Number of guard cells of E and B that are exchanged immediatly after the main PSATD push + // Number of guard cells of E and B that are exchanged immediately after the main PSATD push amrex::IntVect ng_afterPushPSATD = amrex::IntVect::TheZeroVector(); // Number of guard cells for local deposition of J and rho diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 179afe86ae2..79c624e212a 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -904,7 +904,7 @@ WarpX::SyncCurrent ( // SumBoundary even if there is only a single process, because a process // may have multiple boxes. Furthermore, even if there is only a single // box on a single process, SumBoundary should also be called if there - // are periodic boundaries. So we always call SumBounary even if it + // are periodic boundaries. So we always call SumBoundary even if it // might be a no-op in some cases, because the function does not perform // any communication if not necessary. // @@ -923,13 +923,13 @@ WarpX::SyncCurrent ( // have all been properly filtered and summed. For the current level, if // there are levels below this, we need to process this level's cp data // just like we have done for the finer level. The iteration continues - // until we reache level 0. There are however two additional + // until we reach level 0. There are however two additional // complications. // // The first complication is that simply calling ParallelAdd to add the // finer level's cp data to the current level's fp data does not // work. Suppose there are multiple boxes on the current level (or just - // a single box with periodic bounaries). A given (i,j,k) can be present + // a single box with periodic boundaries). A given (i,j,k) can be present // in more than one box for nodal data in AMReX. // At the time of calling ParallelAdd, the current // level's fp data have not been summed. Because of how ParallelAdd diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp index 1ef5ecad229..0d8ffc5b8ae 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp @@ -31,7 +31,7 @@ ParticleCreationFunc::ParticleCreationFunc (const std::string collision_name, // Proton-Boron fusion produces 3 alpha particles per fusion reaction m_num_products_host.push_back(3); #ifndef AMREX_USE_GPU - // On CPU, the device vector can be filled immediatly + // On CPU, the device vector can be filled immediately m_num_products_device.push_back(3); #endif } @@ -43,7 +43,7 @@ ParticleCreationFunc::ParticleCreationFunc (const std::string collision_name, m_num_products_host.push_back(1); m_num_products_host.push_back(1); #ifndef AMREX_USE_GPU - // On CPU, the device vector can be filled immediatly + // On CPU, the device vector can be filled immediately m_num_products_device.push_back(1); m_num_products_device.push_back(1); #endif diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 8735366e018..fd74c9029ed 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -152,7 +152,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, // wqx, wqy wqz are particle current in each direction #if defined(WARPX_DIM_RZ) // In RZ, wqx is actually wqr, and wqy is wqtheta - // Convert to cylinderical at the mid point + // Convert to cylindrical at the mid point const amrex::Real xpmid = xp + relative_time*vx; const amrex::Real ypmid = yp + relative_time*vy; const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); diff --git a/Source/Particles/Deposition/SharedDepositionUtils.H b/Source/Particles/Deposition/SharedDepositionUtils.H index 646a6add811..e28835a57df 100644 --- a/Source/Particles/Deposition/SharedDepositionUtils.H +++ b/Source/Particles/Deposition/SharedDepositionUtils.H @@ -132,7 +132,7 @@ void depositComponent (const GetParticlePosition& GetPosition, // pcurrent is the particle current in the deposited direction #if defined(WARPX_DIM_RZ) // In RZ, wqx is actually wqr, and wqy is wqtheta - // Convert to cylinderical at the mid point + // Convert to cylindrical at the mid point const amrex::Real xpmid = xp + relative_time*vx; const amrex::Real ypmid = yp + relative_time*vy; const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp index a43453da925..968ab8b85f1 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp +++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp @@ -30,7 +30,7 @@ using namespace std; using namespace amrex; namespace pxr_sr = picsar::multi_physics::utils::serialization; -//This file provides a wrapper aroud the breit_wheeler engine +//This file provides a wrapper around the breit_wheeler engine //provided by the PICSAR library // Factory class ============================= diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H b/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H index 3b2093c40b7..4f26c256c76 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H @@ -36,7 +36,7 @@ /** * PICSAR uses internally some specifiers analogous to * AMREX_RESTRICT and AMREX_FORCE_INLINE. These definitions - * set the aformentioned specifiers to AMREX_RESTRICT and + * set the aforementioned specifiers to AMREX_RESTRICT and * AMREX_FORCE_INLINE. */ #define PXRMP_RESTRICT AMREX_RESTRICT diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp index 2a52829272d..c6548d91612 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp +++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp @@ -30,7 +30,7 @@ using namespace std; using namespace amrex; namespace pxr_sr = picsar::multi_physics::utils::serialization; -//This file provides a wrapper aroud the quantum_sync engine +//This file provides a wrapper around the quantum_sync engine //provided by the PICSAR library // Factory class ============================= diff --git a/Source/Particles/Filter/FilterFunctors.H b/Source/Particles/Filter/FilterFunctors.H index 3b2ddbc9dc8..54cf09fb3d7 100644 --- a/Source/Particles/Filter/FilterFunctors.H +++ b/Source/Particles/Filter/FilterFunctors.H @@ -157,7 +157,7 @@ struct GeometryFilter GeometryFilter(bool a_is_active, amrex::RealBox a_domain) : m_is_active(a_is_active), m_domain(a_domain) {} /** - * \brief return 1 if the partcile is within the region described by the RealBox + * \brief return 1 if the particle is within the region described by the RealBox * \param p one particle * \return whether or not the particle is inside the region defined by m_domain */ diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index a45d9fd0dc0..c32ff07464d 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -111,14 +111,14 @@ public: /// /// This pushes the particle positions by one half time step for all the species in the - /// MultiParticleContainer. It is used to desynchronize the particles after initializaton + /// MultiParticleContainer. It is used to desynchronize the particles after initialization /// or when restarting from a checkpoint. /// void PushX (amrex::Real dt); /// /// This pushes the particle momenta by dt for all the species in the - /// MultiParticleContainer. It is used to desynchronize the particles after initializaton + /// MultiParticleContainer. It is used to desynchronize the particles after initialization /// or when restarting from a checkpoint. It is also used to synchronize particles at the /// the end of the run. This is the electromagnetic version. /// @@ -193,7 +193,7 @@ public: /** This function computes the box outside which Schwinger process is disabled. The box is * defined by m_qed_schwinger_xmin/xmax/ymin/ymax/zmin/zmax and the warpx level 0 geometry - * object (to make the link between Real and int quatities). + * object (to make the link between Real and int quantities). */ [[nodiscard]] amrex::Box ComputeSchwingerGlobalBox () const; #endif diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index 5a26dadaaa8..c9d0eae3c4b 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -25,7 +25,7 @@ /** * Photon particles have no mass, they deposit no charge, and see specific QED * effects. For these reasons, they are stored in the separate particle - * container PhotonParticleContainer, that inherts from + * container PhotonParticleContainer, that inherits from * PhysicalParticleContainer. The particle pusher and current deposition, in * particular, are overriden in this container. */ diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 3fe1941775d..182577a81cf 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1907,7 +1907,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // Rotate the momentum // This because, when the flux direction is e.g. "r" // the `inj_mom` objects generates a v*Gaussian distribution - // along the Cartesian "x" directionm by default. This + // along the Cartesian "x" direction by default. This // needs to be rotated along "r". Real ur = pu.x; Real ut = pu.y; @@ -2696,7 +2696,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, void PhysicalParticleContainer::ContinuousInjection (const RealBox& injection_box) { - // Inject plasma on level 0. Paticles will be redistributed. + // Inject plasma on level 0. Particles will be redistributed. const int lev=0; for (auto const& plasma_injector : plasma_injectors) { AddPlasma(*plasma_injector, lev, injection_box); diff --git a/Source/Particles/Pusher/PushSelector.H b/Source/Particles/Pusher/PushSelector.H index f6fa3ba164f..637d2ac74db 100644 --- a/Source/Particles/Pusher/PushSelector.H +++ b/Source/Particles/Pusher/PushSelector.H @@ -31,7 +31,7 @@ * \param ux, uy, uz Particle momentum * \param Ex, Ey, Ez Electric field on particles. * \param Bx, By, Bz Magnetic field on particles. - * \param ion_lev Ionization level of this particle (0 if ioniziation not on) + * \param ion_lev Ionization level of this particle (0 if ionization not on) * \param m Mass of this species. * \param a_q Charge of this species. * \param pusher_algo 0: Boris, 1: Vay, 2: HigueraCary diff --git a/Source/Particles/Pusher/UpdateMomentumHigueraCary.H b/Source/Particles/Pusher/UpdateMomentumHigueraCary.H index dc998caee34..8da3a8cfb2c 100644 --- a/Source/Particles/Pusher/UpdateMomentumHigueraCary.H +++ b/Source/Particles/Pusher/UpdateMomentumHigueraCary.H @@ -46,7 +46,7 @@ void UpdateMomentumHigueraCary( const T sigma = gamma - betam; // Get u* const T ust = (umx*betax + umy*betay + umz*betaz)*invclight; - // Get new gamma inversed + // Get new gamma inverse gamma = 1._prt/std::sqrt(0.5_prt*(sigma + std::sqrt(sigma*sigma + 4._prt*(betam + ust*ust)) )); // Compute t const T tx = gamma*betax; diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index acbe3a0493d..0d5bf0d0f0e 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -122,7 +122,7 @@ private: amrex::Vector zinject_plane_levels; - // Temporary quantites + // Temporary quantities amrex::ParticleReal zinject_plane_lev; amrex::ParticleReal zinject_plane_lev_previous; bool done_injecting_lev; diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 4e4d644123b..d0e19af155a 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -307,7 +307,7 @@ RigidInjectedParticleContainer::Evolve (int lev, zinject_plane_levels[lev] -= dt*WarpX::beta_boost*PhysConst::c; zinject_plane_lev = zinject_plane_levels[lev]; - // Set the done injecting flag whan the inject plane moves out of the + // Set the done injecting flag when the inject plane moves out of the // simulation domain. // It is much easier to do this check, rather than checking if all of the // particles have crossed the inject plane. diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 5011623bdf8..024cd8d9157 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -171,7 +171,7 @@ public: /// /// This pushes the particle positions by one half time step. - /// It is used to desynchronize the particles after initializaton + /// It is used to desynchronize the particles after initialization /// or when restarting from a checkpoint. /// void PushX ( amrex::Real dt); diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index ed8186a325c..b361aab14e9 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -20,7 +20,7 @@ #include #include -// Define dictionary with correspondance between user-input strings, +// Define dictionary with correspondence between user-input strings, // and corresponding integer for use inside the code const std::map grid_to_int = { diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 42a97354248..af40c5e9841 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -601,7 +601,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, // guard region both radially and longitudinally. These are the PML cells in the overlapping // longitudinal region. FillBoundary normally does not update these cells. // This update is needed so that the cells at the end of the FABs are updated appropriately - // with the data shifted from the nieghboring FAB. Without this update, the RZ PML becomes + // with the data shifted from the neighboring FAB. Without this update, the RZ PML becomes // unstable with the moving grid. // This code creates a temporary MultiFab using a BoxList where the radial size of all of // its boxes is increased so that the radial guard cells are included in the boxes valid domain. diff --git a/Source/WarpX.H b/Source/WarpX.H index 3e6078d4f43..85fdecda841 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -552,7 +552,7 @@ public: amrex::Vector mirror_z_width; amrex::Vector mirror_z_npoints; - /// object with all reduced diagnotics, similar to MultiParticleContainer for species. + /// object with all reduced diagnostics, similar to MultiParticleContainer for species. std::unique_ptr reduced_diags; void applyMirrors(amrex::Real time); diff --git a/Tools/Algorithms/psatd_pml.ipynb b/Tools/Algorithms/psatd_pml.ipynb index 216be77fd87..897ffa70b9e 100644 --- a/Tools/Algorithms/psatd_pml.ipynb +++ b/Tools/Algorithms/psatd_pml.ipynb @@ -228,7 +228,7 @@ " #print(r'Eigenpairs:')\n", " #display(MX_eigenpairs)\n", "\n", - " # Verify that the eigenpairs satisfy the charcteristic equations\n", + " # Verify that the eigenpairs satisfy the characteristic equations\n", " for ep in MX_eigenpairs:\n", " for j in range(ep[1]):\n", " diff = MX * ep[2][j] - ep[0] * ep[2][j]\n", diff --git a/Tools/DevUtils/compute_domain.py b/Tools/DevUtils/compute_domain.py index d67cc86f53a..b54412639bd 100644 --- a/Tools/DevUtils/compute_domain.py +++ b/Tools/DevUtils/compute_domain.py @@ -11,7 +11,7 @@ The user specifies the minimal size of the physical domain and the resolution in each dimension, and the scripts computes: -- the number of cells and physical domain to satify the user-specified domain +- the number of cells and physical domain to satisfy the user-specified domain size and resolution AND make sure that the number of cells along each direction is a multiple of max_grid_size. - a starting point on how to parallelize on Cori KNL (number of nodes, etc.). diff --git a/Tools/DevUtils/update_benchmarks_from_azure_output.py b/Tools/DevUtils/update_benchmarks_from_azure_output.py index 28c3dc6b5f2..ec7b17d1050 100644 --- a/Tools/DevUtils/update_benchmarks_from_azure_output.py +++ b/Tools/DevUtils/update_benchmarks_from_azure_output.py @@ -51,7 +51,7 @@ # Raw Azure output comes with a prefix at the beginning of each line that we do # not need here. The first line that we will read is the prefix followed by the # "{" character, so we determine how long the prefix is by finding the last - # occurence of the "{" character in this line. + # occurrence of the "{" character in this line. azure_indent = line.rfind('{') first_line_read = True new_file_string += line[azure_indent:] diff --git a/Tools/LibEnsemble/run_libensemble_on_warpx.py b/Tools/LibEnsemble/run_libensemble_on_warpx.py index 1be152821f6..b86c0249d01 100644 --- a/Tools/LibEnsemble/run_libensemble_on_warpx.py +++ b/Tools/LibEnsemble/run_libensemble_on_warpx.py @@ -102,7 +102,7 @@ ('ramp_down_2', float, (1,)), # input parameter: position of the focusing lens. ('zlens_1', float, (1,)), - # Relative stength of the lens (1. is from + # Relative strength of the lens (1. is from # back-of-the-envelope calculation) ('adjust_factor', float, (1,)), ], diff --git a/Tools/PerformanceTests/run_alltests.py b/Tools/PerformanceTests/run_alltests.py index e9ff899fd2a..b1083fc6f45 100644 --- a/Tools/PerformanceTests/run_alltests.py +++ b/Tools/PerformanceTests/run_alltests.py @@ -259,7 +259,7 @@ def process_analysis(): log_line = '## year month day run_name compiler architecture n_node n_mpi ' +\ 'n_omp time_initialization time_one_iteration Redistribute '+\ 'FillBoundary ParallelCopy CurrentDeposition FieldGather '+\ - 'ParthiclePush Copy Evolve Checkpoint '+\ + 'ParticlePush Copy Evolve Checkpoint '+\ 'WriteParticles Write_FabArray '+\ 'WriteMultiLevelPlotfile '+\ 'RedistributeMPI(unit: second)\n' diff --git a/Tools/PerformanceTests/run_alltests_1node.py b/Tools/PerformanceTests/run_alltests_1node.py index f2ebb73124e..f112552b36e 100644 --- a/Tools/PerformanceTests/run_alltests_1node.py +++ b/Tools/PerformanceTests/run_alltests_1node.py @@ -26,7 +26,7 @@ # > --architecture=knl --mode=run --input_file=uniform_plasma # > --n_node=1 --log_file='my_performance_log.txt' -# ---- Running the pre-drefined automated tests ---- +# ---- Running the pre-defined automated tests ---- # Compile and run: # > python run_alltests_1node.py --automated --recompile # Just run: diff --git a/Tools/PerformanceTests/run_automated.py b/Tools/PerformanceTests/run_automated.py index 58fc02d6ac3..73ab79b00df 100644 --- a/Tools/PerformanceTests/run_automated.py +++ b/Tools/PerformanceTests/run_automated.py @@ -87,7 +87,7 @@ parser.add_argument('--n_node_list', dest='n_node_list', default=[], - help='list ofnumber of nodes for the runs', type=str) + help='list of number of nodes for the runs', type=str) parser.add_argument('--start_date', dest='start_date' ) parser.add_argument('--compiler', diff --git a/Tools/PostProcessing/plot_distribution_mapping.py b/Tools/PostProcessing/plot_distribution_mapping.py index 35fc2c197fa..19628551f26 100644 --- a/Tools/PostProcessing/plot_distribution_mapping.py +++ b/Tools/PostProcessing/plot_distribution_mapping.py @@ -98,7 +98,7 @@ def _get_costs_reduced_diagnostics(self, directory, prange): j_non_zero = j_blocks[j_blocks != 0] k_non_zero = k_blocks[k_blocks != 0] - # only one block in a dir - or smalles block size + # only one block in a dir - or smallest block size i_blocking_factor = 1 if len(i_non_zero) == 0 else i_non_zero.min() j_blocking_factor = 1 if len(j_non_zero) == 0 else j_non_zero.min() k_blocking_factor = 1 if len(k_non_zero) == 0 else k_non_zero.min() diff --git a/Tools/PostProcessing/plot_parallel.py b/Tools/PostProcessing/plot_parallel.py index e4b75c0e406..a4309b3896e 100644 --- a/Tools/PostProcessing/plot_parallel.py +++ b/Tools/PostProcessing/plot_parallel.py @@ -239,7 +239,7 @@ def reduce_evolved_quantity(z, q): file_list.sort() nfiles = len(file_list) -# Get list of particle speciess to plot +# Get list of particle species to plot pslist = get_species(file_list); rank = 0 From 11c5904475c6fc1117ccc6163f217cb39b1d40d6 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 5 Dec 2023 21:50:12 -0800 Subject: [PATCH 024/176] pybind11: v2.11.1+ (#4473) Update pybind11 to the latest release. --- cmake/dependencies/pybind11.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index 8d097d869ff..e65cd206f34 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -37,7 +37,7 @@ function(find_pybind11) mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11) endif() else() - find_package(pybind11 2.10.1 CONFIG REQUIRED) + find_package(pybind11 2.11.1 CONFIG REQUIRED) message(STATUS "pybind11: Found version '${pybind11_VERSION}'") endif() endfunction() @@ -52,7 +52,7 @@ option(WarpX_pybind11_internal "Download & build pybind11" ON) set(WarpX_pybind11_repo "https://github.com/pybind/pybind11.git" CACHE STRING "Repository URI to pull and build pybind11 from if(WarpX_pybind11_internal)") -set(WarpX_pybind11_branch "v2.10.1" +set(WarpX_pybind11_branch "v2.11.1" CACHE STRING "Repository branch for WarpX_pybind11_repo if(WarpX_pybind11_internal)") From e43e50f8dc04a8079c8acdcb98e4c4e98c05a539 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Wed, 6 Dec 2023 14:32:33 -0600 Subject: [PATCH 025/176] Set alpha mass directly from NIST data (#4478) * Set alpha mass directly from NIST data * Delete extra line --- Source/Particles/SpeciesPhysicalProperties.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Source/Particles/SpeciesPhysicalProperties.cpp b/Source/Particles/SpeciesPhysicalProperties.cpp index 118db98f2c2..20ca5ed48b9 100644 --- a/Source/Particles/SpeciesPhysicalProperties.cpp +++ b/Source/Particles/SpeciesPhysicalProperties.cpp @@ -126,8 +126,9 @@ namespace { constexpr auto quiet_NaN = std::numeric_limits::quiet_NaN(); - // The atomic mass data below is from this NIST page + // The atomic mass data below is from these NIST pages // https://physics.nist.gov/cgi-bin/Compositions/stand_alone.pl?ele=&ascii=ascii2&isotype=some + // https://physics.nist.gov/cgi-bin/cuu/Value?malu const std::map species_to_properties { @@ -177,7 +178,7 @@ namespace { amrex::Real(4.00260325413) * PhysConst::m_u, amrex::Real(2) * PhysConst::q_e}}, {PhysicalSpecies::alpha, Properties{ - amrex::Real(4.00260325413) * PhysConst::m_u - amrex::Real(2) * PhysConst::m_e, + amrex::Real(4.001506179127) * PhysConst::m_u, amrex::Real(2) * PhysConst::q_e}}, {PhysicalSpecies::lithium, Properties{ amrex::Real(6.967) * PhysConst::m_u, From a2eb5549dc9f932a42c8706302c1d3335c91ae6d Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Wed, 6 Dec 2023 16:11:59 -0600 Subject: [PATCH 026/176] Add `latex_theory/allbibs.bib` to bibliography list and delete duplicates (#4481) * Add `latex_theory/allbibs.bib` to conf.py * Delete additional redundant reference and make intro.rst match intro.tex --- Docs/source/conf.py | 2 +- Docs/source/latex_theory/allbibs.bib | 37 +- Docs/source/refs.bib | 392 +----------------- Docs/source/theory/intro.rst | 11 +- .../Physics_applications/laser_ion/README.rst | 2 +- .../plasma_acceleration/README.rst | 2 +- .../plasma_mirror/README.rst | 2 +- 7 files changed, 31 insertions(+), 417 deletions(-) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 9a2def319e4..74556cde1ec 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -57,7 +57,7 @@ templates_path = ['_templates'] # Relative path to bibliography file, bibliography style -bibtex_bibfiles = ['./refs.bib'] +bibtex_bibfiles = ['latex_theory/allbibs.bib', 'refs.bib'] bibtex_default_style = 'unsrt' # The suffix(es) of source filenames. diff --git a/Docs/source/latex_theory/allbibs.bib b/Docs/source/latex_theory/allbibs.bib index 81a02e75494..bdc84d5cc12 100644 --- a/Docs/source/latex_theory/allbibs.bib +++ b/Docs/source/latex_theory/allbibs.bib @@ -20,6 +20,7 @@ @book{HockneyEastwoodBook author = {Hockney, R W and Eastwood, J W}, isbn = {0-85274-392-0}, pages = {xxi+540 pp}, +publisher = {Routledge}, title = {{Computer simulation using particles}}, type = {Book}, year = {1988} @@ -54,7 +55,7 @@ @article{Godfreyjcp74 year = {1974} } @article{Quickpic, -author = {Huang, C and Decyk, V K and Ren, C and Zhou, M and Lu, W and Mori, W B and Cooley, J H and {Antonsen Jr.}, T M and Katsouleas, T}, +author = {Huang, C and Decyk, V K and Ren, C and Zhou, M and Lu, W and Mori, W B and Cooley, J H and Antonsen, Jr, T M and Katsouleas, T}, doi = {10.1016/J.Jcp.2006.01.039}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -308,16 +309,15 @@ @article{VayFRACAD2014 } @article{AndriyashPoP2016, - author = "Andriyash, Igor A. and Lehe, Remi and Lifschitz, Agustin", - title = "Laser-plasma interactions with a Fourier-Bessel particle-in-cell method", - journal = "Physics of Plasmas", - year = "2016", - volume = "23", - number = "3", - eid = 033110, - pages = "", - url = "http://scitation.aip.org/content/aip/journal/pop/23/3/10.1063/1.4943281", - doi = "http://dx.doi.org/10.1063/1.4943281" +author = {Andriyash, Igor A. and Lehe, Remi and Lifschitz, Agustin}, +doi = {10.1063/1.4943281}, +eid = {033110}, +journal = {Physics of Plasmas}, +number = {3}, +pages = {}, +title = {{Laser-plasma interactions with a Fourier-Bessel particle-in-cell method}}, +volume = {23}, +year = {2016} } @article{DawsonRMP83, @@ -1280,7 +1280,7 @@ @article{DavidsonJCP2015 author = {Davidson, A. and Tableman, A. and An, W. and Tsung, F.S. and Lu, W. and Vieira, J. and Fonseca, R.A. and Silva, L.O. and Mori, W.B.}, doi = {10.1016/j.jcp.2014.10.064}, -issn = {00219991}, +issn = {0021-9991}, journal = {Journal of Computational Physics}, pages = {1063--1077}, title = {{Implementation of a hybrid particle code with a PIC description in r–z and a gridless description in \Phi into OSIRIS}}, @@ -1538,7 +1538,7 @@ @article{Sprangleprl90 month = {apr}, number = {17}, pages = {2011--2014}, -title = {{Nonlinear-Theory Of Intense Laser-Plasma Interactions}}, +title = {{Nonlinear theory of intense laser-plasma interactions}}, volume = {64}, year = {1990} } @@ -2093,7 +2093,7 @@ @book{Geddesdissertation05 year = {2005} } @article{Tsungpop06, -author = {Tsung, Fs and Lu, W and Tzoufras, M and Mori, Wb and Joshi, C and Vieira, Jm and Silva, Lo and Fonseca, Ra}, +author = {Tsung, F S and W Lu, and Tzoufras, M and Mori, W B and Joshi, C and Vieira, Jm and Silva, Lo and Fonseca, Ra}, doi = {10.1063/1.2198535}, issn = {1070-664X}, journal = {Physics Of Plasmas}, @@ -2219,9 +2219,8 @@ @article{LeheARXIV2016 } @book{godfrey1985iprop, - title={The IPROP Three-Dimensional Beam Propagation Code}, - author={Godfrey, B.B.}, - url={https://books.google.com/books?id=hos\_OAAACAAJ}, - year={1985}, - publisher={Defense Technical Information Center} +author={Godfrey, B.B.}, +publisher={Defense Technical Information Center}, +title={{The IPROP Three-Dimensional Beam Propagation Code}}, +year={1985}, } diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 6c94869caa5..e12e885dd33 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -1,28 +1,6 @@ -@article{Tsung2006, -author = {Tsung,F. S. and Lu,W. and Tzoufras,M. and Mori,W. B. and Joshi,C. and Vieira,J. M. and Silva,L. O. and Fonseca,R. A. }, -title = {Simulation of monoenergetic electron generation via laser wakefield accelerators for 5–25TW lasers}, -journal = {Physics of Plasmas}, -volume = {13}, -number = {5}, -pages = {056708}, -year = {2006}, -doi = {10.1063/1.2198535} -} - -@article{Geddes2008, -doi = {10.1088/1742-6596/125/1/012002}, -year = {2008}, -volume = {125}, -number = {1}, -pages = {012002}, -author = {C G R Geddes and D L Bruhwiler and J R Cary and W B Mori and J-L Vay and S F Martins and T Katsouleas and E Cormier-Michel and W M Fawley and C Huang and X Wang and B Cowan and V K Decyk and E Esarey and R A Fonseca and W Lu and P Messmer and P Mullowney and K Nakamura and K Paul and G R Plateau and C B Schroeder and L O Silva and C Toth and F S Tsung and M Tzoufras and T Antonsen and J Vieira and W P Leemans}, -title = {Computational studies and optimization of wakefield accelerators}, -journal = {Journal of Physics: Conference Series} -} - @article{TajimaDawson1982, author = {Tajima, T. and Dawson, J. M.}, - title = "{Laser accelerator by plasma waves}", + title = {{Laser accelerator by plasma waves}}, journal = {AIP Conference Proceedings}, volume = {91}, number = {1}, @@ -38,7 +16,7 @@ @article{TajimaDawson1982 @article{Esarey1996, author={Esarey, E. and Sprangle, P. and Krall, J. and Ting, A.}, journal={IEEE Transactions on Plasma Science}, - title={Overview of plasma-based accelerator concepts}, + title={{Overview of plasma-based accelerator concepts}}, year={1996}, volume={24}, number={2}, @@ -46,370 +24,6 @@ @article{Esarey1996 doi={10.1109/27.509991} } -@techreport{Geddes2009, -title={Laser plasma particle accelerators: Large fields for smaller facility sources}, -author={Geddes, Cameron GR and Cormier-Michel, Estelle and Esarey, Eric H and Schroeder, Carl B and Vay, Jean-Luc and Leemans, Wim P and Bruhwiler, David L and Cary, John R and Cowan, Ben and Durant, Marc and others}, -year={2009}, -institution={Lawrence Berkeley National Laboratory (LBNL), Berkeley, CA (United States)} -} - -@article{Geddes2010, -title={Scaled simulation design of high quality laser wakefield accelerator stages}, -author={Geddes, CGR}, -journal={}, -year={2010} -} - -@article{Huang2009, -doi = {10.1088/1742-6596/180/1/012005}, -year = {2009}, -volume = {180}, -number = {1}, -pages = {012005}, -author = {C Huang and W An and V K Decyk and W Lu and W B Mori and F S Tsung and M Tzoufras and S Morshed and T Antonsen and B Feng and T Katsouleas and R A Fonseca and S F Martins and J Vieira and L O Silva and E Esarey and C G R Geddes and W P Leemans and E Cormier-Michel and J-L Vay and D L Bruhwiler and B Cowan and J R Cary and K Paul}, -title = {Recent results and future challenges for large scale particle-in-cell simulations of plasma-based accelerator concepts}, -journal = {Journal of Physics: Conference Series} -} - -@article{Leemans2014, -title = {Multi-GeV Electron Beams from Capillary-Discharge-Guided Subpetawatt Laser Pulses in the Self-Trapping Regime}, -author = {Leemans, W. P. and Gonsalves, A. J. and Mao, H.-S. and Nakamura, K. and Benedetti, C. and Schroeder, C. B. and T\'oth, Cs. and Daniels, J. and Mittelberger, D. E. and Bulanov, S. S. and Vay, J.-L. and Geddes, C. G. R. and Esarey, E.}, -journal = {Phys. Rev. Lett.}, -volume = {113}, -issue = {24}, -pages = {245002}, -numpages = {5}, -year = {2014}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.113.245002} -} - -@article{Blumenfeld2007, -title={Energy doubling of 42 GeV electrons in a metre-scale plasma wakefield accelerator}, -author={Blumenfeld, Ian and Clayton, Christopher E and Decker, Franz-Josef and Hogan, Mark J and Huang, Chengkun and Ischebeck, Rasmus and Iverson, Richard and Joshi, Chandrashekhar and Katsouleas, Thomas and Kirby, Neil and others}, -journal={Nature}, -volume={445}, -number={7129}, -pages={741--744}, -year={2007}, -publisher={Nature Publishing Group UK London}, -doi={10.1038/nature05538} -} - -@article{Bulanov2014, -doi = {10.3367/UFNe.0184.201412a.1265}, -year = {2014}, -publisher = {Turpion Ltd and the Russian Academy of Sciences}, -volume = {57}, -number = {12}, -pages = {1149}, -author = {S V Bulanov and J J Wilkens and T Zh Esirkepov and G Korn and G Kraft and S D Kraft and M Molls and V S Khoroshkov}, -title = {Laser ion acceleration for hadron therapy}, -journal = {Physics-Uspekhi} -} - -@article{Steinke2016, -title={Multistage coupling of independent laser-plasma accelerators}, -author={Steinke, S and Van Tilborg, J and Benedetti, C and Geddes, CGR and Schroeder, CB and Daniels, J and Swanson, KK and Gonsalves, AJ and Nakamura, K and Matlis, NH and others}, -journal={Nature}, -volume={530}, -number={7589}, -pages={190--193}, -year={2016}, -publisher={Nature Publishing Group UK London}, -doi={10.1038/nature16525} -} - -@article{Sprangle1990, -title = {Nonlinear theory of intense laser-plasma interactions}, -author = {Sprangle, P. and Esarey, E. and Ting, A.}, -journal = {Phys. Rev. Lett.}, -volume = {64}, -issue = {17}, -pages = {2011--2014}, -year = {1990}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.64.2011} -} - -@article{Antonsen1992, -title = {Self-focusing and Raman scattering of laser pulses in tenuous plasmas}, -author = {Antonsen, T. M. and Mora, P.}, -journal = {Phys. Rev. Lett.}, -volume = {69}, -issue = {15}, -pages = {2204--2207}, -year = {1992}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.69.2204} -} - -@article{Krall1993, -title = {Enhanced acceleration in a self-modulated-laser wake-field accelerator}, -author = {Krall, J. and Ting, A. and Esarey, E. and Sprangle, P.}, -journal = {Phys. Rev. E}, -volume = {48}, -issue = {3}, -pages = {2157--2161}, -year = {1993}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevE.48.2157} -} - -@article{Mora1997, -author = {Mora,Patrick and Antonsen, Jr.,Thomas M. }, -title = {Kinetic modeling of intense, short laser pulses propagating in tenuous plasmas}, -journal = {Physics of Plasmas}, -volume = {4}, -number = {1}, -pages = {217-229}, -year = {1997}, -doi = {10.1063/1.872134} -} - -@article{Huang2006, -title = {QUICKPIC: A highly efficient particle-in-cell code for modeling wakefield acceleration in plasmas}, -journal = {Journal of Computational Physics}, -volume = {217}, -number = {2}, -pages = {658-679}, -year = {2006}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2006.01.039}, -author = {C. Huang and V.K. Decyk and C. Ren and M. Zhou and W. Lu and W.B. Mori and J.H. Cooley and T.M. Antonsen and T. Katsouleas} -} - -@article{Benedetti2010, -author = {Benedetti,C. and Schroeder,C. B. and Esarey,E. and Geddes,C. G. R. and Leemans,W. P. }, -title = {Efficient Modeling of Laser‐Plasma Accelerators with INF\&RNO}, -journal = {AIP Conference Proceedings}, -volume = {1299}, -number = {1}, -pages = {250-255}, -year = {2010}, -doi = {10.1063/1.3520323} -} - -@article{Cowan2011, -title = {Characteristics of an envelope model for laser–plasma accelerator simulation}, -journal = {Journal of Computational Physics}, -volume = {230}, -number = {1}, -pages = {61-86}, -year = {2011}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2010.09.009}, -author = {Benjamin M. Cowan and David L. Bruhwiler and Estelle Cormier-Michel and Eric Esarey and Cameron G.R. Geddes and Peter Messmer and Kevin M. Paul} -} - -@article{Vay2007, -title = {Noninvariance of Space- and Time-Scale Ranges under a Lorentz Transformation and the Implications for the Study of Relativistic Interactions}, -author = {Vay, J.-L.}, -journal = {Phys. Rev. Lett.}, -volume = {98}, -issue = {13}, -pages = {130405}, -year = {2007}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.98.130405} -} - -@article{Vay2009a, -doi = {10.1088/1742-6596/180/1/012006}, -year = {2009}, -volume = {180}, -number = {1}, -pages = {012006}, -author = {J-L Vay and D L Bruhwiler and C G R Geddes and W M Fawley and S F Martins and J R Cary and E Cormier-Michel and B Cowan and R A Fonseca and M A Furman and W Lu and W B Mori and L O Silva}, -title = {Simulating relativistic beam and plasma systems using an optimal boosted frame}, -journal = {Journal of Physics: Conference Series} -} - -@article{Vay2009b, -title = {Application of the reduction of scale range in a Lorentz boosted frame to the numerical simulation of particle acceleration devices.}, -author = {Vay, J and Fawley, W M and Geddes, C G and Cormier-Michel, E and Grote, D P}, -url = {https://www.osti.gov/biblio/952754}, -journal = {}, -place = {United States}, -year = {2009}, -} - -@article{Vay2010, -author = {Vay,J.‐L. and Geddes,C. G. R. and Benedetti,C. and Bruhwiler,D. L. and Cormier‐Michel,E. and Cowan,B. M. and Cary,J. R. and Grote,D. P. }, -title = {Modeling Laser Wakefield Accelerators in a Lorentz Boosted Frame}, -journal = {AIP Conference Proceedings}, -volume = {1299}, -number = {1}, -pages = {244-249}, -year = {2010}, -doi = {10.1063/1.3520322} -} - -@article{Vay2011a, -title = {Numerical methods for instability mitigation in the modeling of laser wakefield accelerators in a Lorentz-boosted frame}, -journal = {Journal of Computational Physics}, -volume = {230}, -number = {15}, -pages = {5908-5929}, -year = {2011}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2011.04.003}, -author = {J.-L. Vay and C.G.R. Geddes and E. Cormier-Michel and D.P. Grote} -} - -@article{Vay2011b, -author = {Vay,J.-L. and Geddes,C. G. R. and Cormier-Michel,E. and Grote,D. P. }, -title = {Effects of hyperbolic rotation in Minkowski space on the modeling of plasma accelerators in a Lorentz boosted frame}, -journal = {Physics of Plasmas}, -volume = {18}, -number = {3}, -pages = {030701}, -year = {2011}, -doi = {10.1063/1.3559483} -} - -@article{Vay2011c, -author = {Vay,J.-L. and Geddes,C. G. R. and Esarey,E. and Schroeder,C. B. and Leemans,W. P. and Cormier-Michel,E. and Grote,D. P. }, -title = {Modeling of 10 GeV-1 TeV laser-plasma accelerators using Lorentz boosted simulations}, -journal = {Physics of Plasmas}, -volume = {18}, -number = {12}, -pages = {123103}, -year = {2011}, -doi = {10.1063/1.3663841} -} - -@article{Martins2010a, -title = {Numerical simulations of laser wakefield accelerators in optimal Lorentz frames}, -journal = {Computer Physics Communications}, -volume = {181}, -number = {5}, -pages = {869-875}, -year = {2010}, -issn = {0010-4655}, -doi = {10.1016/j.cpc.2009.12.023}, -author = {Samuel F. Martins and Ricardo A. Fonseca and Luís O. Silva and Wei Lu and Warren B. Mori} -} - -@article{Martins2010b, -title={Exploring laser-wakefield-accelerator regimes for near-term lasers using particle-in-cell simulation in Lorentz-boosted frames}, -author={Martins, Samuel F and Fonseca, RA and Lu, Wei and Mori, Warren B and Silva, LO}, -journal={Nature Physics}, -volume={6}, -number={4}, -pages={311--316}, -year={2010}, -publisher={Nature Publishing Group UK London}, -doi={10.1038/nphys1538} -} - -@article{Martins2010c, -author = {Martins,S. F. and Fonseca,R. A. and Vieira,J. and Silva,L. O. and Lu,W. and Mori,W. B. }, -title = {Modeling laser wakefield accelerator experiments with ultrafast particle-in-cell simulations in boosted frames}, -journal = {Physics of Plasmas}, -volume = {17}, -number = {5}, -pages = {056705}, -year = {2010}, -doi = {10.1063/1.3358139} -} - -@article{Bruhwiler2009, -author = {Bruhwiler,David L. and Cary,John R. and Cowan,Benjamin M. and Paul,Kevin and Geddes,Cameron G. R. and Mullowney,Paul J. and Messmer,Peter and Esarey,Eric and Cormier‐Michel,Estelle and Leemans,Wim and Vay,Jean‐Luc }, -title = {New Developments in the Simulation of Advanced Accelerator Concepts}, -journal = {AIP Conference Proceedings}, -volume = {1086}, -number = {1}, -pages = {29-37}, -year = {2009}, -doi = {10.1063/1.3080922} -} - -@article{Yu2016, -title = {Enabling Lorentz boosted frame particle-in-cell simulations of laser wakefield acceleration in quasi-3D geometry}, -journal = {Journal of Computational Physics}, -volume = {316}, -pages = {747-759}, -year = {2016}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2016.04.014}, -author = {Peicheng Yu and Xinlu Xu and Asher Davidson and Adam Tableman and Thamine Dalichaouch and Fei Li and Michael D. Meyers and Weiming An and Frank S. Tsung and Viktor K. Decyk and Frederico Fiuza and Jorge Vieira and Ricardo A. Fonseca and Wei Lu and Luis O. Silva and Warren B. Mori} -} - -@inproceedings{Godfrey1985, -title={The IPROP Three-Dimensional Beam Propagation Code}, -booktitle={}, -author={B. B. Godfrey}, -year={1985} -} - -@article{Lifschitz2009, -title = {Particle-in-Cell modelling of laser–plasma interaction using Fourier decomposition}, -journal = {Journal of Computational Physics}, -volume = {228}, -number = {5}, -pages = {1803-1814}, -year = {2009}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2008.11.017}, -author = {A.F. Lifschitz and X. Davoine and E. Lefebvre and J. Faure and C. Rechatin and V. Malka} -} - -@article{Davidson2015, -title = {Implementation of a hybrid particle code with a PIC description in r–z and a gridless description in ϕ into OSIRIS}, -journal = {Journal of Computational Physics}, -volume = {281}, -pages = {1063-1077}, -year = {2015}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2014.10.064}, -author = {A. Davidson and A. Tableman and W. An and F.S. Tsung and W. Lu and J. Vieira and R.A. Fonseca and L.O. Silva and W.B. Mori} -} - -@article{Lehe2016, -title = {A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm}, -journal = {Computer Physics Communications}, -volume = {203}, -pages = {66-82}, -year = {2016}, -issn = {0010-4655}, -doi = {10.1016/j.cpc.2016.02.007}, -author = {Rémi Lehe and Manuel Kirchen and Igor A. Andriyash and Brendan B. Godfrey and Jean-Luc Vay} -} - -@article{Andriyash2016, -author = {Andriyash,Igor A. and Lehe,Remi and Lifschitz,Agustin }, -title = {Laser-plasma interactions with a Fourier-Bessel particle-in-cell method}, -journal = {Physics of Plasmas}, -volume = {23}, -number = {3}, -pages = {033110}, -year = {2016}, -doi = {10.1063/1.4943281} -} - -@article{Shadwick2009, -author = {Shadwick,B. A. and Schroeder,C. B. and Esarey,E. }, -title = {Nonlinear laser energy depletion in laser-plasma accelerators}, -journal = {Physics of Plasmas}, -volume = {16}, -number = {5}, -pages = {056704}, -year = {2009}, -doi = {10.1063/1.3124185} -} - -@article{CormierMichel2009, -author = {Cormier‐Michel,Estelle and Geddes,C. G. R. and Esarey,E. and Schroeder,C. B. and Bruhwiler,D. L. and Paul,K. and Cowan,B. and Leemans,W. P. }, -title = {Scaled simulations of a 10 GeV accelerator}, -journal = {AIP Conference Proceedings}, -volume = {1086}, -number = {1}, -pages = {297-302}, -year = {2009}, -doi = {10.1063/1.3080921} -} - @ARTICLE{Birdsall1991, author = {Birdsall, C.K.}, journal = {IEEE Transactions on Plasma Science}, @@ -434,7 +48,7 @@ @misc{Lim2007 @article{Turner2013, author = {Turner, M. M. and Derzsi, A. and Donkó, Z. and Eremin, D. and Kelly, S. J. and Lafleur, T. and Mussenbrock, T.}, -title = "{Simulation benchmarks for low-pressure plasmas: Capacitive discharges}", +title = {{Simulation benchmarks for low-pressure plasmas: Capacitive discharges}}, journal = {Physics of Plasmas}, volume = {20}, number = {1}, diff --git a/Docs/source/theory/intro.rst b/Docs/source/theory/intro.rst index cbf56c919df..10fbba679c2 100644 --- a/Docs/source/theory/intro.rst +++ b/Docs/source/theory/intro.rst @@ -8,16 +8,17 @@ Introduction Plasma laser-driven (top) and charged-particles-driven (bottom) acceleration (rendering from 3-D Particle-In-Cell simulations). A laser beam (red and blue disks in top picture) or a charged particle beam (red dots in bottom picture) propagating (from left to right) through an under-dense plasma (not represented) displaces electrons, creating a plasma wakefield that supports very high electric fields (pale blue and yellow). These electric fields, which can be orders of magnitude larger than with conventional techniques, can be used to accelerate a short charged particle beam (white) to high-energy over a very short distance. -Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`Tsung2006,Geddes2008,Geddes2009,Geddes2010,Huang2009`, with -accurate modeling of wake formation, electron self-trapping and acceleration requiring fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`Leemans2014,Blumenfeld2007,Bulanov2014,Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play. +Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`i-Tsungpop06,i-Geddesjp08,i-Geddesscidac09,i-Geddespac09,i-Huangscidac09`, with +accurate modeling of wake formation, electron self-trapping and acceleration requiring fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`i-LeemansPRL2014,i-Blumenfeld2007,i-BulanovSV2014,i-Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play. For such simulations, the most popular algorithm is the Particle-In-Cell (or PIC) technique, which represents electromagnetic fields on a grid and particles by a sample of macroparticles. However, these simulations are extremely computationally intensive, due to the need to resolve the evolution of a driver (laser or particle beam) and an accelerated beam into a structure that is orders of magnitude longer and wider than the accelerated beam. -Various techniques or reduced models have been developed to allow multidimensional simulations at manageable computational costs: quasistatic approximation :cite:p:`Sprangle1990,Antonsen1992,Krall1993,Mora1997,Huang2006`, -ponderomotive guiding center (PGC) models :cite:p:`Antonsen1992,Krall1993,Huang2006,Benedetti2010,Cowan2011`, simulation in an optimal Lorentz boosted frame :cite:p:`Vay2007,Bruhwiler2009,Vay2009a,Vay2009b,Vay2010,Martins2010a,Martins2010b,Martins2010c,Vay2011a,Vay2011b,Vay2011c,Yu2016`, -expanding the fields into a truncated series of azimuthal modes :cite:p:`Godfrey1985,Lifschitz2009,Davidson2015,Lehe2016,Andriyash2016`, fluid approximation :cite:p:`Krall1993,Shadwick2009,Benedetti2010` and scaled parameters :cite:p:`CormierMichel2009`. +Various techniques or reduced models have been developed to allow multidimensional simulations at manageable computational costs: quasistatic approximation :cite:p:`i-Sprangleprl90,i-Antonsenprl1992,i-Krallpre1993,i-Morapop1997,i-Quickpic`, +ponderomotive guiding center (PGC) models :cite:p:`i-Antonsenprl1992,i-Krallpre1993,i-Quickpic,i-Benedettiaac2010,i-Cowanjcp11`, simulation in an optimal Lorentz boosted frame :cite:p:`i-Vayprl07,i-Bruhwileraac08,i-Vayscidac09,i-Vaypac09,i-Martinspac09,i-VayAAC2010,i-Martinsnaturephysics10,i-Martinspop10,i-Martinscpc10,i-Vayjcp2011,i-VayPOPL2011,i-Vaypop2011,i-Yu2016`, +expanding the fields into a truncated series of azimuthal modes :cite:p:`i-godfrey1985iprop,i-LifschitzJCP2009,i-DavidsonJCP2015,i-Lehe2016,i-AndriyashPoP2016`, fluid approximation :cite:p:`i-Krallpre1993,i-Shadwickpop09,i-Benedettiaac2010` and scaled parameters :cite:p:`i-Cormieraac08,i-Geddespac09`. .. bibliography:: + :keyprefix: i- diff --git a/Examples/Physics_applications/laser_ion/README.rst b/Examples/Physics_applications/laser_ion/README.rst index 25e25f70d84..f382b36bc85 100644 --- a/Examples/Physics_applications/laser_ion/README.rst +++ b/Examples/Physics_applications/laser_ion/README.rst @@ -3,7 +3,7 @@ Laser-Ion Acceleration with a Planar Target =========================================== -This example shows how to model laser-ion acceleration with planar targets of solid density :cite:p:`ex-Wilks2001,Bulanov2008,ex-Macchi2013`. +This example shows how to model laser-ion acceleration with planar targets of solid density :cite:p:`ex-Wilks2001,ex-Bulanov2008,ex-Macchi2013`. The acceleration mechanism in this scenario depends on target parameters Although laser-ion acceleration requires full 3D modeling for adequate description of the acceleration dynamics, especially the acceleration field lengths and decay times, this example models a 2D example. diff --git a/Examples/Physics_applications/plasma_acceleration/README.rst b/Examples/Physics_applications/plasma_acceleration/README.rst index b6a2b9952c6..d5605dd93ef 100644 --- a/Examples/Physics_applications/plasma_acceleration/README.rst +++ b/Examples/Physics_applications/plasma_acceleration/README.rst @@ -3,7 +3,7 @@ Beam-Driven Wakefield Acceleration of Electrons =============================================== -This example shows how to model a beam-driven plasma-wakefield accelerator (PWFA) :cite:p:`TajimaDawson1982,Esarey1996`. +This example shows how to model a beam-driven plasma-wakefield accelerator (PWFA) :cite:p:`ex-TajimaDawson1982,ex-Esarey1996`. PWFA is best performed in 3D and quasi-cylindrical (RZ) geometry, which ensures that the plasma wavelength of the wakefield is modelled with the right scale lengths. RZ modeling enables efficient modeling if effects of asymmetry shall be ignored (e.g., asymmetric beams and transverse profiles, hosing of the injected beam, etc.). diff --git a/Examples/Physics_applications/plasma_mirror/README.rst b/Examples/Physics_applications/plasma_mirror/README.rst index b115d6fa586..8741db09699 100644 --- a/Examples/Physics_applications/plasma_mirror/README.rst +++ b/Examples/Physics_applications/plasma_mirror/README.rst @@ -3,7 +3,7 @@ Plasma-Mirror ============= -This example shows how to model a plasma mirror, using a planar target of solid density :cite:p:`Dromey2004,Roedel2010`. +This example shows how to model a plasma mirror, using a planar target of solid density :cite:p:`ex-Dromey2004,ex-Roedel2010`. Although laser-solid interaction modeling requires full 3D modeling for adequate description of the dynamics at play, this example models a 2D example. 2D modeling provide a qualitative overview of the dynamics, but mostly saves computational costs since the plasma frequency (and Debye length) of the surface plasma determines the resolution need in laser-solid interaction modeling. From 044ab2917a8b13f0edafadfe5dc3f8da0b351ed8 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Sat, 9 Dec 2023 14:33:28 -0800 Subject: [PATCH 027/176] macOS CI: Fix brew installation (#4493) It appears that we need to call brew upgrade. --- .github/workflows/macos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 32a31ea66f8..0e8819032e3 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -23,6 +23,7 @@ jobs: set +e brew unlink gcc brew update + brew upgrade || true brew install --overwrite python brew install ccache brew install fftw From 10253709915bcfda396a700dcab42b90211bea8b Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Sat, 9 Dec 2023 17:18:06 -0800 Subject: [PATCH 028/176] sqrt -> std::sqrt (#4490) Unlike std::sqrt that has overloads for double and float, sqrt is a C function that takes a double. If the argument is a float, it will have to be promoted to double first. --- Source/Initialization/InjectorMomentum.H | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index d5f19214e3c..98caae1bb23 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -341,7 +341,7 @@ struct InjectorMomentumBoltzmann for (auto& el : u) el = 0.0_rt; const amrex::Real beta = velocity(x,y,z); int const dir = velocity.direction(); - const auto gamma = static_cast(1._rt/sqrt(1._rt-beta*beta)); + const auto gamma = 1._rt/std::sqrt(1._rt-beta*beta); u[dir] = gamma*beta; return amrex::XDim3 {u[0],u[1],u[2]}; } @@ -444,7 +444,7 @@ struct InjectorMomentumJuttner for (auto& el : u) el = 0.0_rt; amrex::Real const beta = velocity(x,y,z); int const dir = velocity.direction(); - auto const gamma = static_cast(1._rt/sqrt(1._rt-beta*beta)); + auto const gamma = 1._rt/std::sqrt(1._rt-beta*beta); u[dir] = gamma*beta; return amrex::XDim3 {u[0],u[1],u[2]}; } From a78691226ae6c2f8367497f85a3191db0498c082 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Mon, 11 Dec 2023 02:13:57 -0800 Subject: [PATCH 029/176] Clang-Tidy: Update performance and readability (#4492) For performance and readability, instead of including checks explicitly, switch to excluding checks explicitly. These checks are excluded because currently WarpX fails at them. Add portability and diagnostic checks. --- .clang-tidy | 53 +++++++++++++++++++++-------------------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index a659b8dc5be..5bf23c82692 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -7,6 +7,7 @@ Checks: ' -bugprone-unchecked-optional-access, cert-*, -cert-err58-cpp, + clang-diagnostic-*, cppcoreguidelines-*, -cppcoreguidelines-avoid-c-arrays, -cppcoreguidelines-avoid-magic-numbers, @@ -29,38 +30,26 @@ Checks: ' -modernize-return-braced-init-list, -modernize-use-trailing-return-type, mpi-*, - performance-faster-string-find, - performance-for-range-copy, - performance-implicit-conversion-in-loop, - performance-inefficient-algorithm, - performance-inefficient-string-concatenation, - performance-inefficient-vector-operation, - performance-move-const-arg, - performance-move-constructor-init, - performance-no-automatic-move, - performance-no-int-to-ptr, - readability-avoid-const-params-in-decls, - readability-const-return-type, - readability-container-contains, - readability-container-data-pointer, - readability-container-size-empty, - readability-non-const-parameter, - readability-redundant-control-flow, - readability-redundant-declaration, - readability-redundant-function-ptr-dereference, - readability-redundant-member-init, - readability-redundant-preprocessor, - readability-redundant-smartptr-get, - readability-redundant-string-cstr, - readability-redundant-string-init, - readability-simplify-boolean-expr, - readability-simplify-subscript-expr, - readability-static-accessed-through-instance, - readability-static-definition-in-anonymous-namespace, - readability-string-compare, - readability-suspicious-call-argument, - readability-uniqueptr-delete-release, - readability-use-anyofallof, + performance-*, + -performance-noexcept-move-constructor, + -performance-type-promotion-in-math-fn, + -performance-unnecessary-copy-initialization, + -performance-unnecessary-value-param, + portability-*, + readability-*, + -readability-braces-around-statements, + -readability-convert-member-functions-to-static, + -readability-duplicate-include, + -readability-else-after-return, + -readability-function-cognitive-complexity, + -readability-identifier-length, + -readability-implicit-bool-conversion, + -readability-inconsistent-declaration-parameter-name, + -readability-isolate-declaration, + -readability-magic-numbers, + -readability-make-member-function-const, + -readability-named-parameter, + -readability-qualified-auto, -readability-uppercase-literal-suffix ' From 8995d1bf4d8b735fbd1b818d817a76205f32f34a Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Mon, 11 Dec 2023 02:15:26 -0800 Subject: [PATCH 030/176] Clang-Tidy CI: Keep Going after Errors (#4491) Add `--keep-going` to the make command in Clang-Tidy CI. With that, the job will keep going and show all the clang-tidy check errors instead of stopping on the first error. --- .github/workflows/clang_tidy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index c0d039b7f17..b8b0053adaa 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -50,7 +50,7 @@ jobs: cmake --build build_clang_tidy -j 2 ${{github.workspace}}/.github/workflows/source/makeMakefileForClangTidy.py --input ${{github.workspace}}/ccache.log.txt - make -j2 -f clang-tidy-ccache-misses.mak \ + make -j2 --keep-going -f clang-tidy-ccache-misses.mak \ CLANG_TIDY=clang-tidy \ CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*" From ee98582bfe961dc8cf313a1760a4b37c15061fce Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Mon, 11 Dec 2023 14:23:50 -0800 Subject: [PATCH 031/176] Tidy-Clang: performance-type-promotion-in-math-fn (#4497) --- .clang-tidy | 1 - Source/Diagnostics/BTDiagnostics.cpp | 2 +- Source/Initialization/WarpXInitData.cpp | 10 +++++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 5bf23c82692..8ef1234b125 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -32,7 +32,6 @@ Checks: ' mpi-*, performance-*, -performance-noexcept-move-constructor, - -performance-type-promotion-in-math-fn, -performance-unnecessary-copy-initialization, -performance-unnecessary-value-param, portability-*, diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index aed62f8e02c..dd8f17af3ad 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -873,7 +873,7 @@ BTDiagnostics::k_index_zlab (int i_buffer, int lev) const amrex::Real prob_domain_zmin_lab = m_snapshot_domain_lab[i_buffer].lo( m_moving_window_dir ); auto ref_ratio = amrex::IntVect(1); if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1); - const int k_lab = static_cast(floor ( + const int k_lab = static_cast(std::floor ( ( m_current_z_lab[i_buffer] - (prob_domain_zmin_lab ) ) / dz_lab( warpx.getdt(lev), ref_ratio[m_moving_window_dir] ) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index c1abf74cba5..b742a7ceb19 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1507,8 +1507,8 @@ WarpX::ReadExternalFieldFromFile ( #if defined(WARPX_DIM_RZ) // Get index of the external field array - int const ir = floor( (x0-offset0)/file_dr ); - int const iz = floor( (x1-offset1)/file_dz ); + int const ir = std::floor( (x0-offset0)/file_dr ); + int const iz = std::floor( (x1-offset1)/file_dz ); // Get coordinates of external grid point amrex::Real const xx0 = offset0 + ir * file_dr; @@ -1521,9 +1521,9 @@ WarpX::ReadExternalFieldFromFile ( else { x2 = real_box.lo(2) + k*dx[2] + 0.5_rt*dx[2]; } // Get index of the external field array - int const ix = floor( (x0-offset0)/file_dx ); - int const iy = floor( (x1-offset1)/file_dy ); - int const iz = floor( (x2-offset2)/file_dz ); + int const ix = std::floor( (x0-offset0)/file_dx ); + int const iy = std::floor( (x1-offset1)/file_dy ); + int const iz = std::floor( (x2-offset2)/file_dz ); // Get coordinates of external grid point amrex::Real const xx0 = offset0 + ix * file_dx; From 903a67cfad71569dbfc5279d6e25d44e86aa7205 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Mon, 11 Dec 2023 23:24:36 +0100 Subject: [PATCH 032/176] Move external field parameters out of WarpX class (#4441) * move external field parameters into a dedicated class * add full header in Python/WarpX.cpp * fix bug introduced in Make.package * enforce inputfile syntax * fix issue found with clang-tidy * remove unwanted includes * added missing include * replace init function with constructor * update doc * fix bug and add curly braces --- Source/Initialization/CMakeLists.txt | 9 +- Source/Initialization/ExternalField.H | 69 +++++ Source/Initialization/ExternalField.cpp | 180 ++++++++++++ Source/Initialization/ExternalField_fwd.H | 13 + Source/Initialization/Make.package | 11 +- Source/Initialization/WarpXInitData.cpp | 316 +++++++++------------- Source/Parallelization/WarpXRegrid.cpp | 5 +- Source/Python/WarpX.cpp | 1 + Source/Utils/WarpXMovingWindow.cpp | 35 +-- Source/WarpX.H | 38 +-- Source/WarpX.cpp | 30 +- 11 files changed, 432 insertions(+), 275 deletions(-) create mode 100644 Source/Initialization/ExternalField.H create mode 100644 Source/Initialization/ExternalField.cpp create mode 100644 Source/Initialization/ExternalField_fwd.H diff --git a/Source/Initialization/CMakeLists.txt b/Source/Initialization/CMakeLists.txt index 35767e87c50..8931de740ad 100644 --- a/Source/Initialization/CMakeLists.txt +++ b/Source/Initialization/CMakeLists.txt @@ -2,14 +2,15 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE - WarpXAMReXInit.cpp + ExternalField.cpp + GetTemperature.cpp + GetVelocity.cpp InjectorDensity.cpp InjectorMomentum.cpp PlasmaInjector.cpp - WarpXInitData.cpp TemperatureProperties.cpp VelocityProperties.cpp - GetTemperature.cpp - GetVelocity.cpp + WarpXAMReXInit.cpp + WarpXInitData.cpp ) endforeach() diff --git a/Source/Initialization/ExternalField.H b/Source/Initialization/ExternalField.H new file mode 100644 index 00000000000..7b64339a042 --- /dev/null +++ b/Source/Initialization/ExternalField.H @@ -0,0 +1,69 @@ +/* Copyright 2023 Luca Fedeli + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef EXTERNAL_FIELD_H_ +#define EXTERNAL_FIELD_H_ + +#include "ExternalField_fwd.H" + +#include +#include +#include +#include + +#include +#include + +enum class ExternalFieldType +{ + default_zero, + constant, + parse_ext_grid_function, + read_from_file +}; + +/** + * \brief Struct to store data related to external electromagnetic fields + * (flags, field values, and field parsers) + */ +struct ExternalFieldParams +{ + + /** + * \brief The constructor reads and stores the parameters related to the external fields. + * "pp_warpx" must point at the "warpx" parameter group in the inputfile. + */ + ExternalFieldParams(const amrex::ParmParse& pp_warpx); + + //! Initial electric field on the grid + amrex::GpuArray E_external_grid = {0,0,0}; + //! Initial magnetic field on the grid + amrex::GpuArray B_external_grid = {0,0,0}; + + //! Initialization type for external magnetic field on the grid + ExternalFieldType B_ext_grid_type = ExternalFieldType::default_zero; + //! Initialization type for external electric field on the grid + ExternalFieldType E_ext_grid_type = ExternalFieldType::default_zero; + + //! User-defined parser to initialize x-component of the magnetic field on the grid + std::unique_ptr Bxfield_parser; + //! User-defined parser to initialize y-component of the magnetic field on the grid + std::unique_ptr Byfield_parser; + //! User-defined parser to initialize z-component of the magnetic field on the grid + std::unique_ptr Bzfield_parser; + //! User-defined parser to initialize x-component of the electric field on the grid + std::unique_ptr Exfield_parser; + //! User-defined parser to initialize y-component of the electric field on the grid + std::unique_ptr Eyfield_parser; + //! User-defined parser to initialize z-component of the electric field on the grid + std::unique_ptr Ezfield_parser; + + //! Path of the file where external fields are stored + std::string external_fields_path; +}; + +#endif //EXTERNAL_FIELD_H_ diff --git a/Source/Initialization/ExternalField.cpp b/Source/Initialization/ExternalField.cpp new file mode 100644 index 00000000000..04d34c59a32 --- /dev/null +++ b/Source/Initialization/ExternalField.cpp @@ -0,0 +1,180 @@ +/* Copyright 2023 Luca Fedeli + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "ExternalField.H" + +#include "Utils/TextMsg.H" +#include "Utils/Parser/ParserUtils.H" + +#include + +#include +#include + +namespace +{ + + enum class EMFieldType{E, B}; + + template + ExternalFieldType string_to_external_field_type(std::string s) + { + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + + if constexpr (T == EMFieldType::E){ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(s != "parse_b_ext_grid_function", + "parse_B_ext_grid_function can be used only for B_ext_grid_init_style"); + } + else{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(s != "parse_e_ext_grid_function", + "parse_E_ext_grid_function can be used only for E_ext_grid_init_style"); + } + + if ( s.empty() || s == "default"){ + return ExternalFieldType::default_zero; + } + else if ( s == "constant"){ + return ExternalFieldType::constant; + } + else if ( s == "parse_b_ext_grid_function" || s == "parse_e_ext_grid_function"){ + return ExternalFieldType::parse_ext_grid_function; + } + else if ( s == "read_from_file"){ + return ExternalFieldType::read_from_file; + } + else{ + WARPX_ABORT_WITH_MESSAGE( + "'" + s + "' is an unknown external field type!"); + } + + return ExternalFieldType::default_zero; + } +} + +ExternalFieldParams::ExternalFieldParams(const amrex::ParmParse& pp_warpx) +{ + // default values of E_external_grid and B_external_grid + // are used to set the E and B field when "constant" or + // "parser" is not explicitly used in the input. + std::string B_ext_grid_s; + pp_warpx.query("B_ext_grid_init_style", B_ext_grid_s); + B_ext_grid_type = string_to_external_field_type(B_ext_grid_s); + + std::string E_ext_grid_s; + pp_warpx.query("E_ext_grid_init_style", E_ext_grid_s); + E_ext_grid_type = string_to_external_field_type(E_ext_grid_s); + + // + // Constant external field + // + + // if the input string is "constant", the values for the + // external grid must be provided in the input. + auto v_B = std::vector(3); + if (B_ext_grid_type == ExternalFieldType::constant) + utils::parser::getArrWithParser(pp_warpx, "B_external_grid", v_B); + std::copy(v_B.begin(), v_B.end(), B_external_grid.begin()); + + // if the input string is "constant", the values for the + // external grid must be provided in the input. + auto v_E = std::vector(3); + if (E_ext_grid_type == ExternalFieldType::constant) + utils::parser::getArrWithParser(pp_warpx, "E_external_grid", v_E); + std::copy(v_E.begin(), v_E.end(), E_external_grid.begin()); + //___________________________________________________________________________ + + + // + // External E field with parser + // + + // if the input string for the B-field is "parse_b_ext_grid_function", + // then the analytical expression or function must be + // provided in the input file. + if (B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + + //! Strings storing parser function to initialize the components of the magnetic field on the grid + std::string str_Bx_ext_grid_function; + std::string str_By_ext_grid_function; + std::string str_Bz_ext_grid_function; + +#ifdef WARPX_DIM_RZ + std::stringstream warnMsg; + warnMsg << "Parser for external B (r and theta) fields does not work with RZ\n" + << "The initial Br and Bt fields are currently hardcoded to 0.\n" + << "The initial Bz field should only be a function of z.\n"; + ablastr::warn_manager::WMRecordWarning( + "Inputs", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); + str_Bx_ext_grid_function = "0"; + str_By_ext_grid_function = "0"; +#else + utils::parser::Store_parserString(pp_warpx, "Bx_external_grid_function(x,y,z)", + str_Bx_ext_grid_function); + utils::parser::Store_parserString(pp_warpx, "By_external_grid_function(x,y,z)", + str_By_ext_grid_function); +#endif + utils::parser::Store_parserString(pp_warpx, "Bz_external_grid_function(x,y,z)", + str_Bz_ext_grid_function); + + Bxfield_parser = std::make_unique( + utils::parser::makeParser(str_Bx_ext_grid_function,{"x","y","z"})); + Byfield_parser = std::make_unique( + utils::parser::makeParser(str_By_ext_grid_function,{"x","y","z"})); + Bzfield_parser = std::make_unique( + utils::parser::makeParser(str_Bz_ext_grid_function,{"x","y","z"})); + } + //___________________________________________________________________________ + + + // + // External B field with parser + // + + // if the input string for the E-field is "parse_e_ext_grid_function", + // then the analytical expression or function must be + // provided in the input file. + if (E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + +#ifdef WARPX_DIM_RZ + WARPX_ABORT_WITH_MESSAGE( + "E parser for external fields does not work with RZ -- TO DO"); +#endif + + //! Strings storing parser function to initialize the components of the electric field on the grid + std::string str_Ex_ext_grid_function; + std::string str_Ey_ext_grid_function; + std::string str_Ez_ext_grid_function; + + utils::parser::Store_parserString(pp_warpx, "Ex_external_grid_function(x,y,z)", + str_Ex_ext_grid_function); + utils::parser::Store_parserString(pp_warpx, "Ey_external_grid_function(x,y,z)", + str_Ey_ext_grid_function); + utils::parser::Store_parserString(pp_warpx, "Ez_external_grid_function(x,y,z)", + str_Ez_ext_grid_function); + + Exfield_parser = std::make_unique( + utils::parser::makeParser(str_Ex_ext_grid_function,{"x","y","z"})); + Eyfield_parser = std::make_unique( + utils::parser::makeParser(str_Ey_ext_grid_function,{"x","y","z"})); + Ezfield_parser = std::make_unique( + utils::parser::makeParser(str_Ez_ext_grid_function,{"x","y","z"})); + } + //___________________________________________________________________________ + + + // + // External fields from file + // + + if (E_ext_grid_type == ExternalFieldType::read_from_file || + B_ext_grid_type == ExternalFieldType::read_from_file){ + std::string read_fields_from_path="./"; + pp_warpx.query("read_fields_from_path", external_fields_path); + } + //___________________________________________________________________________ +} diff --git a/Source/Initialization/ExternalField_fwd.H b/Source/Initialization/ExternalField_fwd.H new file mode 100644 index 00000000000..f7a8547ecce --- /dev/null +++ b/Source/Initialization/ExternalField_fwd.H @@ -0,0 +1,13 @@ +/* Copyright 2023 Luca Fedeli + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef EXTERNAL_FIELD_FWD_H_ +#define EXTERNAL_FIELD_FWD_H_ + +struct ExternalFieldParams; + +#endif //EXTERNAL_FIELD_FWD_H_ diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index 13eed49639f..8b4a4c1d669 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -1,11 +1,12 @@ -CEXE_sources += WarpXAMReXInit.cpp -CEXE_sources += WarpXInitData.cpp -CEXE_sources += PlasmaInjector.cpp +CEXE_sources += ExternalField.cpp +CEXE_sources += GetTemperature.cpp +CEXE_sources += GetVelocity.cpp CEXE_sources += InjectorDensity.cpp CEXE_sources += InjectorMomentum.cpp +CEXE_sources += PlasmaInjector.cpp CEXE_sources += TemperatureProperties.cpp -CEXE_sources += GetTemperature.cpp CEXE_sources += VelocityProperties.cpp -CEXE_sources += GetVelocity.cpp +CEXE_sources += WarpXAMReXInit.cpp +CEXE_sources += WarpXInitData.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Initialization diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index b742a7ceb19..bd8e6334f48 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -20,6 +20,7 @@ #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Filter/BilinearFilter.H" #include "Filter/NCIGodfreyFilter.H" +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Utils/Algorithms/LinearInterpolation.H" #include "Utils/Logo/GetLogo.H" @@ -506,12 +507,12 @@ void WarpX::AddExternalFields () { for (int lev = 0; lev <= finest_level; ++lev) { // FIXME: RZ multimode has more than one component for all these - if (add_external_E_field) { + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { amrex::MultiFab::Add(*Efield_fp[lev][0], *Efield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][1], *Efield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][2], *Efield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); } - if (add_external_B_field) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { amrex::MultiFab::Add(*Bfield_fp[lev][0], *Bfield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][1], *Bfield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][2], *Bfield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); @@ -729,34 +730,6 @@ WarpX::PostRestart () void WarpX::InitLevelData (int lev, Real /*time*/) { - - const ParmParse pp_warpx("warpx"); - - // default values of E_external_grid and B_external_grid - // are used to set the E and B field when "constant" or - // "parser" is not explicitly used in the input. - pp_warpx.query("B_ext_grid_init_style", B_ext_grid_s); - std::transform(B_ext_grid_s.begin(), - B_ext_grid_s.end(), - B_ext_grid_s.begin(), - ::tolower); - - pp_warpx.query("E_ext_grid_init_style", E_ext_grid_s); - std::transform(E_ext_grid_s.begin(), - E_ext_grid_s.end(), - E_ext_grid_s.begin(), - ::tolower); - - // if the input string is "constant", the values for the - // external grid must be provided in the input. - if (B_ext_grid_s == "constant") - utils::parser::getArrWithParser(pp_warpx, "B_external_grid", B_external_grid); - - // if the input string is "constant", the values for the - // external grid must be provided in the input. - if (E_ext_grid_s == "constant") - utils::parser::getArrWithParser(pp_warpx, "E_external_grid", E_external_grid); - // initialize the averaged fields only if the averaged algorithm // is activated ('psatd.do_time_averaging=1') const ParmParse pp_psatd("psatd"); @@ -766,37 +739,40 @@ WarpX::InitLevelData (int lev, Real /*time*/) // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. // The default maxlevel_extEMfield_init value is the total number of levels in the simulation - if ( ( B_ext_grid_s == "constant") && (lev <= maxlevel_extEMfield_init) ) + const auto is_B_ext_const = + m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::constant || + m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::default_zero; + if ( is_B_ext_const && (lev <= maxlevel_extEMfield_init) ) { - Bfield_fp[lev][i]->setVal(B_external_grid[i]); - if (fft_do_time_averaging) { - Bfield_avg_fp[lev][i]->setVal(B_external_grid[i]); - } + Bfield_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + if (fft_do_time_averaging) + Bfield_avg_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); if (lev > 0) { - Bfield_aux[lev][i]->setVal(B_external_grid[i]); - Bfield_cp[lev][i]->setVal(B_external_grid[i]); - if (fft_do_time_averaging) { - Bfield_avg_cp[lev][i]->setVal(B_external_grid[i]); - } + Bfield_aux[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + Bfield_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + if (fft_do_time_averaging) + Bfield_avg_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); } } + // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. // The default maxlevel_extEMfield_init value is the total number of levels in the simulation - if ( ( E_ext_grid_s == "constant") && (lev <= maxlevel_extEMfield_init) ) + const auto is_E_ext_const = + m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::constant || + m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::default_zero; + if ( is_E_ext_const && (lev <= maxlevel_extEMfield_init) ) { - Efield_fp[lev][i]->setVal(E_external_grid[i]); - if (fft_do_time_averaging) { - Efield_avg_fp[lev][i]->setVal(E_external_grid[i]); + Efield_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + if (fft_do_time_averaging) + Efield_avg_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + + if (lev > 0) { + Efield_aux[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + Efield_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + if (fft_do_time_averaging) + Efield_avg_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); } - - if (lev > 0) { - Efield_aux[lev][i]->setVal(E_external_grid[i]); - Efield_cp[lev][i]->setVal(E_external_grid[i]); - if (fft_do_time_averaging) { - Efield_avg_cp[lev][i]->setVal(E_external_grid[i]); - } - } } } @@ -809,72 +785,47 @@ WarpX::InitLevelData (int lev, Real /*time*/) // provided in the input file. // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. // The default maxlevel_extEMfield_init value is the total number of levels in the simulation - if (B_ext_grid_s == "parse_b_ext_grid_function" && (lev <= maxlevel_extEMfield_init)) { - - //! Strings storing parser function to initialize the components of the magnetic field on the grid - std::string str_Bx_ext_grid_function; - std::string str_By_ext_grid_function; - std::string str_Bz_ext_grid_function; + if ((m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) + && (lev <= maxlevel_extEMfield_init)) { + + // Initialize Bfield_fp with external function + InitializeExternalFieldsOnGridUsingParser( + Bfield_fp[lev][0].get(), + Bfield_fp[lev][1].get(), + Bfield_fp[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::fine); -#ifdef WARPX_DIM_RZ - std::stringstream warnMsg; - warnMsg << "Parser for external B (r and theta) fields does not work with RZ\n" - << "The initial Br and Bt fields are currently hardcoded to 0.\n" - << "The initial Bz field should only be a function of z.\n"; - ablastr::warn_manager::WMRecordWarning( - "Inputs", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); - str_Bx_ext_grid_function = "0"; - str_By_ext_grid_function = "0"; -#else - utils::parser::Store_parserString(pp_warpx, "Bx_external_grid_function(x,y,z)", - str_Bx_ext_grid_function); - utils::parser::Store_parserString(pp_warpx, "By_external_grid_function(x,y,z)", - str_By_ext_grid_function); -#endif - utils::parser::Store_parserString(pp_warpx, "Bz_external_grid_function(x,y,z)", - str_Bz_ext_grid_function); - - Bxfield_parser = std::make_unique( - utils::parser::makeParser(str_Bx_ext_grid_function,{"x","y","z"})); - Byfield_parser = std::make_unique( - utils::parser::makeParser(str_By_ext_grid_function,{"x","y","z"})); - Bzfield_parser = std::make_unique( - utils::parser::makeParser(str_Bz_ext_grid_function,{"x","y","z"})); - - // Initialize Bfield_fp with external function - InitializeExternalFieldsOnGridUsingParser(Bfield_fp[lev][0].get(), - Bfield_fp[lev][1].get(), - Bfield_fp[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::fine); - if (lev > 0) { - InitializeExternalFieldsOnGridUsingParser(Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::fine); - - InitializeExternalFieldsOnGridUsingParser(Bfield_cp[lev][0].get(), - Bfield_cp[lev][1].get(), - Bfield_cp[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::coarse); - } + if (lev > 0) { + InitializeExternalFieldsOnGridUsingParser( + Bfield_aux[lev][0].get(), + Bfield_aux[lev][1].get(), + Bfield_aux[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::fine); + + InitializeExternalFieldsOnGridUsingParser( + Bfield_cp[lev][0].get(), + Bfield_cp[lev][1].get(), + Bfield_cp[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::coarse); + } } // if the input string for the E-field is "parse_e_ext_grid_function", @@ -882,75 +833,55 @@ WarpX::InitLevelData (int lev, Real /*time*/) // provided in the input file. // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. // The default maxlevel_extEMfield_init value is the total number of levels in the simulation - if (E_ext_grid_s == "parse_e_ext_grid_function" && (lev <= maxlevel_extEMfield_init)) { - -#ifdef WARPX_DIM_RZ - WARPX_ABORT_WITH_MESSAGE( - "E and B parser for external fields does not work with RZ -- TO DO"); -#endif - - //! Strings storing parser function to initialize the components of the electric field on the grid - std::string str_Ex_ext_grid_function; - std::string str_Ey_ext_grid_function; - std::string str_Ez_ext_grid_function; - - utils::parser::Store_parserString(pp_warpx, "Ex_external_grid_function(x,y,z)", - str_Ex_ext_grid_function); - utils::parser::Store_parserString(pp_warpx, "Ey_external_grid_function(x,y,z)", - str_Ey_ext_grid_function); - utils::parser::Store_parserString(pp_warpx, "Ez_external_grid_function(x,y,z)", - str_Ez_ext_grid_function); - - Exfield_parser = std::make_unique( - utils::parser::makeParser(str_Ex_ext_grid_function,{"x","y","z"})); - Eyfield_parser = std::make_unique( - utils::parser::makeParser(str_Ey_ext_grid_function,{"x","y","z"})); - Ezfield_parser = std::make_unique( - utils::parser::makeParser(str_Ez_ext_grid_function,{"x","y","z"})); + if ((m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) + && (lev <= maxlevel_extEMfield_init)) { // Initialize Efield_fp with external function - InitializeExternalFieldsOnGridUsingParser(Efield_fp[lev][0].get(), - Efield_fp[lev][1].get(), - Efield_fp[lev][2].get(), - Exfield_parser->compile<3>(), - Eyfield_parser->compile<3>(), - Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::fine); + InitializeExternalFieldsOnGridUsingParser( + Efield_fp[lev][0].get(), + Efield_fp[lev][1].get(), + Efield_fp[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::fine); #ifdef AMREX_USE_EB // We initialize ECTRhofield consistently with the Efield if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - m_fdtd_solver_fp[lev]->EvolveECTRho(Efield_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); - + m_fdtd_solver_fp[lev]->EvolveECTRho( + Efield_fp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], lev); } #endif if (lev > 0) { - InitializeExternalFieldsOnGridUsingParser(Efield_aux[lev][0].get(), - Efield_aux[lev][1].get(), - Efield_aux[lev][2].get(), - Exfield_parser->compile<3>(), - Eyfield_parser->compile<3>(), - Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::fine); - - InitializeExternalFieldsOnGridUsingParser(Efield_cp[lev][0].get(), - Efield_cp[lev][1].get(), - Efield_cp[lev][2].get(), - Exfield_parser->compile<3>(), - Eyfield_parser->compile<3>(), - Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::coarse); + InitializeExternalFieldsOnGridUsingParser( + Efield_aux[lev][0].get(), + Efield_aux[lev][1].get(), + Efield_aux[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::fine); + + InitializeExternalFieldsOnGridUsingParser( + Efield_cp[lev][0].get(), + Efield_cp[lev][1].get(), + Efield_cp[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::coarse); #ifdef AMREX_USE_EB if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { // We initialize ECTRhofield consistently with the Efield @@ -963,34 +894,31 @@ WarpX::InitLevelData (int lev, Real /*time*/) } // Reading external fields from data file - if (add_external_B_field) { - std::string read_fields_from_path="./"; - pp_warpx.query("read_fields_from_path", read_fields_from_path); + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { + #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][0].get(), "B", "r"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][1].get(), "B", "t"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); #else - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][0].get(), "B", "x"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][1].get(), "B", "y"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][2].get(), "B", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); #endif } - if (add_external_E_field) { - std::string read_fields_from_path="./"; - pp_warpx.query("read_fields_from_path", read_fields_from_path); + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][0].get(), "E", "r"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][1].get(), "E", "t"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); #else - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][0].get(), "E", "x"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][1].get(), "E", "y"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][2].get(), "E", "z"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); #endif } diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index d890046dac6..5cd5be168a9 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -11,6 +11,7 @@ #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "EmbeddedBoundary/WarpXFaceInfoBox.H" +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Particles/ParticleBoundaryBuffer.H" #include "Particles/WarpXParticleContainer.H" @@ -170,10 +171,10 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi { RemakeMultiFab(Bfield_fp[lev][idim], dm, true ,lev); RemakeMultiFab(Efield_fp[lev][idim], dm, true ,lev); - if (add_external_B_field) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { RemakeMultiFab(Bfield_fp_external[lev][idim], dm, true ,lev); } - if (add_external_E_field) { + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { RemakeMultiFab(Efield_fp_external[lev][idim], dm, true ,lev); } RemakeMultiFab(current_fp[lev][idim], dm, false ,lev); diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 1a5b319577c..2ceb17be116 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -25,6 +25,7 @@ #endif // use PSATD ifdef #include #include +#include #include #include #include diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index af40c5e9841..6f2a97f5550 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -12,6 +12,7 @@ #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) # include "BoundaryConditions/PML_RZ.H" #endif +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Fluids/MultiFluidContainer.H" #include "Fluids/WarpXFluidContainer.H" @@ -218,27 +219,29 @@ WarpX::MoveWindow (const int step, bool move_j) amrex::ParserExecutor<3> Efield_parser; bool use_Bparser = false; bool use_Eparser = false; - if (B_ext_grid_s == "parse_b_ext_grid_function") { + if (m_p_ext_field_params->B_ext_grid_type == + ExternalFieldType::parse_ext_grid_function) { use_Bparser = true; - if (dim == 0) Bfield_parser = Bxfield_parser->compile<3>(); - if (dim == 1) Bfield_parser = Byfield_parser->compile<3>(); - if (dim == 2) Bfield_parser = Bzfield_parser->compile<3>(); + if (dim == 0) Bfield_parser = m_p_ext_field_params->Bxfield_parser->compile<3>(); + if (dim == 1) Bfield_parser = m_p_ext_field_params->Byfield_parser->compile<3>(); + if (dim == 2) Bfield_parser = m_p_ext_field_params->Bzfield_parser->compile<3>(); } - if (E_ext_grid_s == "parse_e_ext_grid_function") { + if (m_p_ext_field_params->E_ext_grid_type == + ExternalFieldType::parse_ext_grid_function) { use_Eparser = true; - if (dim == 0) Efield_parser = Exfield_parser->compile<3>(); - if (dim == 1) Efield_parser = Eyfield_parser->compile<3>(); - if (dim == 2) Efield_parser = Ezfield_parser->compile<3>(); + if (dim == 0) Efield_parser = m_p_ext_field_params->Exfield_parser->compile<3>(); + if (dim == 1) Efield_parser = m_p_ext_field_params->Eyfield_parser->compile<3>(); + if (dim == 2) Efield_parser = m_p_ext_field_params->Ezfield_parser->compile<3>(); } shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); if (fft_do_time_averaging) { shiftMF(*Bfield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params-> E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); @@ -260,16 +263,16 @@ WarpX::MoveWindow (const int step, bool move_j) if (lev > 0) { // coarse grid shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); if (fft_do_time_averaging) { shiftMF(*Bfield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost); diff --git a/Source/WarpX.H b/Source/WarpX.H index 85fdecda841..bf39d9275bf 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -20,6 +20,7 @@ #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties_fwd.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel_fwd.H" #include "Filter/NCIGodfreyFilter_fwd.H" +#include "Initialization/ExternalField_fwd.H" #include "Particles/ParticleBoundaryBuffer_fwd.H" #include "Particles/MultiParticleContainer_fwd.H" #include "Particles/WarpXParticleContainer_fwd.H" @@ -34,12 +35,12 @@ # include "FieldSolver/SpectralSolver/SpectralSolver_fwd.H" # endif #endif +#include "AcceleratorLattice/AcceleratorLattice.H" #include "Evolve/WarpXDtType.H" #include "FieldSolver/ElectrostaticSolver.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" #include "Filter/BilinearFilter.H" #include "Parallelization/GuardCellManager.H" -#include "AcceleratorLattice/AcceleratorLattice.H" #include "Utils/Parser/IntervalsParser.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/export.H" @@ -139,34 +140,6 @@ public: */ [[nodiscard]] std::string GetAuthors () const { return m_authors; } - //! Initial electric field on the grid - static amrex::Vector E_external_grid; - //! Initial magnetic field on the grid - static amrex::Vector B_external_grid; - - //! Initialization type for external magnetic field on the grid - static std::string B_ext_grid_s; - //! Initialization type for external electric field on the grid - static std::string E_ext_grid_s; - - //! Whether to apply the effect of an externally-defined electric field - static bool add_external_E_field; - //! Whether to apply the effect of an externally-defined magnetic field - static bool add_external_B_field; - - //! User-defined parser to initialize x-component of the magnetic field on the grid - std::unique_ptr Bxfield_parser; - //! User-defined parser to initialize y-component of the magnetic field on the grid - std::unique_ptr Byfield_parser; - //! User-defined parser to initialize z-component of the magnetic field on the grid - std::unique_ptr Bzfield_parser; - //! User-defined parser to initialize x-component of the electric field on the grid - std::unique_ptr Exfield_parser; - //! User-defined parser to initialize y-component of the electric field on the grid - std::unique_ptr Eyfield_parser; - //! User-defined parser to initialize z-component of the electric field on the grid - std::unique_ptr Ezfield_parser; - /** Maximum level up to which the externally defined electric and magnetic fields are initialized. * The default value is set to the max levels in the simulation. * if lev > maxlevel_extEMfield_init, the fields on those levels will have a default value of 0 @@ -1180,11 +1153,11 @@ protected: * then, the E and B fields at all the levels are initialized with * user-defined values for E_external_grid and B_external_grid. * If the initialization type for B-field is set to - * "parse_B_ext_grid_function", then, the parser is used to read + * "parse_ext_grid_function", then, the parser is used to read * Bx_external_grid_function(x,y,z), By_external_grid_function(x,y,z), * and Bz_external_grid_function(x,y,z). * Similarly, if the E-field initialization type is set to - * "parse_E_ext_grid_function", then, the parser is used to read + * "parse_ext_grid_function", then, the parser is used to read * Ex_external_grid_function(x,y,z), Ey_external_grid_function(x,y,z), * and Ex_external_grid_function(x,y,z). The parser for the E and B * initialization assumes that the function has three independent @@ -1559,6 +1532,9 @@ private: #endif amrex::Real v_particle_pml; + // External fields parameters + std::unique_ptr m_p_ext_field_params; + amrex::Real moving_window_x = std::numeric_limits::max(); // Plasma injection parameters diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index adc7423a157..597a561ae2c 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -29,6 +29,7 @@ #endif // use PSATD ifdef #include "FieldSolver/WarpX_FDTD.H" #include "Filter/NCIGodfreyFilter.H" +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Fluids/MultiFluidContainer.H" #include "Fluids/WarpXFluidContainer.H" @@ -85,14 +86,6 @@ using namespace amrex; -Vector WarpX::E_external_grid(3, 0.0); -Vector WarpX::B_external_grid(3, 0.0); - -std::string WarpX::B_ext_grid_s; -std::string WarpX::E_ext_grid_s; -bool WarpX::add_external_E_field = false; -bool WarpX::add_external_B_field = false; - int WarpX::do_moving_window = 0; int WarpX::start_moving_window_step = 0; int WarpX::end_moving_window_step = -1; @@ -724,20 +717,11 @@ WarpX::ReadParameters () moving_window_v *= PhysConst::c; } - pp_warpx.query("B_ext_grid_init_style", WarpX::B_ext_grid_s); - pp_warpx.query("E_ext_grid_init_style", WarpX::E_ext_grid_s); - - if (WarpX::B_ext_grid_s == "read_from_file") - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level == 0, - "External field reading is not implemented for more than one level"); - add_external_B_field = true; - } - if (WarpX::E_ext_grid_s == "read_from_file") - { + m_p_ext_field_params = std::make_unique(pp_warpx); + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file || + m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file){ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level == 0, - "External field reading is not implemented for more than one level"); - add_external_E_field = true; + "External field reading is not implemented for more than one level"); } maxlevel_extEMfield_init = maxLevel(); @@ -2223,12 +2207,12 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(current_fp[lev][2], amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[z]", 0.0_rt); // Match external field MultiFabs to fine patch - if (add_external_B_field) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[y]", 0.0_rt); AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[z]", 0.0_rt); } - if (add_external_E_field) { + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); From cfce3eee7af9cf2958d6828297077c91d2e72fd4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 16:02:00 -0800 Subject: [PATCH 033/176] [pre-commit.ci] pre-commit autoupdate (#4500) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/isort: 5.12.0 → 5.13.0](https://github.com/pycqa/isort/compare/5.12.0...5.13.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb298d39b68..21577b0604a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,7 +76,7 @@ repos: # Sorts Python imports according to PEP8 # https://www.python.org/dev/peps/pep-0008/#imports - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.0 hooks: - id: isort name: isort (python) From 8327f23a8eb24b77123b48b6b0e267410fcf9b4c Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Tue, 12 Dec 2023 01:20:54 -0600 Subject: [PATCH 034/176] Custom class for bibliography style (#4482) * bib-formatting * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Rename bib style class * Delete unused imports * Add comments to conf.py * Tweaks to .bib files * Add pybtex to Doc requirements * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/requirements.txt | 1 + Docs/source/conf.py | 28 ++++++++++++++++++++-- Docs/source/latex_theory/allbibs.bib | 36 ++++++++++++++-------------- Docs/source/refs.bib | 26 ++++++++++---------- Docs/spack.yaml | 3 ++- 5 files changed, 60 insertions(+), 34 deletions(-) diff --git a/Docs/requirements.txt b/Docs/requirements.txt index 86235035252..b43cc1329fb 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -15,6 +15,7 @@ picmistandard==0.28.0 # for development against an unreleased PICMI version, use: # picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python +pybtex pygments recommonmark # Sphinx<7.2 because we are waiting for diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 74556cde1ec..3735459bf76 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -30,6 +30,8 @@ import sys import urllib.request +import pybtex.plugin +from pybtex.style.formatting.unsrt import Style as UnsrtStyle import sphinx_rtd_theme sys.path.insert(0, os.path.join( os.path.abspath(__file__), '../Python') ) @@ -43,7 +45,8 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', +extensions = [ + 'sphinx.ext.autodoc', 'sphinx.ext.mathjax', 'sphinx.ext.napoleon', 'sphinx.ext.viewcode', @@ -58,7 +61,28 @@ # Relative path to bibliography file, bibliography style bibtex_bibfiles = ['latex_theory/allbibs.bib', 'refs.bib'] -bibtex_default_style = 'unsrt' + +# An brief introduction to custom BibTex formatting can be found in the Sphinx documentation: +# https://sphinxcontrib-bibtex.readthedocs.io/en/latest/usage.html#bibtex-custom-formatting +# +# More details can be gleaned from looking at the pybtex dist-package files. +# Some examples include the following: +# BaseStyle class in pybtex/style/formatting/__init__.py +# UnsrtStyle class in pybtex/style/formating/unsrt.py +class WarpXBibStyle(UnsrtStyle): + # We want the family name, i.e, "last" name, of an author to appear first. + default_name_style = 'lastfirst' + + def __init__(self, *args, **kwargs): + # We want the given names of an author to be abbreviated to just initials. + # Example: "Jean-Luc Vay" becomes "Vay, J.-L." + # Set 'abbreviate_names' to True before calling the superclass (BaseStyle class) initializer + kwargs['abbreviate_names'] = True + super().__init__(*args, **kwargs) + +pybtex.plugin.register_plugin('pybtex.style.formatting', 'warpxbibstyle', WarpXBibStyle) + +bibtex_default_style = 'warpxbibstyle' # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: diff --git a/Docs/source/latex_theory/allbibs.bib b/Docs/source/latex_theory/allbibs.bib index bdc84d5cc12..32ede278416 100644 --- a/Docs/source/latex_theory/allbibs.bib +++ b/Docs/source/latex_theory/allbibs.bib @@ -139,7 +139,7 @@ @article{Vay2000 year = {2000} } @article{Vayjcp2011, -author = {Vay, J L and Geddes, C G R and Cormier-Michel, E and Grote, D P}, +author = {Vay, J-L and Geddes, C G R and Cormier-Michel, E and Grote, D P}, doi = {10.1016/J.Jcp.2011.04.003}, journal = {Journal of Computational Physics}, month = {jul}, @@ -549,7 +549,7 @@ @article{Vaynim2005 } @article{BulanovSV2014, -author = {{Bulanov S V and Wilkens J J and Esirkepov T Zh and Korn G and Kraft G and Kraft S D and Molls M and Khoroshkov V S}}, +author = {Bulanov, S V and Wilkens, J J and Esirkepov, T Zh and Korn, G and Kraft, G and Kraft, S D and Molls, M and Khoroshkov, V S}, issn = {1063-7869}, journal = {Physics-Uspekhi}, number = {12}, @@ -616,7 +616,7 @@ @article{Yu2014 year = {2014} } @article{Vaypop2011, -author = {Vay, J -L. and Geddes, C G R and Esarey, E and Schroeder, C B and Leemans, W P and Cormier-Michel, E and Grote, D P}, +author = {Vay, J.-L. and Geddes, C G R and Esarey, E and Schroeder, C B and Leemans, W P and Cormier-Michel, E and Grote, D P}, doi = {10.1063/1.3663841}, issn = {1070-664X}, journal = {Physics Of Plasmas}, @@ -784,7 +784,7 @@ @article{Cohenprstab2009 } @article{Kaganovich2012, -author = {Kaganovich, Igor D. and Massidda, Scott and Startsev, Edward A. and Davidson, Ronald C. and Vay, Jean Luc and Friedman, Alex}, +author = {Kaganovich, Igor D. and Massidda, Scott and Startsev, Edward A. and Davidson, Ronald C. and Vay, Jean-Luc and Friedman, Alex}, journal = {Nuclear Instruments and Methods in Physics Research, Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, keywords = {Beam dynamics,Longitudinal compression,Voltage errors}, pages = {48--63}, @@ -793,7 +793,7 @@ @article{Kaganovich2012 year = {2012} } @article{Vincenti2016, -author = {Vincenti, H. and Lehe, R. and Sasanka, R. and Vay, J-L.}, +author = {Vincenti, H. and Lehe, R. and Sasanka, R. and Vay, J.-L.}, file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Vincenti et al. - 2016 - An efficient and portable SIMD algorithm for chargecurrent deposition in Particle-In-Cell codes.pdf:pdf}, journal = {Computer Programs in Physics}, pages = {To appear}, @@ -802,7 +802,7 @@ @article{Vincenti2016 year = {2016} } @article{Vaypop2008, -author = {Vay, J L}, +author = {Vay, J-L}, doi = {10.1063/1.2837054}, journal = {Physics Of Plasmas}, month = {may}, @@ -856,7 +856,7 @@ @article{Greenwoodjcp04 } @article{GodfreyIEEE2014, -author = {Godfrey, Brendan B. and Vay, Jean Luc and Haber, Irving}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc and Haber, Irving}, journal = {IEEE Transactions on Plasma Science}, keywords = {Accelerators,numerical stability,particle beams,particle-in-cell (PIC),relativistic effects,simulation,spectral methods}, number = {5}, @@ -885,7 +885,7 @@ @article{Wuprstab2011 year = {2011} } @article{VayAAC2010, -author = {Vay, J -. L and Geddes, C G R and Benedetti, C and Bruhwiler, D L and Cormier-Michel, E and Cowan, B M and Cary, J R and Grote, D P}, +author = {Vay, J-L and Geddes, C G R and Benedetti, C and Bruhwiler, D L and Cormier-Michel, E and Cowan, B M and Cary, J R and Grote, D P}, doi = {10.1063/1.3520322}, journal = {Aip Conference Proceedings}, keywords = {[978-0-7354-0853-1/10/{\$}30.00]}, @@ -906,7 +906,7 @@ @article{Vayprl07 } @article{VayJCP2013, -author = {Vay, Jean Luc and Haber, Irving and Godfrey, Brendan B.}, +author = {Vay, Jean-Luc and Haber, Irving and Godfrey, Brendan B.}, journal = {Journal of Computational Physics}, keywords = {Domain decomposition,Electromagnetic,FFT,Fast fourier transform,Parallel,Particle-In-Cell,Spectral}, pages = {260--268}, @@ -1099,7 +1099,7 @@ @article{Kalmykovprl09 } @article{Geddes2015, -author = {Geddes, Cameron G R and Rykovanov, Sergey and Matlis, Nicholas H. and Steinke, Sven and Vay, Jean Luc and Esarey, Eric H. and Ludewigt, Bernhard and Nakamura, Kei and Quiter, Brian J. and Schroeder, Carl B. and Toth, Csaba and Leemans, Wim P.}, +author = {Geddes, Cameron G R and Rykovanov, Sergey and Matlis, Nicholas H. and Steinke, Sven and Vay, Jean-Luc and Esarey, Eric H. and Ludewigt, Bernhard and Nakamura, Kei and Quiter, Brian J. and Schroeder, Carl B. and Toth, Csaba and Leemans, Wim P.}, journal = {Nuclear Instruments and Methods in Physics Research, Section B: Beam Interactions with Materials and Atoms}, keywords = {Active interrogation,Homeland security,Laser plasma accelerator,Monoenergetic photon source,Nonproliferation}, pages = {116--121}, @@ -1121,7 +1121,7 @@ @article{Antonsenprl1992 } @article{GodfreyCPC2015, -author = {Godfrey, Brendan B. and Vay, Jean Luc}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Computer Physics Communications}, keywords = {Numerical stability,Particle-in-cell,Pseudo-Spectral Time-Domain,Relativistic beam}, pages = {221--225}, @@ -1198,7 +1198,7 @@ @article{Vaycpc04 } @article{GodfreyJCP2014_PSATD, -author = {Godfrey, Brendan B. and Vay, Jean Luc and Haber, Irving}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc and Haber, Irving}, journal = {Journal of Computational Physics}, keywords = {Numerical stability,Particle-in-cell,Pseudo-spectral,Relativistic beam}, pages = {689--704}, @@ -1302,7 +1302,7 @@ @article{GodfreyJCP2014 } @article{Godfrey2013, -author = {Godfrey, Brendan B. and Vay, Jean Luc}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Journal of Computational Physics}, keywords = {Esirkepov algorithm,Numerical stability,Particle-in-cell,Relativistic beam}, pages = {33--46}, @@ -1544,7 +1544,7 @@ @article{Sprangleprl90 } @article{Molvikpop2007, annote = {48Th Annual Meeting Of The Division Of Plasma Physics Of The Aps, Philadelphia, Pa, Jan 30-Nov 03, 2006}, -author = {Molvik, A W and Covo, M Kireeff and Cohen, R and Friedman, A and Lund, S M and Sharp, W and Vay, J-L. and Baca, D and Bieniosek, F and Leister, C and Seidl, P}, +author = {Molvik, A W and Covo, M Kireeff and Cohen, R and Friedman, A and Lund, S M and Sharp, W and Vay, J-L and Baca, D and Bieniosek, F and Leister, C and Seidl, P}, doi = {10.1063/1.2436850}, institution = {Aps, Div Plasma Phys}, issn = {1070-664X}, @@ -1714,7 +1714,7 @@ @article{Vaydpf09 } @misc{Vay2014, -author = {Vay, Jean Luc and Godfrey, Brendan B.}, +author = {Vay, Jean-Luc and Godfrey, Brendan B.}, booktitle = {Comptes Rendus - Mecanique}, keywords = {Numerical instability,Particle-In-Cell,Plasma simulation,Special relativity}, number = {10-11}, @@ -1732,7 +1732,7 @@ @article{Lehearxiv2015 } @article{Friedman2014, -author = {Friedman, Alex and Cohen, Ronald H. and Grote, David P. and Lund, Steven M. and Sharp, William M. and Vay, Jean Luc and Haber, Irving and Kishek, Rami A.}, +author = {Friedman, Alex and Cohen, Ronald H. and Grote, David P. and Lund, Steven M. and Sharp, William M. and Vay, Jean-Luc and Haber, Irving and Kishek, Rami A.}, journal = {IEEE Transactions on Plasma Science}, keywords = {Algorithms,Maxwell,Ned Birdsall,computer,laser,numerical simulation,particle beam,particle-in-cell,plasma}, number = {5}, @@ -1878,7 +1878,7 @@ @book{Birdsalllangdon } @inproceedings{Grote2005, -author = {Grote, David P. and Friedman, Alex and Vay, Jean Luc and Haber, Irving}, +author = {Grote, David P. and Friedman, Alex and Vay, Jean-Luc and Haber, Irving}, booktitle = {AIP Conference Proceedings}, pages = {55--58}, title = {{The WARP code: Modeling high intensity ion beams}}, @@ -2172,7 +2172,7 @@ @article{Londrillo2010 year = {2010} } @article{GodfreyJCP2014_FDTD, -author = {Godfrey, Brendan B. and Vay, Jean Luc}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Journal of Computational Physics}, keywords = {Finite difference time-domain,Numerical stability,Particle-in-cell,Relativistic beam}, pages = {1--6}, diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index e12e885dd33..817d47fa3ae 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -27,7 +27,7 @@ @article{Esarey1996 @ARTICLE{Birdsall1991, author = {Birdsall, C.K.}, journal = {IEEE Transactions on Plasma Science}, - title = {Particle-in-cell charged-particle simulations, plus Monte Carlo collisions with neutral atoms, PIC-MCC}, + title = {{Particle-in-cell charged-particle simulations, plus Monte Carlo collisions with neutral atoms, PIC-MCC}}, year = {1991}, volume = {19}, number = {2}, @@ -40,7 +40,7 @@ @misc{Lim2007 issn = {0419-4217}, language = {eng}, number = {3}, -title = {The interaction of energetic charged particles with gas and boundaries in the particle simulation of plasmas}, +title = {{The interaction of energetic charged particles with gas and boundaries in the particle simulation of plasmas}}, volume = {69}, year = {2007}, url = {https://search.library.berkeley.edu/permalink/01UCS_BER/s4lks2/cdi_proquest_miscellaneous_35689087} @@ -61,7 +61,7 @@ @article{Turner2013 } @misc{winske2022hybrid, -title={Hybrid codes (massless electron fluid)}, +title={{Hybrid codes (massless electron fluid)}}, author={D. Winske and Homa Karimabadi and Ari Le and N. Omidi and Vadim Roytershteyn and Adam Stanier}, year={2022}, eprint={2204.01676}, @@ -70,7 +70,7 @@ @misc{winske2022hybrid } @incollection{NIELSON1976, -title = {Particle-Code Models in the Nonradiative Limit}, +title = {{Particle-Code Models in the Nonradiative Limit}}, editor = {JOHN KILLEEN}, series = {Methods in Computational Physics: Advances in Research and Applications}, publisher = {Elsevier}, @@ -84,7 +84,7 @@ @incollection{NIELSON1976 } @article{MUNOZ2018, -title = {A new hybrid code (CHIEF) implementing the inertial electron fluid equation without approximation}, +title = {{A new hybrid code (CHIEF) implementing the inertial electron fluid equation without approximation}}, journal = {Computer Physics Communications}, volume = {224}, pages = {245-264}, @@ -98,7 +98,7 @@ @article{MUNOZ2018 @article{Le2016, author = {Le, A. and Daughton, W. and Karimabadi, H. and Egedal, J.}, -title = "{Hybrid simulations of magnetic reconnection with kinetic ions and fluid electron pressure anisotropy}", +title = {{Hybrid simulations of magnetic reconnection with kinetic ions and fluid electron pressure anisotropy}}, journal = {Physics of Plasmas}, volume = {23}, number = {3}, @@ -112,7 +112,7 @@ @article{Le2016 } @article{Stanier2020, -title = {A cancellation problem in hybrid particle-in-cell schemes due to finite particle size}, +title = {{A cancellation problem in hybrid particle-in-cell schemes due to finite particle size}}, journal = {Journal of Computational Physics}, volume = {420}, pages = {109705}, @@ -131,14 +131,14 @@ @book{Stix1992 isbn = {978-0-88318-859-0}, lccn = {lc91033341}, publisher = {American Inst. of Physics}, - title = {Waves in {Plasmas}}, + title = {{Waves in Plasmas}}, url = {https://books.google.com/books?id=OsOWJ8iHpmMC}, year = {1992}, bdsk-url-1 = {https://books.google.com/books?id=OsOWJ8iHpmMC} } @article{Macchi2013, - title = {Ion acceleration by superintense laser-plasma interaction}, + title = {{Ion acceleration by superintense laser-plasma interaction}}, author = {Macchi, Andrea and Borghesi, Marco and Passoni, Matteo}, journal = {Rev. Mod. Phys.}, volume = {85}, @@ -154,7 +154,7 @@ @article{Macchi2013 @article{Wilks2001, author = {Wilks, S. C. and Langdon, A. B. and Cowan, T. E. and Roth, M. and Singh, M. and Hatchett, S. and Key, M. H. and Pennington, D. and MacKinnon, A. and Snavely, R. A.}, - title = "{Energetic proton generation in ultra-intense laser–solid interactions}", + title = {{Energetic proton generation in ultra-intense laser–solid interactions}}, journal = {Physics of Plasmas}, volume = {8}, number = {2}, @@ -169,7 +169,7 @@ @article{Wilks2001 } @article{Bulanov2008, - title = {Accelerating monoenergetic protons from ultrathin foils by flat-top laser pulses in the directed-Coulomb-explosion regime}, + title = {{Accelerating monoenergetic protons from ultrathin foils by flat-top laser pulses in the directed-Coulomb-explosion regime}}, author = {Bulanov, S. S. and Brantov, A. and Bychenkov, V. Yu. and Chvykov, V. and Kalinchenko, G. and Matsuoka, T. and Rousseau, P. and Reed, S. and Yanovsky, V. and Litzenberg, D. W. and Krushelnick, K. and Maksimchuk, A.}, journal = {Phys. Rev. E}, volume = {78}, @@ -185,7 +185,7 @@ @article{Bulanov2008 @article{Dromey2004, author = {Dromey, B. and Kar, S. and Zepf, M. and Foster, P.}, - title = "{The plasma mirror—A subpicosecond optical switch for ultrahigh power lasers}", + title = {{The plasma mirror—A subpicosecond optical switch for ultrahigh power lasers}}, journal = {Review of Scientific Instruments}, volume = {75}, number = {3}, @@ -200,7 +200,7 @@ @article{Dromey2004 } @article{Roedel2010, - title = {High repetition rate plasma mirror for temporal contrast enhancement of terawatt femtosecond laser pulses by three orders of magnitude}, + title = {{High repetition rate plasma mirror for temporal contrast enhancement of terawatt femtosecond laser pulses by three orders of magnitude}}, volume = {103}, ISSN = {1432-0649}, url = {http://dx.doi.org/10.1007/s00340-010-4329-7}, diff --git a/Docs/spack.yaml b/Docs/spack.yaml index 9f4cc71b184..db1add0018a 100644 --- a/Docs/spack.yaml +++ b/Docs/spack.yaml @@ -19,10 +19,11 @@ spack: - doxygen - graphviz - python - - py-sphinx - py-breathe - py-recommonmark + - py-pybtex - py-pygments + - py-sphinx - py-sphinx-copybutton - py-sphinx-design - py-sphinx-rtd-theme From c9e4d7c71689b93d1db001c2f99677fb57f5c958 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 12 Dec 2023 07:46:50 -0800 Subject: [PATCH 035/176] AMReX/pyAMReX/PICSAR: Weekly Update (#4489) * AMReX: Weekly Update * pyAMReX: Weekly Update * Python: Update breaking APIs Functions are now upstream and properties. --- .github/workflows/cuda.yml | 2 +- Python/pywarpx/fields.py | 16 ++++++++-------- Python/pywarpx/particle_containers.py | 4 ++-- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- .../Particles/PinnedMemoryParticleContainer.cpp | 5 ----- .../Python/Particles/WarpXParticleContainer.cpp | 1 - cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 10 files changed, 16 insertions(+), 22 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index b94b19141a6..0ce06cb96b9 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 23.12 && cd - + cd ../amrex && git checkout --detach edb4c25027efbbc465c88d453441dcd7115d8651 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index 988bdb7203f..dc0de9d9491 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -120,7 +120,7 @@ def shape(self): min_box = self.mf.box_array().minimal_box() shape = list(min_box.size - min_box.small_end) if self.include_ghosts: - nghosts = self.mf.n_grow_vect() + nghosts = self.mf.n_grow_vect shape = [shape[i] + 2*nghosts[i] for i in range(self.dim)] shape.append(self.mf.nComp) return tuple(shape) @@ -159,7 +159,7 @@ def mesh(self, direction): if self.include_ghosts: # The ghost cells are added to the upper and lower end of the global domain. - nghosts = self.mf.n_grow_vect() + nghosts = self.mf.n_grow_vect ilo -= nghosts[idir] ihi += nghosts[idir] @@ -199,7 +199,7 @@ def _get_indices(self, index, missing): def _get_n_ghosts(self): """Return the list of number of ghosts. This includes the component dimension.""" - nghosts = list(self._get_indices(self.mf.n_grow_vect(), 0)) + nghosts = list(self._get_indices(self.mf.n_grow_vect, 0)) # The components always has nghosts = 0 nghosts.append(0) return nghosts @@ -208,7 +208,7 @@ def _get_min_indices(self): """Returns the minimum indices, expanded to length 3""" min_box = self.mf.box_array().minimal_box() if self.include_ghosts: - min_box.grow(self.mf.n_grow_vect()) + min_box.grow(self.mf.n_grow_vect) imin = self._get_indices(min_box.small_end, 0) return imin @@ -217,7 +217,7 @@ def _get_max_indices(self): """ min_box = self.mf.box_array().minimal_box() if self.include_ghosts: - min_box.grow(self.mf.n_grow_vect()) + min_box.grow(self.mf.n_grow_vect) imax = self._get_indices(min_box.big_end, 0) return imax @@ -341,7 +341,7 @@ def _get_intersect_slice(self, mfi, starts, stops, icstart, icstop): """ box = mfi.tilebox() if self.include_ghosts: - box.grow(self.mf.n_grow_vect()) + box.grow(self.mf.n_grow_vect) ilo = self._get_indices(box.small_end, 0) ihi = self._get_indices(box.big_end, 0) @@ -409,7 +409,7 @@ def __getitem__(self, index): ixstart, ixstop = self._find_start_stop(ii[0], ixmin, ixmax+1, 0) iystart, iystop = self._find_start_stop(ii[1], iymin, iymax+1, 1) izstart, izstop = self._find_start_stop(ii[2], izmin, izmax+1, 2) - icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp(), 3) + icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp, 3) # Gather the data to be included in a list to be sent to other processes starts = [ixstart, iystart, izstart] @@ -499,7 +499,7 @@ def __setitem__(self, index, value): ixstart, ixstop = self._find_start_stop(ii[0], ixmin, ixmax+1, 0) iystart, iystop = self._find_start_stop(ii[1], iymin, iymax+1, 1) izstart, izstop = self._find_start_stop(ii[2], izmin, izmax+1, 2) - icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp(), 3) + icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp, 3) if isinstance(value, np.ndarray): # Expand the shape of the input array to match the shape of the global array diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index 84f78a029ad..273a981f4bd 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -124,7 +124,7 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, built_in_attrs += 1 # --- The number of extra attributes (including the weight) - nattr = self.particle_container.num_real_comps() - built_in_attrs + nattr = self.particle_container.num_real_comps - built_in_attrs attr = np.zeros((maxlen, nattr)) attr[:,0] = w @@ -818,7 +818,7 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) data_array = [] if comp_name == 'step_scraped': # the step scraped is always the final integer component - comp_idx = part_container.num_int_comps() - 1 + comp_idx = part_container.num_int_comps - 1 for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): soa = pti.soa() data_array.append(xp.array(soa.GetIntData(comp_idx), copy=False)) diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 38ea288eacd..b667b4999b1 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 23.12 +branch = edb4c25027efbbc465c88d453441dcd7115d8651 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index fed2fcae5a6..5d69d278602 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 23.12 +branch = edb4c25027efbbc465c88d453441dcd7115d8651 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp index d048b4d9c8f..600d56a62c9 100644 --- a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp +++ b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp @@ -15,9 +15,4 @@ void init_PinnedMemoryParticleContainer (py::module& m) PinnedMemoryParticleContainer, amrex::ParticleContainer<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> > pmpc (m, "PinnedMemoryParticleContainer"); - pmpc - .def("num_int_comps", - [](PinnedMemoryParticleContainer& pc) { return pc.NumIntComps(); } - ) - ; } diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index 80bb68cfa5b..1473a750941 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -85,7 +85,6 @@ void init_WarpXParticleContainer (py::module& m) py::arg("nattr_int"), py::arg("attr_int"), py::arg("uniqueparticles"), py::arg("id")=-1 ) - .def("num_real_comps", &WarpXParticleContainer::NumRealComps) .def("get_comp_index", [](WarpXParticleContainer& pc, std::string comp_name) { diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index d7790805608..781104bdf54 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "23.12" +set(WarpX_amrex_branch "edb4c25027efbbc465c88d453441dcd7115d8651" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 792d52327c1..73aeff1f32b 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "23.12" +set(WarpX_pyamrex_branch "2b8f2a414cbfb137b118724f25676f03ce04af7f" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index b0aa7ff909e..38415da65b4 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 23.12 && cd - +cd amrex && git checkout --detach edb4c25027efbbc465c88d453441dcd7115d8651 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From fcab4978a7d203c31d5a606ea9c14ac7de0d058a Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 12 Dec 2023 18:09:46 +0100 Subject: [PATCH 036/176] Clang-tidy: enable readability-duplicate-include check in clang-tidy CI test (#4496) * enable readability-duplicate-include check in clang-tidy CI test * remove duplicate includes --- .clang-tidy | 1 - Source/ablastr/fields/PoissonSolver.H | 26 ++++++++------------- Source/ablastr/fields/VectorPoissonSolver.H | 21 ++++++----------- 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 8ef1234b125..9fada1bbdfe 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -38,7 +38,6 @@ Checks: ' readability-*, -readability-braces-around-statements, -readability-convert-member-functions-to-static, - -readability-duplicate-include, -readability-else-after-return, -readability-function-cognitive-complexity, -readability-identifier-length, diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 953c67ab54e..06416168e88 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -12,21 +12,6 @@ #include #include -#include -#include -#include -#include -#include -#include -#if defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ) -# include -#endif -#ifdef AMREX_USE_EB -# include -#endif -#include -#include -#include #include #include @@ -45,12 +30,21 @@ #include #include #include +#include #include +#include +#include #include +#include #include #include #include -#include +#if defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ) +# include +#endif +#ifdef AMREX_USE_EB +# include +#endif #include #include diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index 75957974fdb..6f824c3be34 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -13,19 +13,6 @@ #include #include -#include -#include -#include -#include -#include -#include -#ifdef AMREX_USE_EB -# include -#endif -#include -#include -#include - #include #include #include @@ -43,12 +30,18 @@ #include #include #include +#include +#include +#include #include #include +#include #include #include #include -#include +#ifdef AMREX_USE_EB +# include +#endif #include #include From 5344f8097f3a6243e18b31cd922f23df84e81e5a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 12 Dec 2023 09:13:28 -0800 Subject: [PATCH 037/176] Use more consistent species types in fusion module (#4480) * Force user to select helium4 in the p-B fusion reaction * Update helium4 mass in fusion module and tests * Replace proton with hydrogen1 * Update hydrogen1 mass in fusion module and tests * Update benchmarks --- .../analysis_proton_boron_fusion.py | 10 +- .../nuclear_fusion/inputs_proton_boron_2d | 33 +-- .../nuclear_fusion/inputs_proton_boron_3d | 33 +-- .../Proton_Boron_Fusion_2D.json | 156 +++++------ .../Proton_Boron_Fusion_3D.json | 258 +++++++++--------- .../BinaryCollision/BinaryCollisionUtils.cpp | 20 +- .../ProtonBoronFusionCrossSection.H | 5 +- .../ProtonBoronFusionInitializeMomentum.H | 2 +- 8 files changed, 259 insertions(+), 258 deletions(-) diff --git a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py index eef371589f5..eec2ba4fffb 100755 --- a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py +++ b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -65,11 +65,11 @@ keV_to_Joule = scc.e*1e3 MeV_to_Joule = scc.e*1e6 barn_to_square_meter = 1.e-28 -m_p = scc.m_p # Proton mass +m_p = 1.00782503223*scc.m_u # Proton mass m_b = 11.00930536*scc.m_u # Boron 11 mass m_reduced = m_p*m_b/(m_p+m_b) -m_a = 4.002602*scc.m_u # Alpha mass -m_be = 7.94748*m_p # Beryllium 8 mass +m_a = 4.00260325413*scc.m_u # Alpha mass +m_be = 7.94748*scc.m_p # Beryllium 8 mass Z_boron = 5. Z_proton = 1. E_Gamow = (Z_boron*Z_proton*np.pi*scc.fine_structure)**2*2.*m_reduced*scc.c**2 @@ -395,7 +395,7 @@ def p_sq_boron_frame_to_E_COM_frame(p_proton_sq): # Use invariant E**2 - p**2c**2 of 4-momentum norm to compute energy in center of mass frame E_com = np.sqrt(E_lab**2 - p_proton_sq*scc.c**2) # Corresponding kinetic energy - E_com_kin = E_com - (m_b+scc.m_p)*scc.c**2 + E_com_kin = E_com - (m_b+m_p)*scc.c**2 return E_com_kin*(p_proton_sq>0.) def p_sq_to_kinetic_energy(p_sq, m): @@ -525,7 +525,7 @@ def check_initial_energy2(data): ## Proton kinetic energy in the lab frame before fusion E_proton_nonrelativistic = Energy_step*slice_number**2 ## Corresponding square norm of proton momentum - p_proton_sq = 2.*scc.m_p*E_proton_nonrelativistic + p_proton_sq = 2.*m_p*E_proton_nonrelativistic ## Kinetic energy in the lab frame after ## proton + boron 11 -> alpha + beryllium 8 E_after_fusion = E_proton_nonrelativistic + E_fusion diff --git a/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d b/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d index 36414552c11..30988b1ca2f 100644 --- a/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d +++ b/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d @@ -33,11 +33,12 @@ particles.species_names = proton1 boron1 alpha1 proton2 boron2 alpha2 proton3 bo proton4 boron4 alpha4 proton5 boron5 alpha5 my_constants.m_b11 = 11.00930536*m_u # Boron 11 mass -my_constants.m_reduced = m_p*m_b11/(m_p+m_b11) +my_constants.m_h = 1.00782503223*m_u # Hydrogen 1 mass +my_constants.m_reduced = m_h*m_b11/(m_h+m_b11) my_constants.keV_to_J = 1.e3*q_e my_constants.Energy_step = 22. * keV_to_J -proton1.species_type = proton +proton1.species_type = hydrogen1 proton1.injection_style = "NRandomPerCell" proton1.num_particles_per_cell = 80000 proton1.profile = constant @@ -46,7 +47,7 @@ proton1.momentum_distribution_type = "parse_momentum_function" proton1.momentum_function_ux(x,y,z) = 0. proton1.momentum_function_uy(x,y,z) = 0. ## Thanks to the floor, all particles in the same cell have the exact same momentum -proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_p*clight) +proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_h*clight) proton1.do_not_push = 1 proton1.do_not_deposit = 1 @@ -63,14 +64,14 @@ boron1.momentum_function_uz(x,y,z) = -sqrt(2*m_reduced*Energy_step*(floor(z)**2) boron1.do_not_push = 1 boron1.do_not_deposit = 1 -alpha1.species_type = helium +alpha1.species_type = helium4 alpha1.do_not_push = 1 alpha1.do_not_deposit = 1 my_constants.background_dens = 1.e26 my_constants.beam_dens = 1.e20 -proton2.species_type = proton +proton2.species_type = hydrogen1 proton2.injection_style = "NRandomPerCell" proton2.num_particles_per_cell = 8000 proton2.profile = "parse_density_function" @@ -81,7 +82,7 @@ proton2.momentum_distribution_type = "parse_momentum_function" proton2.momentum_function_ux(x,y,z) = 0. proton2.momentum_function_uy(x,y,z) = 0. proton2.momentum_function_uz(x,y,z) = "if(x - floor(x) < 0.1, - 0., sqrt(2*m_p*Energy_step*(floor(z)**2))/(m_p*clight))" + 0., sqrt(2*m_h*Energy_step*(floor(z)**2))/(m_h*clight))" proton2.do_not_push = 1 proton2.do_not_deposit = 1 @@ -94,19 +95,19 @@ boron2.momentum_distribution_type = "constant" boron2.do_not_push = 1 boron2.do_not_deposit = 1 -alpha2.species_type = helium +alpha2.species_type = helium4 alpha2.do_not_push = 1 alpha2.do_not_deposit = 1 my_constants.temperature = 44. * keV_to_J -proton3.species_type = proton +proton3.species_type = hydrogen1 proton3.injection_style = "NRandomPerCell" proton3.num_particles_per_cell = 4800 proton3.profile = constant proton3.density = 1.e28 proton3.momentum_distribution_type = "maxwell_boltzmann" -proton3.theta = temperature/(m_p*clight**2) +proton3.theta = temperature/(m_h*clight**2) proton3.do_not_push = 1 proton3.do_not_deposit = 1 @@ -120,19 +121,19 @@ boron3.theta = temperature/(m_b11*clight**2) boron3.do_not_push = 1 boron3.do_not_deposit = 1 -alpha3.species_type = helium +alpha3.species_type = helium4 alpha3.do_not_push = 1 alpha3.do_not_deposit = 1 my_constants.proton4_energy = 550*keV_to_J -proton4.species_type = proton +proton4.species_type = hydrogen1 proton4.injection_style = "NRandomPerCell" proton4.num_particles_per_cell = 800 proton4.profile = "constant" proton4.density = 1.e35 proton4.momentum_distribution_type = "constant" -proton4.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton4.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton4.do_not_push = 1 proton4.do_not_deposit = 1 @@ -145,17 +146,17 @@ boron4.momentum_distribution_type = "constant" boron4.do_not_push = 1 boron4.do_not_deposit = 1 -alpha4.species_type = helium +alpha4.species_type = helium4 alpha4.do_not_push = 1 alpha4.do_not_deposit = 1 -proton5.species_type = proton +proton5.species_type = hydrogen1 proton5.injection_style = "NRandomPerCell" proton5.num_particles_per_cell = 800 proton5.profile = "constant" proton5.density = 1.e35 proton5.momentum_distribution_type = "constant" -proton5.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton5.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton5.do_not_push = 1 proton5.do_not_deposit = 1 @@ -168,7 +169,7 @@ boron5.momentum_distribution_type = "constant" boron5.do_not_push = 1 boron5.do_not_deposit = 1 -alpha5.species_type = helium +alpha5.species_type = helium4 alpha5.do_not_push = 1 alpha5.do_not_deposit = 1 diff --git a/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d b/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d index 46744013986..421e39d2765 100644 --- a/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d +++ b/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d @@ -33,11 +33,12 @@ particles.species_names = proton1 boron1 alpha1 proton2 boron2 alpha2 proton3 bo proton4 boron4 alpha4 proton5 boron5 alpha5 my_constants.m_b11 = 11.00930536*m_u # Boron 11 mass -my_constants.m_reduced = m_p*m_b11/(m_p+m_b11) +my_constants.m_h = 1.00782503223*m_u # Hydrogen 1 mass +my_constants.m_reduced = m_h*m_b11/(m_h+m_b11) my_constants.keV_to_J = 1.e3*q_e my_constants.Energy_step = 22. * keV_to_J -proton1.species_type = proton +proton1.species_type = hydrogen1 proton1.injection_style = "NRandomPerCell" proton1.num_particles_per_cell = 10000 proton1.profile = constant @@ -46,7 +47,7 @@ proton1.momentum_distribution_type = "parse_momentum_function" proton1.momentum_function_ux(x,y,z) = 0. proton1.momentum_function_uy(x,y,z) = 0. ## Thanks to the floor, all particles in the same cell have the exact same momentum -proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_p*clight) +proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_h*clight) proton1.do_not_push = 1 proton1.do_not_deposit = 1 @@ -63,14 +64,14 @@ boron1.momentum_function_uz(x,y,z) = -sqrt(2*m_reduced*Energy_step*(floor(z)**2) boron1.do_not_push = 1 boron1.do_not_deposit = 1 -alpha1.species_type = helium +alpha1.species_type = helium4 alpha1.do_not_push = 1 alpha1.do_not_deposit = 1 my_constants.background_dens = 1.e26 my_constants.beam_dens = 1.e20 -proton2.species_type = proton +proton2.species_type = hydrogen1 proton2.injection_style = "NRandomPerCell" proton2.num_particles_per_cell = 1000 proton2.profile = "parse_density_function" @@ -81,7 +82,7 @@ proton2.momentum_distribution_type = "parse_momentum_function" proton2.momentum_function_ux(x,y,z) = 0. proton2.momentum_function_uy(x,y,z) = 0. proton2.momentum_function_uz(x,y,z) = "if(y - floor(y) < 0.1, - 0., sqrt(2*m_p*Energy_step*(floor(z)**2))/(m_p*clight))" + 0., sqrt(2*m_h*Energy_step*(floor(z)**2))/(m_h*clight))" proton2.do_not_push = 1 proton2.do_not_deposit = 1 @@ -94,19 +95,19 @@ boron2.momentum_distribution_type = "constant" boron2.do_not_push = 1 boron2.do_not_deposit = 1 -alpha2.species_type = helium +alpha2.species_type = helium4 alpha2.do_not_push = 1 alpha2.do_not_deposit = 1 my_constants.temperature = 44. * keV_to_J -proton3.species_type = proton +proton3.species_type = hydrogen1 proton3.injection_style = "NRandomPerCell" proton3.num_particles_per_cell = 600 proton3.profile = constant proton3.density = 1.e28 proton3.momentum_distribution_type = "maxwell_boltzmann" -proton3.theta = temperature/(m_p*clight**2) +proton3.theta = temperature/(m_h*clight**2) proton3.do_not_push = 1 proton3.do_not_deposit = 1 @@ -120,19 +121,19 @@ boron3.theta = temperature/(m_b11*clight**2) boron3.do_not_push = 1 boron3.do_not_deposit = 1 -alpha3.species_type = helium +alpha3.species_type = helium4 alpha3.do_not_push = 1 alpha3.do_not_deposit = 1 my_constants.proton4_energy = 550*keV_to_J -proton4.species_type = proton +proton4.species_type = hydrogen1 proton4.injection_style = "NRandomPerCell" proton4.num_particles_per_cell = 100 proton4.profile = "constant" proton4.density = 1.e35 proton4.momentum_distribution_type = "constant" -proton4.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton4.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton4.do_not_push = 1 proton4.do_not_deposit = 1 @@ -145,17 +146,17 @@ boron4.momentum_distribution_type = "constant" boron4.do_not_push = 1 boron4.do_not_deposit = 1 -alpha4.species_type = helium +alpha4.species_type = helium4 alpha4.do_not_push = 1 alpha4.do_not_deposit = 1 -proton5.species_type = proton +proton5.species_type = hydrogen1 proton5.injection_style = "NRandomPerCell" proton5.num_particles_per_cell = 100 proton5.profile = "constant" proton5.density = 1.e35 proton5.momentum_distribution_type = "constant" -proton5.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton5.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton5.do_not_push = 1 proton5.do_not_deposit = 1 @@ -168,7 +169,7 @@ boron5.momentum_distribution_type = "constant" boron5.do_not_push = 1 boron5.do_not_deposit = 1 -alpha5.species_type = helium +alpha5.species_type = helium4 alpha5.do_not_push = 1 alpha5.do_not_deposit = 1 diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json index 979cffc0aa7..7f359f76e94 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json @@ -1,117 +1,117 @@ { - "alpha1": { - "particle_momentum_x": 4.714227948839551e-15, - "particle_momentum_y": 4.676078701959904e-15, - "particle_momentum_z": 4.676142266462326e-15, - "particle_position_x": 463834.9497057527, - "particle_position_y": 977592.6452214213, - "particle_weight": 2.940000093807323e-28 + "lev=0": { + "rho": 0.0 }, "alpha2": { - "particle_momentum_x": 4.081538674143691e-15, - "particle_momentum_y": 4.122349693618236e-15, - "particle_momentum_z": 4.201699189448955e-15, + "particle_momentum_x": 4.0815251136381854e-15, + "particle_momentum_y": 4.122335955939925e-15, + "particle_momentum_z": 4.201726051973563e-15, "particle_position_x": 410664.0477768091, "particle_position_y": 868193.1483606595, - "particle_weight": 2.959852107088932e+18 + "particle_weight": 2.947401325355292e+18 }, "alpha3": { - "particle_momentum_x": 4.934249084589244e-16, - "particle_momentum_y": 4.791039564460439e-16, - "particle_momentum_z": 4.698462130524534e-16, - "particle_position_x": 52309.63968324501, - "particle_position_y": 104322.0547863817, - "particle_weight": 1.491736282762796e+27 - }, - "alpha4": { - "particle_momentum_x": 2.342240913445794e-14, - "particle_momentum_y": 2.34177318709557e-14, - "particle_momentum_z": 2.353174771675334e-14, - "particle_position_x": 2457367.458278153, - "particle_position_y": 4915112.044373058, - "particle_weight": 384.0000000000002 - }, - "alpha5": { - "particle_momentum_x": 2.330088308142745e-14, - "particle_momentum_y": 2.344976927616691e-14, - "particle_momentum_z": 2.361827647381584e-14, - "particle_position_x": 2457556.857163842, - "particle_position_y": 4914659.635379327, - "particle_weight": 3.839999999999998e-19 + "particle_momentum_x": 4.925995964082186e-16, + "particle_momentum_y": 4.780894181553476e-16, + "particle_momentum_z": 4.688680302272935e-16, + "particle_position_x": 52240.870920562535, + "particle_position_y": 103873.30862578771, + "particle_weight": 1.4784234459335885e+27 }, - "boron1": { + "proton1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 81921136.14476715, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983792, + "particle_position_y": 81919772.69310114, "particle_weight": 128.00000000000261 }, - "boron2": { + "alpha4": { + "particle_momentum_x": 2.3422553081132076e-14, + "particle_momentum_y": 2.3416860254545166e-14, + "particle_momentum_z": 2.3532380279126124e-14, + "particle_position_x": 2457367.4582781536, + "particle_position_y": 4915112.044373058, + "particle_weight": 384.0000000000002 + }, + "proton4": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.015821768, - "particle_position_y": 819270.9858143466, - "particle_weight": 1.2799999999016446e+28 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 819198.7077771134, + "particle_weight": 1.2800000000000004e+37 }, "boron3": { "particle_momentum_x": 9.277692671587846e-15, "particle_momentum_y": 9.268409636965691e-15, "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.166422465, - "particle_position_y": 8192499.706038672, - "particle_weight": 6.399502754572407e+30 + "particle_position_x": 4096178.1664224654, + "particle_position_y": 8192499.7060386725, + "particle_weight": 6.399507192184686e+30 }, - "boron5": { + "boron2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_position_x": 409547.3312927569, - "particle_position_y": 819118.5558814355, - "particle_weight": 127.99999999999999 - }, - "lev=0": { - "rho": 0.0 + "particle_position_x": 409798.015821768, + "particle_position_y": 819270.9858143466, + "particle_weight": 1.2799999999017534e+28 }, - "proton1": { + "boron1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40960140.72983792, - "particle_position_y": 81919772.69310114, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 81921136.14476715, "particle_weight": 128.00000000000261 }, - "proton2": { + "proton3": { + "particle_momentum_x": 1.684673285246867e-15, + "particle_momentum_y": 1.6827557106531144e-15, + "particle_momentum_z": 1.6802642612723895e-15, + "particle_position_x": 2457259.537951346, + "particle_position_y": 4914248.771185745, + "particle_weight": 1.2795071921846893e+30 + }, + "proton5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.371679089706304e-14, - "particle_position_x": 4095630.6981353555, - "particle_position_y": 8192073.551798361, - "particle_weight": 1.2885512789516445e+28 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.2877618571, + "particle_position_y": 819101.3225783394, + "particle_weight": 1.2800000000000004e+37 }, - "proton3": { - "particle_momentum_x": 1.684214433067055e-15, - "particle_momentum_y": 1.682290595962152e-15, - "particle_momentum_z": 1.679806121802876e-15, - "particle_position_x": 2457258.407562205, - "particle_position_y": 4914234.891662369, - "particle_weight": 1.279502754572413e+30 + "alpha1": { + "particle_momentum_x": 4.714228774936194e-15, + "particle_momentum_y": 4.676079523757908e-15, + "particle_momentum_z": 4.676143086393951e-15, + "particle_position_x": 463834.9497057527, + "particle_position_y": 977592.6452214213, + "particle_weight": 2.9280275175431764e-28 }, - "proton4": { + "alpha5": { + "particle_momentum_x": 2.3300446729308862e-14, + "particle_momentum_y": 2.3449941485701153e-14, + "particle_momentum_z": 2.3618372938672865e-14, + "particle_position_x": 2457556.8571638414, + "particle_position_y": 4914659.635379325, + "particle_weight": 3.839999999999998e-19 + }, + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 819198.7077771134, - "particle_weight": 1.2800000000000004e+37 + "particle_momentum_z": 2.3723248133690294e-14, + "particle_position_x": 4095630.6981353555, + "particle_position_y": 8192073.551798361, + "particle_weight": 1.2885512789517532e+28 }, - "proton5": { + "boron5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409638.2877618571, - "particle_position_y": 819101.3225783394, - "particle_weight": 1.2800000000000004e+37 + "particle_momentum_z": 0.0, + "particle_position_x": 409547.3312927569, + "particle_position_y": 819118.5558814355, + "particle_weight": 127.99999999999999 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json index b510fb10ff9..c3517c32b5b 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json @@ -1,131 +1,131 @@ { - "alpha1": { - "particle_momentum_x": 4.659631949650719e-15, - "particle_momentum_y": 4.617308377231923e-15, - "particle_momentum_z": 4.617389330519975e-15, - "particle_position_x": 453219.3436004627, - "particle_position_y": 457393.42114332493, - "particle_position_z": 970481.0580153911, - "particle_weight": 1.9043802455895325e-27 - }, - "alpha2": { - "particle_momentum_x": 4.071678402460901e-15, - "particle_momentum_y": 4.1196410276880726e-15, - "particle_momentum_z": 4.181784710058622e-15, - "particle_position_x": 405848.1200755021, - "particle_position_y": 408011.53191288817, - "particle_position_z": 866330.5923068221, - "particle_weight": 1.9261737121655108e+19 - }, - "alpha3": { - "particle_momentum_x": 4.853582081563116e-16, - "particle_momentum_y": 4.797297548379446e-16, - "particle_momentum_z": 4.759501504671248e-16, - "particle_position_x": 51342.57223394147, - "particle_position_y": 50913.6612890173, - "particle_position_z": 101306.34059008301, - "particle_weight": 1.0360147595748265e+28 - }, - "alpha4": { - "particle_momentum_x": 2.3373908106523292e-14, - "particle_momentum_y": 2.3439419725501755e-14, - "particle_momentum_z": 2.3559411845892612e-14, - "particle_position_x": 2457367.4582781526, - "particle_position_y": 2457512.044373058, - "particle_position_z": 4914475.776513073, - "particle_weight": 3072.000000000002 - }, - "alpha5": { - "particle_momentum_x": 2.333624251961068e-14, - "particle_momentum_y": 2.3455939695798916e-14, - "particle_momentum_z": 2.3574526216382372e-14, - "particle_position_x": 2457556.857163842, - "particle_position_y": 2457059.6353793237, - "particle_position_z": 4915847.043341331, - "particle_weight": 3.0719999999999984e-18 - }, - "boron1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 - }, - "boron2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.0158217681, - "particle_position_y": 409670.9858143465, - "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999999357942e+29 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 4096499.7060386725, - "particle_position_z": 8191465.586938233, - "particle_weight": 5.119654661746806e+31 - }, - "boron5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.33129275695, - "particle_position_y": 409518.5558814353, - "particle_position_z": 819306.5006950963, - "particle_weight": 1023.9999999999999 - }, - "lev=0": { - "rho": 0.0 - }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 - }, - "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3738870507189214e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.0227810240713489e+29 - }, - "proton3": { - "particle_momentum_x": 1.6843510515198723e-15, - "particle_momentum_y": 1.6823818968683368e-15, - "particle_momentum_z": 1.6799961456539653e-15, - "particle_position_x": 2457338.899376694, - "particle_position_y": 2457069.647393952, - "particle_position_z": 4914642.288898885, - "particle_weight": 1.0236546617468092e+31 - }, - "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 409598.7077771135, - "particle_position_z": 818958.0399127571, - "particle_weight": 1.0240000000000003e+38 - }, - "proton5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409638.28776185703, - "particle_position_y": 409501.32257833943, - "particle_position_z": 819309.1804186807, - "particle_weight": 1.0240000000000003e+38 - } + "lev=0": { + "rho": 0.0 + }, + "proton3": { + "particle_momentum_x": 1.684809640261926e-15, + "particle_momentum_y": 1.6828399494797829e-15, + "particle_momentum_z": 1.6804535487104095e-15, + "particle_position_x": 2457338.899376694, + "particle_position_y": 2457069.647393952, + "particle_position_z": 4914642.288898885, + "particle_weight": 1.0236580897818594e+31 + }, + "alpha5": { + "particle_momentum_x": 2.3336421176154494e-14, + "particle_momentum_y": 2.345583787205298e-14, + "particle_momentum_z": 2.3574859187827772e-14, + "particle_position_x": 2457556.857163842, + "particle_position_y": 2457059.6353793233, + "particle_position_z": 4915847.043341331, + "particle_weight": 3.0719999999999984e-18 + }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 4096499.7060386725, + "particle_position_z": 8191465.586938233, + "particle_weight": 5.119658089781855e+31 + }, + "boron1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 + }, + "alpha4": { + "particle_momentum_x": 2.3374367002622734e-14, + "particle_momentum_y": 2.3439200330576348e-14, + "particle_momentum_z": 2.3559200621107925e-14, + "particle_position_x": 2457367.4582781536, + "particle_position_y": 2457512.044373057, + "particle_position_z": 4914475.776513074, + "particle_weight": 3072.000000000002 + }, + "alpha3": { + "particle_momentum_x": 4.857656981301579e-16, + "particle_momentum_y": 4.811177053359803e-16, + "particle_momentum_z": 4.766749863500389e-16, + "particle_position_x": 51282.9136300544, + "particle_position_y": 51045.39533309579, + "particle_position_z": 101578.82008857065, + "particle_weight": 1.0257306544231767e+28 + }, + "alpha2": { + "particle_momentum_x": 4.071664828713061e-15, + "particle_momentum_y": 4.119627260272026e-15, + "particle_momentum_z": 4.181812341443065e-15, + "particle_position_x": 405848.1200755021, + "particle_position_y": 408011.53191288817, + "particle_position_z": 866330.5923068221, + "particle_weight": 1.9180757241165136e+19 + }, + "proton2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.3745333755307162e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503005, + "particle_weight": 1.0227810240716198e+29 + }, + "proton4": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 409598.7077771135, + "particle_position_z": 818958.0399127571, + "particle_weight": 1.0240000000000003e+38 + }, + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, + "particle_weight": 1024.000000000021 + }, + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.28776185703, + "particle_position_y": 409501.32257833943, + "particle_position_z": 819309.1804186807, + "particle_weight": 1.0240000000000003e+38 + }, + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.33129275695, + "particle_position_y": 409518.5558814353, + "particle_position_z": 819306.5006950963, + "particle_weight": 1023.9999999999999 + }, + "alpha1": { + "particle_momentum_x": 4.659632773478301e-15, + "particle_momentum_y": 4.617309192789389e-15, + "particle_momentum_z": 4.6173901462667306e-15, + "particle_position_x": 453219.3436004627, + "particle_position_y": 457393.42114332493, + "particle_position_z": 970481.0580153911, + "particle_weight": 1.8966442168729786e-27 + }, + "boron2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409798.0158217681, + "particle_position_y": 409670.9858143465, + "particle_position_z": 819255.8152412223, + "particle_weight": 1.023999999936064e+29 + } } \ No newline at end of file diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp index 8eddad7c496..82f263e9daf 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp @@ -57,13 +57,11 @@ namespace BinaryCollisionUtils{ ||(product_species1.AmIA() && product_species2.AmIA())){ return NuclearFusionType::DeuteriumDeuteriumToNeutronHelium; } else if ( - (product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA()) + (product_species1.AmIA() && product_species2.AmIA()) ||(product_species1.AmIA() && product_species2.AmIA())){ return NuclearFusionType::DeuteriumDeuteriumToProtonTritium; } else { - WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or tritium and proton"); + WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or hydrogen3 and hydrogen1"); } } else if ((species1.AmIA() && species2.AmIA()) @@ -77,15 +75,15 @@ namespace BinaryCollisionUtils{ auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (product_species1.AmIA() && product_species2.AmIA()) + (product_species1.AmIA() && product_species2.AmIA()) || - (product_species1.AmIA() && product_species2.AmIA()), - "ERROR: Product species of deuterium-helium fusion must be of type proton and helium4"); + (product_species1.AmIA() && product_species2.AmIA()), + "ERROR: Product species of deuterium-helium fusion must be of type hydrogen1 and helium4"); return NuclearFusionType::DeuteriumHeliumToProtonHelium; } - else if ((species1.AmIA() && species2.AmIA()) + else if ((species1.AmIA() && species2.AmIA()) || - (species1.AmIA() && species2.AmIA()) + (species1.AmIA() && species2.AmIA()) ) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -93,8 +91,8 @@ namespace BinaryCollisionUtils{ "ERROR: Proton-boron must contain exactly one product species"); auto& product_species = mypc->GetParticleContainerFromName(product_species_name[0]); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species.AmIA(), - "ERROR: Product species of proton-boron fusion must be of type alpha"); + product_species.AmIA(), + "ERROR: Product species of proton-boron fusion must be of type helium4"); return NuclearFusionType::ProtonBoronToAlphas; } WARPX_ABORT_WITH_MESSAGE("Binary nuclear fusion not implemented between species " + diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H index 4a5d518b750..ec9a1989189 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H @@ -42,8 +42,9 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe // Compute Gamow factor, in MeV constexpr auto one_pr = 1._prt; constexpr auto Z_boron = 5._prt; - constexpr amrex::ParticleReal m_boron = 10.7319_prt * PhysConst::m_p; - constexpr amrex::ParticleReal m_reduced = m_boron / (one_pr + m_boron/PhysConst::m_p); + constexpr amrex::ParticleReal m_boron = 11.00930536_prt * PhysConst::m_u; + constexpr amrex::ParticleReal m_hydrogen = 1.00782503223 * PhysConst::m_u; + constexpr amrex::ParticleReal m_reduced = m_boron / (one_pr + m_boron/m_hydrogen); constexpr amrex::ParticleReal gamow_factor = m_reduced / 2._prt * (PhysConst::q_e*PhysConst::q_e * Z_boron / (2._prt*PhysConst::ep0*PhysConst::hbar)) * diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 0e92bda5beb..7b29267ec32 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -81,7 +81,7 @@ namespace { // which can cause compilation to fail or generate a warning, so we're explicitly setting // them as double. Note that nuclear fusion module does not currently work with single // precision anyways. - constexpr double m_alpha = PhysConst::m_u * 4.002602_prt; + constexpr double m_alpha = PhysConst::m_u * 4.00260325413_prt; constexpr double m_beryllium = PhysConst::m_p * 7.94748_prt; constexpr double mBe_sq = m_beryllium*m_beryllium; constexpr amrex::ParticleReal c_sq = PhysConst::c * PhysConst::c; From a1f1d6727da478d009a4c641eeced1e377f61645 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 12 Dec 2023 10:51:48 -0800 Subject: [PATCH 038/176] AMReX/pyAMReX/PICSAR: Weekly Update (#4503) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 0ce06cb96b9..ff50edc428e 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach edb4c25027efbbc465c88d453441dcd7115d8651 && cd - + cd ../amrex && git checkout --detach ecaa46d0be4b5c79b8806e48e3469000d8bb7252 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index b667b4999b1..ceba435d251 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = edb4c25027efbbc465c88d453441dcd7115d8651 +branch = ecaa46d0be4b5c79b8806e48e3469000d8bb7252 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 5d69d278602..d4f2e68d0d3 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = edb4c25027efbbc465c88d453441dcd7115d8651 +branch = ecaa46d0be4b5c79b8806e48e3469000d8bb7252 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 781104bdf54..2b044f3a485 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "edb4c25027efbbc465c88d453441dcd7115d8651" +set(WarpX_amrex_branch "ecaa46d0be4b5c79b8806e48e3469000d8bb7252" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 73aeff1f32b..57e378c26a0 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "2b8f2a414cbfb137b118724f25676f03ce04af7f" +set(WarpX_pyamrex_branch "0b2d3f6b160991834534164b7391080fabc48ddb" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 38415da65b4..8da087155ba 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach edb4c25027efbbc465c88d453441dcd7115d8651 && cd - +cd amrex && git checkout --detach ecaa46d0be4b5c79b8806e48e3469000d8bb7252 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From b1cec6ec796631f0efef9ebbaeafa3ef3bc9a0e9 Mon Sep 17 00:00:00 2001 From: Ryan Sandberg Date: Tue, 12 Dec 2023 18:10:44 -0800 Subject: [PATCH 039/176] workflow training neural network from warpx data (#4499) * workflow training neural network from warpx data * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * address automated review comments * Apply suggestions from code review Co-authored-by: Axel Huebl * move data and figures to cloud (GitHub and Zenodo) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Change to title case * add ml references to bibliography * Formatting * Test other multi-ref * Refs * One more ref * Finalize Ending --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- Docs/source/refs.bib | 31 ++ Docs/source/usage/workflows.rst | 1 + .../usage/workflows/ml_dataset_training.rst | 256 +++++++++++++ .../workflows/ml_materials/create_dataset.py | 208 ++++++++++ .../ml_materials/neural_network_classes.py | 91 +++++ .../ml_materials/run_warpx_training.py | 354 ++++++++++++++++++ .../usage/workflows/ml_materials/train.py | 166 ++++++++ .../usage/workflows/ml_materials/visualize.py | 158 ++++++++ 8 files changed, 1265 insertions(+) create mode 100644 Docs/source/usage/workflows/ml_dataset_training.rst create mode 100644 Docs/source/usage/workflows/ml_materials/create_dataset.py create mode 100644 Docs/source/usage/workflows/ml_materials/neural_network_classes.py create mode 100644 Docs/source/usage/workflows/ml_materials/run_warpx_training.py create mode 100644 Docs/source/usage/workflows/ml_materials/train.py create mode 100644 Docs/source/usage/workflows/ml_materials/visualize.py diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 817d47fa3ae..0abca79a5d4 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -213,3 +213,34 @@ @article{Roedel2010 month = nov, pages = {295–302} } + +@misc{SandbergPASC24, + author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, + title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, + booktitle = {Proc. of PASC24}, + venue = {Zuerich, Switzerland}, + address = {Zuerich, Switzerland}, + series = {PASC'24 - Platform for Advanced Scientific Computing}, + year = {2024}, + note = {submitted} +} + +@inproceedings{SandbergIPAC23, + author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, + title = {{Hybrid beamline element ML-training for surrogates in the ImpactX beam-dynamics code}}, + booktitle = {Proc. 14th International Particle Accelerator Conference}, + pages = {2885-2888}, + paper = {WEPA101}, + venue = {Venice, Italy}, + address = {Venice, Italy}, + series = {IPAC'23 - 14th International Particle Accelerator Conference}, + number = {14}, + publisher = {JACoW Publishing, Geneva, Switzerland}, + month = {05}, + year = {2023}, + issn = {2673-5490}, + isbn = {978-3-95450-231-8}, + doi = {10.18429/JACoW-IPAC2023-WEPA101}, + url = {https://indico.jacow.org/event/41/contributions/2276}, + language = {English} +} diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index 4fe1b9e155a..7a93d081c6b 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -15,3 +15,4 @@ This section collects typical user workflows and best practices for WarpX. workflows/plot_distribution_mapping workflows/psatd_stencil workflows/archiving + workflows/ml_dataset_training diff --git a/Docs/source/usage/workflows/ml_dataset_training.rst b/Docs/source/usage/workflows/ml_dataset_training.rst new file mode 100644 index 00000000000..ace0577c763 --- /dev/null +++ b/Docs/source/usage/workflows/ml_dataset_training.rst @@ -0,0 +1,256 @@ +.. _ml_dataset_training: + +Training a Surrogate Model from WarpX Data +========================================== + +Suppose we have a WarpX simulation that we wish to replace with a neural network surrogate model. +For example, a simulation determined by the following input script + +.. dropdown:: Python Input for Training Simulation + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/run_warpx_training.py + :language: python + +In this section we walk through a workflow for data processing and model training. +This workflow was developed and first presented in Refs. :cite:t:`SandbergIPAC23` and :cite:t:`SandbergPASC24`. + +This assumes you have an up-to-date environment with PyTorch and openPMD. + +Data Cleaning +------------- + +It is important to inspect the data for artifacts to +check that input/output data make sense. +If we plot the final phase space for beams 1-8, +the particle data is distributed in a single blob. + +.. figure:: https://user-images.githubusercontent.com/10621396/290010209-c55baf1c-dd98-4d56-a675-ad3729481eee.png + :alt: Plot comparing model prediction with simulation output. + + Plot showing the final phase space projections of the training data for stage 1. + +This is as we expect and what is optimal for training neural networks. +On the other hand, the final phase space for beam 0 has a halo of outlying particles. + +.. figure:: https://user-images.githubusercontent.com/10621396/290010282-40560ac4-8509-4599-82ca-167bb1739cff.png + :alt: Plot comparing model prediction with simulation output. + + Plot showing the final phase space projections of the training data for stage 1. + +Looking closer at the z-pz space, we see that some particles got caught in a decelerating +region of the wake, have slipped back and are much slower than the rest of the beam. +To assist our neural network in learning dynamics of interest, we filter out these particles. +It is sufficient for our purposes to select particles that are not too far back, setting +``particle_selection={'z':[0.28002, None]}``. Then a particle tracker is set up to make sure +we consistently filter out these particles from both the initial and final data. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Particle tracking START + :end-before: # Manual: Particle tracking END + +Create Normalized Dataset +------------------------- + +Having chosen training data we are content with, we now need to format the data, +normalize it, and store the normalized data as well as the normalizations. +The script below will take the openPMD data we have selected and +format, normalize, and store it. + +.. dropdown:: Python dataset creation + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/create_dataset.py + :language: python + +Load openPMD Data +^^^^^^^^^^^^^^^^^ + +First the openPMD data is loaded, using the particle selector as chosen above. +The neural network will make predictions from the initial phase space coordinates, +using the final phase space coordinates to measure how well it is making predictions. +Hence we load two sets of particle data, the source and target particle arrays. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Load openPMD START + :end-before: # Manual: Load openPMD END + +Normalize Data +^^^^^^^^^^^^^^ + +Neural networks learn better on appropriately normalized data. +Here we subtract out the mean in each coordinate direction and +divide by the standard deviation in each coordinate direction, +for normalized data that is centered on the origin with unit variance. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Normalization START + :end-before: # Manual: Normalization END + +openPMD to PyTorch Data +^^^^^^^^^^^^^^^^^^^^^^^ + +With the data normalized, it must be stored in a form PyTorch recognizes. +The openPMD data are 6 lists of arrays, for each of the 6 phase space coordinates +:math:`x, y, z, p_x, p_y,` and :math:`p_z`. +This data are converted to an :math:`N\times 6` numpy array and then to a PyTorch :math:`N\times 6` tensor. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Format data START + :end-before: # Manual: Format data END + +Save Normalizations and Normalized Data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +With the data properly normalized, it and the normalizations are saved to file for +use in training and inference. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Save dataset START + :end-before: # Manual: Save dataset END + +Neural Network Structure +------------------------ + +It was found in Ref. :cite:p:`SandbergPASC24` that reasonable surrogate models are obtained with +shallow feedforward neural networks consisting of fewer than 10 hidden layers and +just under 1000 nodes per layer. +The example shown here uses 3 hidden layers and 20 nodes per layer +and is trained for 10 epochs. + + + +Train and Save Neural Network +----------------------------- + +The script below trains the neural network on the dataset just created. +In subsequent sections we discuss the various parts of the training process. + +.. dropdown:: Python neural network training + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/train.py + :language: python3 + +Training Function +^^^^^^^^^^^^^^^^^ + +In the training function, the model weights are updated. +Iterating through batches, the loss function is evaluated on each batch. +PyTorch provides automatic differentiation, so the direction of steepest descent +is determined when the loss function is evaluated and the ``loss.backward()`` function +is invoked. +The optimizer uses this information to update the weights in the ``optimizer.step()`` call. +The training loop then resets the optimizer and updates the summed error for the whole dataset +with the error on the batch and continues iterating through batches. +Note that this function returns the sum of all errors across the entire dataset, +which is later divided by the size of the dataset in the training loop. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Train function START + :end-before: # Manual: Train function END + +Testing Function +^^^^^^^^^^^^^^^^ + +The testing function just evaluates the neural network on the testing data that has not been used +to update the model parameters. +This testing function requires that the testing dataset is small enough to be loaded all at once. +The PyTorch dataloader can load data in batches if this size assumption is not satisfied. +The error, measured by the loss function, is returned by the testing function to be aggregated and stored. +Note that this function returns the sum of all errors across the entire dataset, +which is later divided by the size of the dataset in the training loop. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Test function START + :end-before: # Manual: Test function END + +Train Loop +^^^^^^^^^^ + +The full training loop performs ``n_epochs`` number of iterations. +At each iteration the training and testing functions are called, +the respective errors are divided by the size of the dataset and recorded, +and a status update is printed to the console. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Training loop START + :end-before: # Manual: Training loop END + +Save Neural Network Parameters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The model weights are saved after training to record the updates to the model parameters. +Addtionally, we save some model metainformation with the model for convenience, +including the model hyperparameters, the training and testing losses, and how long the training took. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Save model START + :end-before: # Manual: Save model END + +Evaluate +-------- + +In this section we show two ways to diagnose how well the neural network is learning the data. +First we consider the train-test loss curves, shown in Fig. `[fig:train_test_loss] <#fig:train-test>`__ . +This figure shows the model error on the training data (in blue) and testing data (in green) as a function of the number of epochs seen. +The training data is used to update the model parameters, so training error should be lower than testing error. +A key feature to look for in the train-test loss curve is the inflection point in the test loss trend. +The testing data is set aside as a sample of data the neural network hasn't seen before. +The testing error serves as a metric of model generalizability, indicating how well the model performs +on data it hasn't seen yet. +When the test-loss starts to trend flat or even upward, the neural network is no longer improving its ability to generalize to new data. + +.. figure:: https://user-images.githubusercontent.com/10621396/290010428-f83725ab-a08f-494c-b075-314b0d26cb9a.png + :alt: Plot of training and testing loss curves + + Plot of training (in blue) and testing (in green) loss curves versus number of training epochs. + +A visual inspection of the model prediction can be seen in Fig. `[fig:train_evaluation]` . +This plot compares the model prediction, with dots colored by mean-square error, on the testing data with the actual simulation output in black. + +.. figure:: https://user-images.githubusercontent.com/10621396/290010486-4a3541e7-e0be-4cf1-b33b-57d5e5985196.png + :alt: Plot comparing model prediction with simulation output. + + Plot comparing model prediction (yellow-red dots, colored by mean-squared error) with simulation output (black dots). + +The model obtained with the hyperparameters chosen here trains quickly but is not very accurate. +A more accurate model is obtained with 5 hidden layers and 800 nodes per layer, +as discussed in :cite:t:`SandbergPASC24`. + +These figures can be generated with the following Python script. + +.. dropdown:: Python visualization of progress training neural network + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/visualize.py + :language: python3 + + +Surrogate Usage in Accelerator Physics +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A neural network such as the one we trained here can be incorporated in other BLAST codes. +`Consider the example using neural networks in ImpactX `__. diff --git a/Docs/source/usage/workflows/ml_materials/create_dataset.py b/Docs/source/usage/workflows/ml_materials/create_dataset.py new file mode 100644 index 00000000000..08000105479 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/create_dataset.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +import os +import tarfile +from urllib import request + +import numpy as np + +c = 2.998e8 + +from openpmd_viewer import OpenPMDTimeSeries, ParticleTracker +import torch + +############### + +def sanitize_dir_strings(*dir_strings): + """append '/' to a string for concatenation in building up file tree descriptions + """ + dir_strings = list(dir_strings) + for ii, dir_string in enumerate(dir_strings): + if dir_string[-1] != '/': + dir_strings[ii] = dir_string + '/' + + return dir_strings + +def download_and_unzip(url, data_dir): + request.urlretrieve(url, data_dir) + with tarfile.open(data_dir) as tar_dataset: + tar_dataset.extractall() + +def create_source_target_data(data_dir, + species, + source_index=0, + target_index=-1, + particle_selection=None + ): + """Create dataset from openPMD files + + Parameters + --- + data_dir : string, location of diagnostic data + source_index : int, which index to take source data from + target_index : int, which index to take target data from + particle_selection: dictionary, optional, selection criterion for dataset + + Returns + --- + source_data: Nx6 array of source particle data + source_means: 6 element array of source particle coordinate means + source_stds: 6 element array of source particle coordinate standard deviations + target_data: Nx6 array of target particle data + target_means: 6 element array of target particle coordinate means + target_stds: 6 element array of source particle coordinate standard deviations + relevant times: 2 element array of source and target times + """ + data_dir, = sanitize_dir_strings(data_dir) + data_path = data_dir + print('loading openPMD data from', data_path) + ts = OpenPMDTimeSeries(data_path) + relevant_times = [ts.t[source_index], ts.t[target_index]] + + # Manual: Particle tracking START + iteration = ts.iterations[target_index] + pt = ParticleTracker( ts, + species=species, + iteration=iteration, + select=particle_selection) + # Manual: Particle tracking END + + #### create normalized source, target data sets #### + print('creating data sets') + + # Manual: Load openPMD START + iteration = ts.iterations[source_index] + source_data = ts.get_particle(species=species, + iteration=iteration, + var_list=['x','y','z','ux','uy','uz'], + select=pt) + + iteration = ts.iterations[target_index] + target_data = ts.get_particle(species=species, + iteration=iteration, + var_list=['x','y','z','ux','uy','uz'], + select=pt) + # Manual: Load openPMD END + + # Manual: Normalization START + target_means = np.zeros(6) + target_stds = np.zeros(6) + source_means = np.zeros(6) + source_stds = np.zeros(6) + for jj in range(6): + source_means[jj] = source_data[jj].mean() + source_stds[jj] = source_data[jj].std() + source_data[jj] -= source_means[jj] + source_data[jj] /= source_stds[jj] + + for jj in range(6): + target_means[jj] = target_data[jj].mean() + target_stds[jj] = target_data[jj].std() + target_data[jj] -= target_means[jj] + target_data[jj] /= target_stds[jj] + # Manual: Normalization END + + # Manual: Format data START + source_data = torch.tensor(np.column_stack(source_data)) + target_data = torch.tensor(np.column_stack(target_data)) + # Manual: Format data END + + return source_data, source_means, source_stds, target_data, target_means, target_stds, relevant_times + + +def save_warpx_surrogate_data(dataset_fullpath_filename, + diag_dir, + species, + training_frac, + batch_size, + source_index, + target_index, + survivor_select_index, + particle_selection=None + ): + + source_target_data = create_source_target_data(data_dir=diag_dir, + species=species, + source_index=source_index, + target_index=target_index, + particle_selection=particle_selection + ) + source_data, source_means, source_stds, target_data, target_means, target_stds, times = source_target_data + + # Manual: Save dataset START + full_dataset = torch.utils.data.TensorDataset(source_data.float(), target_data.float()) + + n_samples = full_dataset.tensors[0].size(0) + n_train = int(training_frac*n_samples) + n_test = n_samples - n_train + + train_data, test_data = torch.utils.data.random_split(full_dataset, [n_train, n_test]) + + torch.save({'dataset':full_dataset, + 'train_indices':train_data.indices, + 'test_indices':test_data.indices, + 'source_means':source_means, + 'source_stds':source_stds, + 'target_means':target_means, + 'target_stds':target_stds, + 'times':times, + }, + dataset_fullpath_filename + ) + # Manual: Save dataset END + +######## end utility functions ############# +######## start dataset creation ############ + +data_url = "https://zenodo.org/records/10368972/files/ml_example_training.tar.gz?download=1" +download_and_unzip(data_url, "training_dataset.tar.gz") +data_dir = "lab_particle_diags/lab_particle_diags/" + + +# create data set + +source_index = 0 +target_index = 1 +survivor_select_index = 1 +batch_size=1200 +training_frac = 0.7 + +os.makedirs('datasets', exist_ok=True) + +# improve stage 0 dataset +stage_i = 0 +select = {'z':[0.28002, None]} +species = f'beam_stage_{stage_i}' +dataset_filename = f'dataset_{species}.pt' +dataset_file = 'datasets/' + dataset_filename +save_warpx_surrogate_data(dataset_fullpath_filename=dataset_file, + diag_dir=data_dir, + species=species, + training_frac=training_frac, + batch_size=batch_size, + source_index=source_index, + target_index=target_index, + survivor_select_index=survivor_select_index, + particle_selection=select + ) + +for stage_i in range(1,9): + species = f'beam_stage_{stage_i}' + dataset_filename = f'dataset_{species}.pt' + dataset_file = 'datasets/' + dataset_filename + save_warpx_surrogate_data(dataset_fullpath_filename=dataset_file, + diag_dir=data_dir, + species=species, + training_frac=training_frac, + batch_size=batch_size, + source_index=source_index, + target_index=target_index, + survivor_select_index=survivor_select_index + ) diff --git a/Docs/source/usage/workflows/ml_materials/neural_network_classes.py b/Docs/source/usage/workflows/ml_materials/neural_network_classes.py new file mode 100644 index 00000000000..58b51a1d364 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/neural_network_classes.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +from enum import Enum + +from torch import nn + + +class ActivationType(Enum): + """ + Activation class provides an enumeration type for the supported activation layers + """ + ReLU = 1 + Tanh = 2 + PReLU = 3 + Sigmoid = 4 + +def get_enum_type(type_to_test, EnumClass): + """ + Returns the enumeration type associated to type_to_test in EnumClass + + Parameters + ---------- + type_to_test: EnumClass, int or str + object whose Enum class is to be obtained + EnumClass: Enum class + Enum class to test + """ + if type(type_to_test) is EnumClass: + return type_to_test + if type(type_to_test) is int: + return EnumClass(type_to_test) + if type(type_to_test) is str: + return getattr(EnumClass, type_to_test) + else: + raise Exception("unsupported type entered") + + + +class ConnectedNN(nn.Module): + """ + ConnectedNN is a class of fully connected neural networks + """ + def __init__(self, layers): + super().__init__() + self.stack = nn.Sequential(*layers) + def forward(self, x): + return self.stack(x) + +class OneActNN(ConnectedNN): + """ + OneActNN is class of fully connected neural networks admitting only one activation function + """ + def __init__(self, + n_in, + n_out, + n_hidden_nodes, + n_hidden_layers, + act): + + self.n_in = n_in + self.n_out = n_out + self.n_hidden_layers = n_hidden_layers + self.n_hidden_nodes = n_hidden_nodes + + self.act = get_enum_type(act, ActivationType) + + layers = [nn.Linear(self.n_in, self.n_hidden_nodes)] + + for ii in range(self.n_hidden_layers): + if self.act is ActivationType.ReLU: + layers += [nn.ReLU()] + if self.act is ActivationType.Tanh: + layers += [nn.Tanh()] + if self.act is ActivationType.PReLU: + layers += [nn.PReLU()] + if self.act is ActivationType.Sigmoid: + layers += [nn.Sigmoid()] + + if ii < self.n_hidden_layers - 1: + layers += [nn.Linear(self.n_hidden_nodes,self.n_hidden_nodes)] + + layers += [nn.Linear(self.n_hidden_nodes, self.n_out)] + + super().__init__(layers) diff --git a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py new file mode 100644 index 00000000000..8ee098e3e29 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +# +# Copyright 2022-2023 WarpX contributors +# Authors: WarpX team +# License: BSD-3-Clause-LBNL +# +# -*- coding: utf-8 -*- + +import math + +import numpy as np + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c +q_e = picmi.constants.q_e +m_e = picmi.constants.m_e +m_p = picmi.constants.m_p +ep0 = picmi.constants.ep0 + +# Number of cells +dim = '3' +nx = ny = 128 +nz = 8832 +if dim == 'rz': + nr = nx//2 + +# Computational domain +rmin = 0. +rmax = 128e-6 +zmin = -180e-6 +zmax = 0. + +# Number of processes for static load balancing +# Check with your submit script +num_procs = [1, 1, 16*4] +if dim == 'rz': + num_procs = [1, 16] + +# Number of time steps +gamma_boost = 60. +beta_boost = np.sqrt(1.-gamma_boost**-2) + +# Create grid +if dim == 'rz': + grid = picmi.CylindricalGrid( + number_of_cells=[nr, nz], + guard_cells=[32, 32], + n_azimuthal_modes=2, + lower_bound=[rmin, zmin], + upper_bound=[rmax, zmax], + lower_boundary_conditions=['none', 'damped'], + upper_boundary_conditions=['none', 'damped'], + lower_boundary_conditions_particles=['absorbing', 'absorbing'], + upper_boundary_conditions_particles=['absorbing', 'absorbing'], + moving_window_velocity=[0., c], + warpx_max_grid_size=256, + warpx_blocking_factor=64) +else: + grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + guard_cells=[11, 11, 12], + lower_bound=[-rmax, -rmax, zmin], + upper_bound=[rmax, rmax, zmax], + lower_boundary_conditions=['periodic', 'periodic', 'damped'], + upper_boundary_conditions=['periodic', 'periodic', 'damped'], + lower_boundary_conditions_particles=['periodic', 'periodic', 'absorbing'], + upper_boundary_conditions_particles=['periodic', 'periodic', 'absorbing'], + moving_window_velocity=[0., 0., c], + warpx_max_grid_size=256, + warpx_blocking_factor=32) + + +# plasma region +plasma_rlim = 100.e-6 +N_stage = 9 +L_plasma_bulk = 0.28 +L_ramp = 1.e-9 +L_ramp_up = L_ramp +L_ramp_down = L_ramp +L_stage = L_plasma_bulk + 2*L_ramp + +# focusing +# lens external fields +beam_gamma1 = 15095 +lens_focal_length = 0.015 +lens_width = 0.003 + +stage_spacing = L_plasma_bulk + 2*lens_focal_length + +def get_species_of_accelerator_stage(stage_idx, stage_zmin, stage_zmax, + stage_xmin=-plasma_rlim, stage_xmax=plasma_rlim, + stage_ymin=-plasma_rlim, stage_ymax=plasma_rlim, + Lplus = L_ramp_up, Lp = L_plasma_bulk, + Lminus = L_ramp_down): + # Parabolic density profile + n0 = 1.7e23 + Rc = 40.e-6 + Lstage = Lplus + Lp + Lminus + if not np.isclose(stage_zmax-stage_zmin, Lstage): + print('Warning: zmax disagrees with stage length') + parabolic_distribution = picmi.AnalyticDistribution( + density_expression= + f'n0*(1.+4.*(x**2+y**2)/(kp**2*Rc**4))*(0.5*(1.-cos(pi*(z-{stage_zmin})/Lplus)))*((z-{stage_zmin})=Lplus)*((z-{stage_zmin})<(Lplus+Lp))' \ + + f'+n0*(1.+4.*(x**2+y**2)/(kp**2*Rc**4))*(0.5*(1.+cos(pi*((z-{stage_zmin})-Lplus-Lp)/Lminus)))*((z-{stage_zmin})>=(Lplus+Lp))*((z-{stage_zmin})<(Lplus+Lp+Lminus))', + pi=3.141592653589793, + n0=n0, + kp=q_e/c*math.sqrt(n0/(m_e*ep0)), + Rc=Rc, + Lplus=Lplus, + Lp=Lp, + Lminus=Lminus, + lower_bound=[stage_xmin, stage_ymin, stage_zmin], + upper_bound=[stage_xmax, stage_ymax, stage_zmax], + fill_in=True) + + electrons = picmi.Species( + particle_type='electron', + name=f'electrons{stage_idx}', + initial_distribution=parabolic_distribution) + + ions = picmi.Species( + particle_type='proton', + name=f'ions{stage_idx}', + initial_distribution=parabolic_distribution) + + return electrons, ions + +species_list = [] +for i_stage in range(1): + # Add plasma + zmin_stage = i_stage * stage_spacing + zmax_stage = zmin_stage + L_stage + electrons, ions = get_species_of_accelerator_stage(i_stage+1, zmin_stage, zmax_stage) + species_list.append(electrons) + species_list.append(ions) + +# add beam to species_list +beam_charge = -10.e-15 # in Coulombs +N_beam_particles = int(1e6) +beam_centroid_z = -107.e-6 +beam_rms_z = 2.e-6 +#beam_gammas = [2000 + 13000 * i_stage for i_stage in range(N_stage)] +beam_gammas = [1957, 15188, 28432, 41678, 54926, 68174, 81423,94672, 107922,121171] # From 3D run +beams = [] +for i_stage in range(N_stage): + beam_gamma = beam_gammas[i_stage] + sigma_gamma = 0.10 * beam_gamma + gaussian_distribution = picmi.GaussianBunchDistribution( + n_physical_particles= abs(beam_charge) / q_e, + rms_bunch_size=[2.e-6, 2.e-6, beam_rms_z], + rms_velocity=[8*c, 8*c, sigma_gamma*c], + centroid_position=[0., 0., beam_centroid_z], + centroid_velocity=[0., 0., beam_gamma*c], + ) + beam = picmi.Species( + particle_type='electron', + name=f'beam_stage_{i_stage}', + initial_distribution= gaussian_distribution + ) + beams.append(beam) + +# Laser +antenna_z = -1e-9 +profile_t_peak = 1.46764864e-13 +def get_laser(antenna_z, profile_t_peak, fill_in=True): + profile_focal_distance = 0. + laser = picmi.GaussianLaser( + wavelength=0.8e-06, + waist=36e-06, + duration=7.33841e-14, + focal_position=[0., 0., profile_focal_distance + antenna_z], + centroid_position=[0., 0., antenna_z - c*profile_t_peak], + propagation_direction=[0., 0., 1.], + polarization_direction=[0., 1., 0.], + a0=2.36, + fill_in=fill_in) + laser_antenna = picmi.LaserAntenna( + position=[0., 0., antenna_z], + normal_vector=[0., 0., 1.]) + return (laser, laser_antenna) +lasers = [] +for i_stage in range(1): + fill_in = True + if i_stage == 0: + fill_in = False + lasers.append( + get_laser(antenna_z + i_stage*stage_spacing, + profile_t_peak + i_stage*stage_spacing/c, + fill_in) + ) + +# Electromagnetic solver + +psatd_algo = 'multij' +if psatd_algo == 'galilean': + galilean_velocity = [0.,0.] if dim=='3' else [0.] + galilean_velocity += [-c*beta_boost] + n_pass_z = 1 + do_multiJ = None + do_multi_J_n_depositions=None + J_in_time = None + current_correction = True + divE_cleaning = False +elif psatd_algo == 'multij': + n_pass_z = 4 + galilean_velocity = None + do_multiJ = True + do_multi_J_n_depositions = 2 + J_in_time = "linear" + current_correction = False + divE_cleaning = True +else: + raise Exception(f'PSATD algorithm \'{psatd_algo}\' is not recognized!\n'\ + 'Valid options are \'multiJ\' or \'galilean\'.') +if dim == 'rz': + stencil_order = [8, 16] + smoother = picmi.BinomialSmoother(n_pass=[1,n_pass_z]) + grid_type = 'collocated' +else: + stencil_order = [8, 8, 16] + smoother = picmi.BinomialSmoother(n_pass=[1,1,n_pass_z]) + grid_type = 'hybrid' + + +solver = picmi.ElectromagneticSolver( + grid=grid, + method='PSATD', + cfl=0.9999, + source_smoother=smoother, + stencil_order=stencil_order, + galilean_velocity=galilean_velocity, + warpx_psatd_update_with_rho=True, + warpx_current_correction=current_correction, + divE_cleaning=divE_cleaning, + warpx_psatd_J_in_time=J_in_time + ) + +# Diagnostics +diag_field_list = ['B', 'E', 'J', 'rho'] +diag_particle_list = ['weighting','position','momentum'] +coarse_btd_end = int((L_plasma_bulk+0.001+stage_spacing*(N_stage-1))*100000) +stage_end_snapshots=[f'{int((L_plasma_bulk+stage_spacing*ii)*100000)}:{int((L_plasma_bulk+stage_spacing*ii)*100000+50)}:5' for ii in range(1)] +btd_particle_diag = picmi.LabFrameParticleDiagnostic( + name='lab_particle_diags', + species=beams, + grid=grid, + num_snapshots=25*N_stage, + #warpx_intervals=', '.join([f':{coarse_btd_end}:1000']+stage_end_snapshots), + warpx_intervals=', '.join(['0:0']+stage_end_snapshots), + dt_snapshots=0.00001/c, + data_list=diag_particle_list, + write_dir='lab_particle_diags', + warpx_format='openpmd', + warpx_openpmd_backend='bp') + +btd_field_diag = picmi.LabFrameFieldDiagnostic( + name='lab_field_diags', + grid=grid, + num_snapshots=25*N_stage, + dt_snapshots=stage_spacing/25/c, + data_list=diag_field_list, + warpx_lower_bound=[-128.e-6, 0.e-6, -180.e-6], + warpx_upper_bound=[128.e-6, 0.e-6, 0.], + write_dir='lab_field_diags', + warpx_format='openpmd', + warpx_openpmd_backend='bp') + +field_diag = picmi.FieldDiagnostic( + name='field_diags', + data_list=diag_field_list, + grid=grid, + period=100, + write_dir='field_diags', + lower_bound=[-128.e-6, 0.e-6, -180.e-6], + upper_bound=[128.e-6, 0.e-6, 0.], + warpx_format='openpmd', + warpx_openpmd_backend='h5') + +particle_diag = picmi.ParticleDiagnostic( + name='particle_diags', + species=beams, + period=100, + write_dir='particle_diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5') + +beamrel_red_diag = picmi.ReducedDiagnostic( + diag_type='BeamRelevant', + name='beamrel', + species=beam, + period=1) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, + warpx_numprocs=num_procs, + warpx_compute_max_step_from_btd=True, + verbose=2, + particle_shape='cubic', + gamma_boost=gamma_boost, + warpx_charge_deposition_algo='standard', + warpx_current_deposition_algo='direct', + warpx_field_gathering_algo='momentum-conserving', + warpx_particle_pusher_algo='vay', + warpx_amrex_the_arena_is_managed=False, + warpx_amrex_use_gpu_aware_mpi=True, + warpx_do_multi_J=do_multiJ, + warpx_do_multi_J_n_depositions=do_multi_J_n_depositions, + warpx_grid_type=grid_type, + # default: 2 for staggered grids, 8 for hybrid grids + warpx_field_centering_order=[16,16,16], + # only for hybrid grids, default: 8 + warpx_current_centering_order=[16,16,16] + ) + +for species in species_list: + if dim=='rz': + n_macroparticle_per_cell=[2,4,2] + else: + n_macroparticle_per_cell=[2,2,2] + sim.add_species( + species, + layout=picmi.GriddedLayout(grid=grid, + n_macroparticle_per_cell=n_macroparticle_per_cell) + ) + +for i_stage in range(N_stage): + sim.add_species_through_plane( + species=beams[i_stage], + layout=picmi.PseudoRandomLayout(grid=grid, n_macroparticles=N_beam_particles), + injection_plane_position=0., + injection_plane_normal_vector=[0.,0.,1.]) + +for i_stage in range(1): + # Add laser + (laser, laser_antenna) = lasers[i_stage] + sim.add_laser(laser, injection_method=laser_antenna) + +# Add diagnostics +sim.add_diagnostic(btd_particle_diag) +#sim.add_diagnostic(btd_field_diag) +#sim.add_diagnostic(field_diag) +#sim.add_diagnostic(particle_diag) + +# Add reduced diagnostic +sim.add_diagnostic(beamrel_red_diag) + +sim.write_input_file(f'inputs_training_{N_stage}_stages') + +# Advance simulation until last time step +sim.step() diff --git a/Docs/source/usage/workflows/ml_materials/train.py b/Docs/source/usage/workflows/ml_materials/train.py new file mode 100644 index 00000000000..dc62819061c --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/train.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +import os +import time + +import neural_network_classes as mynn +import torch +import torch.nn.functional as F +import torch.optim as optim + +############# set model parameters ################# + +stage_i = 0 +species = f'beam_stage_{stage_i}' +source_index = 0 +target_index = 4 +survivor_select_index = 4 + +data_dim = 6 +n_in = data_dim +n_out = data_dim + +learning_rate = 0.0001 +n_epochs = 10 +batch_size = 1200 + +loss_fun = F.mse_loss + +n_hidden_nodes = 20 +n_hidden_layers = 3 +activation_type = 'ReLU' + +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +print(f'device={device}') +#################### load dataset ################ +dataset_filename = f'dataset_{species}.pt' +dataset_file = 'datasets/' + dataset_filename + +print(f"trying to load dataset+test-train split in {dataset_file}") + +dataset_with_indices = torch.load(dataset_file) +train_data = torch.utils.data.dataset.Subset(dataset_with_indices['dataset'], dataset_with_indices['train_indices']) +test_data = torch.utils.data.dataset.Subset(dataset_with_indices['dataset'], dataset_with_indices['test_indices']) +source_data = dataset_with_indices['dataset'] +source_means = dataset_with_indices['source_means'] +source_stds = dataset_with_indices['source_stds'] +target_means = dataset_with_indices['target_means'] +target_stds = dataset_with_indices['target_stds'] +print("able to load data and test/train split") + +###### move data to device (GPU) if available ######## +source_device = train_data.dataset.tensors[0].to(device) # equivalently, test_data.tensors[0].to(device) +target_device = train_data.dataset.tensors[1].to(device) +full_dataset_device = torch.utils.data.TensorDataset(source_device.float(), target_device.float()) + +train_data_device = torch.utils.data.dataset.Subset(full_dataset_device, train_data.indices) +test_data_device = torch.utils.data.dataset.Subset(full_dataset_device, test_data.indices) + +train_loader_device = torch.utils.data.DataLoader(train_data_device, batch_size=batch_size, shuffle=True) +test_loader_device = torch.utils.data.DataLoader(test_data_device, batch_size=batch_size, shuffle=True) + +test_source_device = test_data_device.dataset.tensors[0] +test_target_device = test_data_device.dataset.tensors[1] + +training_set_size = len(train_data_device.indices) +testing_set_size = len(test_data_device.indices) + +###### create model ########### + +model = mynn.OneActNN(n_in = n_in, + n_out = n_out, + n_hidden_nodes=n_hidden_nodes, + n_hidden_layers = n_hidden_layers, + act=activation_type + ) + +training_time = 0 +train_loss_list = [] +test_loss_list = [] + +model.to(device=device); + +########## train and test functions #### +# Manual: Train function START +def train(model, optimizer, train_loader, loss_fun): + model.train() + total_loss = 0. + for batch_idx, (data, target) in enumerate(train_loader): + #evaluate network with data + output = model(data) + #compute loss + # sum the differences squared, take mean afterward + loss = loss_fun(output, target,reduction='sum') + #backpropagation: step optimizer and reset gradients + loss.backward() + optimizer.step() + optimizer.zero_grad() + total_loss += loss.item() + return total_loss +# Manual: Train function END + +def test(model, test_loader, loss_fun): + model.eval() + total_loss = 0. + with torch.no_grad(): + for batch_idx, (data, target) in enumerate(test_loader): + output = model(data) + total_loss += loss_fun(output, target, reduction='sum').item() + return total_loss + +# Manual: Test function START +def test_dataset(model, test_source, test_target, loss_fun): + model.eval() + with torch.no_grad(): + output = model(test_source) + return loss_fun(output, test_target, reduction='sum').item() +# Manual: Test function END + +######## training loop ######## + +optimizer = optim.Adam(model.parameters(), lr=learning_rate) + +do_print = True + +t3 = time.time() +# Manual: Training loop START +for epoch in range(n_epochs): + if do_print: + t1 = time.time() + ave_train_loss = train(model, optimizer, train_loader_device, loss_fun) / data_dim / training_set_size + ave_test_loss = test_dataset(model, test_source_device, test_target_device, loss_fun) / data_dim / training_set_size + train_loss_list.append(ave_train_loss) + test_loss_list.append(ave_test_loss) + + if do_print: + t2 = time.time() + print('Train Epoch: {:04d} \tTrain Loss: {:.6f} \tTest Loss: {:.6f}, this epoch: {:.3f} s'.format( + epoch + 1, ave_train_loss, ave_test_loss, t2-t1)) +# Manual: Training loop END +t4 = time.time() +print(f'total training time: {t4-t3:.3f}s') + +######### save model ######### + +os.makedirs('models', exist_ok=True) + +# Manual: Save model START +model.to(device='cpu') +torch.save({ + 'n_hidden_layers':n_hidden_layers, + 'n_hidden_nodes':n_hidden_nodes, + 'activation':activation_type, + 'model_state_dict': model.state_dict(), + 'optimizer_state_dict': optimizer.state_dict(), + 'train_loss_list': train_loss_list, + 'test_loss_list': test_loss_list, + 'training_time': training_time, + }, f'models/{species}_model.pt') +# Manual: Save model END diff --git a/Docs/source/usage/workflows/ml_materials/visualize.py b/Docs/source/usage/workflows/ml_materials/visualize.py new file mode 100644 index 00000000000..e6da1029b67 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/visualize.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +from matplotlib import pyplot as plt +import neural_network_classes as mynn +import numpy as np +import torch +import torch.nn.functional as F + +c = 2.998e8 + + +# open model file +stage_i = 0 +species = f'beam_stage_{stage_i}' +model_data = torch.load(f'models/{species}_model.pt',map_location=torch.device('cpu')) +data_dim = 6 +n_in = data_dim +n_out = data_dim +n_hidden_layers = model_data['n_hidden_layers'] +n_hidden_nodes = model_data['n_hidden_nodes'] +activation_type = model_data['activation'] +train_loss_list = model_data['train_loss_list'] +test_loss_list = model_data['test_loss_list'] +training_time = model_data['training_time'] +loss_fun = F.mse_loss + + +n_epochs = len(train_loss_list) +train_counter = np.arange(n_epochs)+1 +test_counter = train_counter + +do_log_plot = False +fig, ax = plt.subplots() +if do_log_plot: + ax.semilogy(train_counter, train_loss_list, '.-',color='blue',label='training loss') + ax.semilogy(test_counter, test_loss_list, color='green',label='testing loss') +else: + ax.plot(train_counter, train_loss_list, '.-',color='blue',label='training loss') + ax.plot(test_counter, test_loss_list, color='green',label='testing loss') +ax.set_xlabel('number of epochs seen') +ax.set_ylabel(' loss') +ax.legend() +fig_dir = 'figures/' +ax.set_title(f'final test error = {test_loss_list[-1]:.3e} ') +ax.grid() +plt.tight_layout() +plt.savefig(f'{species}_training_testing_error.png') + + +######### plot phase space comparison ####### +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +print(f'device={device}') + +model = mynn.OneActNN(n_in = n_in, + n_out = n_out, + n_hidden_nodes=n_hidden_nodes, + n_hidden_layers = n_hidden_layers, + act = activation_type + ) +model.load_state_dict(model_data['model_state_dict']) +model.to(device=device); + +###### load model data ############### +dataset_filename = f'dataset_{species}.pt' +dataset_dir = 'datasets/' +model_input_data = torch.load(dataset_dir + dataset_filename) +dataset = model_input_data['dataset'] +train_indices = model_input_data['train_indices'] +test_indices = model_input_data['test_indices'] +source_means = model_input_data['source_means'] +source_stds = model_input_data['source_stds'] +target_means = model_input_data['target_means'] +target_stds = model_input_data['target_stds'] +source_time, target_time = model_input_data['times'] + + +source = dataset.tensors[0] +test_source = source[test_indices] +test_source_device = test_source.to(device) +with torch.no_grad(): + evaluation_device = model(test_source_device.float()) +eval_cpu = evaluation_device.to('cpu') + +target = dataset.tensors[1] +test_target = target[test_indices] + +target_si = test_target * target_stds + target_means +eval_cpu_si = eval_cpu * target_stds + target_means +target_mu = np.copy(target_si) +eval_cpu_mu = np.copy(eval_cpu_si) +target_mu[:,2] -= c*target_time +eval_cpu_mu[:,2] -= c*target_time +target_mu[:,:3] *= 1e6 +eval_cpu_mu[:,:3] *= 1e6 + + + +loss_tensor = torch.sum(loss_fun(eval_cpu, + test_target, + reduction='none'), + axis=1)/6 +loss_array = loss_tensor.detach().numpy() + +tinds = np.nonzero(loss_array > 0.0)[0] +skip = 10 + +plt.figure() +fig, axT = plt.subplots(3,3) +axes_label = {0:r'x [$\mu$m]', 1:r'y [$\mu$m]', 2:r'z - %.2f cm [$\mu$m]'%(c*target_time),3:r'$p_x$',4:r'$p_y$',5:r'$p_z$'} +xy_inds = [(0,1),(2,0),(2,1)] +def set_axes(ax, indx, indy): + ax.scatter(target_mu[::skip,indx],target_mu[::skip,indy],s=8,c='k', label='simulation') + ax.scatter(eval_cpu_mu[::skip,indx],eval_cpu_mu[::skip,indy],marker='*',c=loss_array[::skip],s=0.02, label='surrogate',cmap='YlOrRd') + ax.set_xlabel(axes_label[indx]) + ax.set_ylabel(axes_label[indy]) + # return + + +for ii in range(3): + ax = axT[0,ii] + indx,indy = xy_inds[ii] + set_axes(ax,indx,indy) + +for ii in range(2): + indx,indy = xy_inds[ii] + ax = axT[1,ii] + set_axes(ax,indx+3,indy+3) + +for ii in range(3): + ax = axT[2,ii] + indx = ii + indy = ii+3 + set_axes(ax, indx, indy) + + +ax = axT[1,2] +indx = 5 +indy = 4 +ax.scatter(target_mu[::skip,indx],target_mu[::skip,indy],s=8,c='k', label='simulation') +evalplt = ax.scatter(eval_cpu_mu[::skip,indx],eval_cpu_mu[::skip,indy],marker='*',c=loss_array[::skip],s=2, label='surrogate',cmap='YlOrRd') +ax.set_xlabel(axes_label[indx]) +ax.set_ylabel(axes_label[indy]) + +cb = plt.colorbar(evalplt, ax=ax) +cb.set_label('MSE loss') + +fig.suptitle(f'stage {stage_i} prediction') + +plt.tight_layout() + +plt.savefig(f'{species}_model_evaluation.png') From 654e6b3afd66f4553bec366e19a7dacaf7760f72 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Thu, 14 Dec 2023 01:33:43 -0800 Subject: [PATCH 040/176] Clang-Tidy: readability-braces-around-statements (#4511) Fixed by clang-tidy --fix. --- .clang-tidy | 1 - .../LatticeElements/HardEdgedPlasmaLens.cpp | 2 +- .../LatticeElements/HardEdgedQuadrupole.cpp | 2 +- Source/BoundaryConditions/PML.cpp | 14 +- Source/BoundaryConditions/WarpXEvolvePML.cpp | 12 +- .../WarpXFieldBoundaries.cpp | 9 +- Source/BoundaryConditions/WarpX_PEC.H | 23 +-- Source/BoundaryConditions/WarpX_PEC.cpp | 26 +-- .../Diagnostics/BTD_Plotfile_Header_Impl.cpp | 16 +- Source/Diagnostics/BTDiagnostics.cpp | 86 +++++----- .../BoundaryScrapingDiagnostics.cpp | 2 +- .../BackTransformFunctor.cpp | 2 +- .../BackTransformParticleFunctor.cpp | 9 +- .../ParticleReductionFunctor.cpp | 8 +- Source/Diagnostics/Diagnostics.cpp | 4 +- .../FlushFormats/FlushFormatOpenPMD.cpp | 16 +- .../FlushFormats/FlushFormatPlotfile.cpp | 11 +- Source/Diagnostics/FullDiagnostics.cpp | 12 +- Source/Diagnostics/MultiDiagnostics.cpp | 12 +- .../Diagnostics/ReducedDiags/BeamRelevant.cpp | 2 +- .../ReducedDiags/FieldMomentum.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 5 +- .../ReducedDiags/LoadBalanceCosts.cpp | 4 +- .../ReducedDiags/ParticleEnergy.cpp | 2 +- .../ReducedDiags/ParticleHistogram.cpp | 16 +- .../ReducedDiags/ParticleHistogram2D.cpp | 15 +- .../ReducedDiags/ParticleMomentum.cpp | 2 +- .../Diagnostics/ReducedDiags/ReducedDiags.cpp | 2 +- Source/Diagnostics/SliceDiagnostic.cpp | 2 +- Source/Diagnostics/WarpXIO.cpp | 8 +- Source/Diagnostics/WarpXOpenPMD.cpp | 156 ++++++++++-------- Source/Evolve/WarpXComputeDt.cpp | 7 +- Source/Evolve/WarpXEvolve.cpp | 67 ++++---- Source/FieldSolver/ElectrostaticSolver.cpp | 19 ++- .../ApplySilverMuellerBoundary.cpp | 72 +++++--- .../FiniteDifferenceSolver.cpp | 3 +- .../HybridPICModel/HybridPICModel.cpp | 4 +- .../HybridPICSolveE.cpp | 32 ++-- .../MagnetostaticSolver.cpp | 12 +- .../PsatdAlgorithmFirstOrder.cpp | 6 +- .../PsatdAlgorithmJConstantInTime.cpp | 12 +- .../PsatdAlgorithmJLinearInTime.cpp | 4 +- .../SpectralSolver/SpectralFieldData.cpp | 26 +-- .../SpectralSolver/SpectralFieldDataRZ.cpp | 4 +- .../SpectralHankelTransform/BesselRoots.cpp | 4 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 69 ++++---- .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 6 +- Source/Filter/BilinearFilter.cpp | 7 +- Source/Fluids/MusclHancockUtils.H | 56 ++++--- Source/Fluids/WarpXFluidContainer.cpp | 14 +- Source/Initialization/ExternalField.cpp | 6 +- Source/Initialization/InjectorMomentum.H | 10 +- Source/Initialization/InjectorPosition.H | 18 +- Source/Initialization/WarpXInitData.cpp | 26 ++- .../LaserProfileFromFile.cpp | 29 ++-- .../LaserProfileGaussian.cpp | 5 +- Source/Parallelization/GuardCellManager.cpp | 5 +- Source/Parallelization/WarpXComm.cpp | 28 ++-- Source/Parallelization/WarpXRegrid.cpp | 9 +- Source/Parallelization/WarpXSumGuardCells.H | 10 +- .../BackgroundMCC/BackgroundMCCCollision.cpp | 4 +- .../BackgroundMCC/ImpactIonization.H | 2 +- .../Collision/BackgroundMCC/MCCProcess.cpp | 4 +- .../BinaryCollision/BinaryCollision.H | 14 +- .../BinaryCollision/BinaryCollisionUtils.cpp | 15 +- .../Coulomb/ElasticCollisionPerez.H | 2 +- .../BinaryCollision/ParticleCreationFunc.H | 2 +- .../Particles/Deposition/CurrentDeposition.H | 14 +- .../QEDInternals/BreitWheelerEngineWrapper.H | 3 +- .../BreitWheelerEngineWrapper.cpp | 14 +- .../QEDInternals/QuantumSyncEngineWrapper.H | 3 +- .../QEDInternals/QuantumSyncEngineWrapper.cpp | 14 +- Source/Particles/Filter/FilterFunctors.H | 2 +- Source/Particles/Gather/GetExternalFields.H | 4 +- Source/Particles/Gather/GetExternalFields.cpp | 12 +- Source/Particles/Gather/ScaleFields.H | 2 +- Source/Particles/LaserParticleContainer.cpp | 14 +- Source/Particles/MultiParticleContainer.cpp | 40 +++-- Source/Particles/ParticleBoundaries_K.H | 6 +- Source/Particles/ParticleBoundaryBuffer.cpp | 23 +-- .../ParticleCreation/FilterCopyTransform.H | 8 +- .../FilterCreateTransformFromFAB.H | 4 +- Source/Particles/ParticleCreation/SmartCopy.H | 12 +- .../Particles/ParticleCreation/SmartCreate.H | 12 +- Source/Particles/PhotonParticleContainer.cpp | 2 +- .../Particles/PhysicalParticleContainer.cpp | 50 +++--- Source/Particles/Pusher/CopyParticleAttribs.H | 2 +- Source/Particles/Pusher/PushSelector.H | 2 +- .../Particles/Resampling/LevelingThinning.cpp | 3 +- .../RigidInjectedParticleContainer.cpp | 2 +- Source/Particles/WarpXParticleContainer.cpp | 23 +-- Source/Utils/Parser/IntervalsParser.cpp | 6 +- Source/Utils/Parser/ParserUtils.cpp | 2 +- Source/Utils/RelativeCellPosition.cpp | 3 +- Source/Utils/WarpXAlgorithmSelection.cpp | 3 +- Source/Utils/WarpXMovingWindow.cpp | 16 +- Source/Utils/WarpXTagging.cpp | 2 +- Source/Utils/WarpXUtil.cpp | 11 +- Source/Utils/WarpXVersion.cpp | 10 +- Source/WarpX.cpp | 44 ++--- Source/ablastr/coarsen/average.H | 10 +- Source/ablastr/coarsen/sample.H | 8 +- Source/ablastr/coarsen/sample.cpp | 7 +- Source/ablastr/fields/PoissonSolver.H | 8 +- Source/ablastr/fields/VectorPoissonSolver.H | 8 +- Source/ablastr/particles/DepositCharge.H | 11 +- Source/ablastr/profiler/ProfilerWrapper.H | 3 +- Source/ablastr/utils/Communication.cpp | 2 +- Source/ablastr/utils/Serialization.H | 6 +- Source/ablastr/utils/SignalHandling.cpp | 6 +- Source/ablastr/utils/msg_logger/MsgLogger.cpp | 42 +++-- Source/ablastr/warn_manager/WarnManager.cpp | 44 +++-- 112 files changed, 910 insertions(+), 709 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 9fada1bbdfe..1fb2f6e93c1 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -36,7 +36,6 @@ Checks: ' -performance-unnecessary-value-param, portability-*, readability-*, - -readability-braces-around-statements, -readability-convert-member-functions-to-static, -readability-else-after-return, -readability-function-cognitive-complexity, diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp index 9c8726a9c4b..21b1f164777 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp @@ -58,7 +58,7 @@ HardEdgedPlasmaLensDevice::InitHardEdgedPlasmaLensDevice (HardEdgedPlasmaLens co nelements = h_plasmalens.nelements; - if (nelements == 0) return; + if (nelements == 0) { return; } d_zs_arr = h_plasmalens.d_zs.data(); d_ze_arr = h_plasmalens.d_ze.data(); diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp index 51bcf7a2497..bcf50c55666 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp @@ -58,7 +58,7 @@ HardEdgedQuadrupoleDevice::InitHardEdgedQuadrupoleDevice (HardEdgedQuadrupole co nelements = h_quad.nelements; - if (nelements == 0) return; + if (nelements == 0) { return; } d_zs_arr = h_quad.d_zs.data(); d_ze_arr = h_quad.d_ze.data(); diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index b2016e65d2a..1f0b6f09a33 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -513,7 +513,7 @@ MultiSigmaBox::MultiSigmaBox (const BoxArray& ba, const DistributionMapping& dm, void MultiSigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) { - if (dt == dt_B) return; + if (dt == dt_B) { return; } dt_B = dt; @@ -529,7 +529,7 @@ MultiSigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) void MultiSigmaBox::ComputePMLFactorsE (const Real* dx, Real dt) { - if (dt == dt_E) return; + if (dt == dt_E) { return; } dt_E = dt; @@ -597,7 +597,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri auto nge = IntVect(AMREX_D_DECL(2, 2, 2)); auto ngb = IntVect(AMREX_D_DECL(2, 2, 2)); int ngf_int = 0; - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) ngf_int = std::max( ngf_int, 1 ); + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) { + ngf_int = std::max( ngf_int, 1 ); + } auto ngf = IntVect(AMREX_D_DECL(ngf_int, ngf_int, ngf_int)); if (do_moving_window) { @@ -1074,9 +1076,9 @@ void PML::Exchange (const std::array& mf_pml, const int do_pml_in_domain) { const amrex::Geometry& geom = (patch_type == PatchType::fine) ? *m_geom : *m_cgeom; - if (mf_pml[0] && mf[0]) Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); - if (mf_pml[1] && mf[1]) Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); - if (mf_pml[2] && mf[2]) Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); + if (mf_pml[0] && mf[0]) { Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); } + if (mf_pml[1] && mf[1]) { Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); } + if (mf_pml[2] && mf[2]) { Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); } } void diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 8f3c5a1b49b..af721d70b6d 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -51,13 +51,13 @@ void WarpX::DampPML (const int lev) { DampPML(lev, PatchType::fine); - if (lev > 0) DampPML(lev, PatchType::coarse); + if (lev > 0) { DampPML(lev, PatchType::coarse); } } void WarpX::DampPML (const int lev, PatchType patch_type) { - if (!do_pml) return; + if (!do_pml) { return; } WARPX_PROFILE("WarpX::DampPML()"); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) @@ -228,15 +228,15 @@ void WarpX::DampJPML (int lev) { DampJPML(lev, PatchType::fine); - if (lev > 0) DampJPML(lev, PatchType::coarse); + if (lev > 0) { DampJPML(lev, PatchType::coarse); } } void WarpX::DampJPML (int lev, PatchType patch_type) { - if (!do_pml) return; - if (!do_pml_j_damping) return; - if (!pml[lev]) return; + if (!do_pml) { return; } + if (!do_pml_j_damping) { return; } + if (!pml[lev]) { return; } WARPX_PROFILE("WarpX::DampJPML()"); diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index f0a1d406f95..0864c00067b 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -81,11 +81,12 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d applySilverMueller = true; } } - if(applySilverMueller) m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( + if(applySilverMueller) { m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( Efield_fp[lev], Bfield_fp[lev], Geom(lev).Domain(), dt[lev], WarpX::field_boundary_lo, WarpX::field_boundary_hi); + } } } @@ -105,14 +106,14 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) PEC::ApplyPECtoRhofield(rho, lev, patch_type); + if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoRhofield(rho, lev, patch_type); } } void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) PEC::ApplyPECtoJfield(Jx, Jy, Jz, lev, patch_type); + if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoJfield(Jx, Jy, Jz, lev, patch_type); } } #ifdef WARPX_DIM_RZ @@ -139,7 +140,7 @@ WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex const amrex::Real rmin = xyzmin[0]; // Skip blocks that don't touch the axis - if (rmin > 0._rt) continue; + if (rmin > 0._rt) { continue; } amrex::Array4 const& Er_arr = Er->array(mfi); amrex::Array4 const& Et_arr = Et->array(mfi); diff --git a/Source/BoundaryConditions/WarpX_PEC.H b/Source/BoundaryConditions/WarpX_PEC.H index ef3d3dc2ad5..62b92ba94b5 100644 --- a/Source/BoundaryConditions/WarpX_PEC.H +++ b/Source/BoundaryConditions/WarpX_PEC.H @@ -191,7 +191,7 @@ using namespace amrex; : (dom_hi[idim] + 1 - ig)); GuardCell = true; // tangential components are inverted across PEC boundary - if (is_tangent_to_PEC) sign *= -1._rt; + if (is_tangent_to_PEC) { sign *= -1._rt; } #if (defined WARPX_DIM_RZ) if (icomp == 0 && idim == 0 && iside == 1) { // Add radial scale so that drEr/dr = 0. @@ -329,7 +329,7 @@ using namespace amrex; : (dom_hi[idim] + 1 - ig)); GuardCell = true; // Sign of the normal component in guard cell is inverted - if (is_normal_to_PEC) sign *= -1._rt; + if (is_normal_to_PEC) { sign *= -1._rt; } #if (defined WARPX_DIM_RZ) if (icomp == 0 && idim == 0 && iside == 1) { // Add radial scale so that drBr/dr = 0. @@ -391,17 +391,17 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } // Get the mirror guard cell index amrex::IntVect iv_mirror = ijk_vec; iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; // On the PEC boundary the charge/current density is set to 0 - if (ijk_vec == iv_mirror) field(ijk_vec, n) = 0._rt; + if (ijk_vec == iv_mirror) { + field(ijk_vec, n) = 0._rt; // otherwise update the internal cell if the mirror guard cell exists - else if (fabbox.contains(iv_mirror)) - { + } else if (fabbox.contains(iv_mirror)) { field(ijk_vec,n) += psign[idim][iside] * field(iv_mirror,n); } } @@ -412,16 +412,17 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } amrex::IntVect iv_mirror = ijk_vec; iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; if (ijk_vec != iv_mirror && fabbox.contains(iv_mirror)) { - if (tangent_to_bndy[idim]) + if (tangent_to_bndy[idim]) { field(iv_mirror, n) = -field(ijk_vec, n); - else + } else { field(iv_mirror, n) = field(ijk_vec, n); + } } } } @@ -454,7 +455,7 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } // Get the mirror guard cell index amrex::IntVect iv_mirror = ijk_vec; @@ -464,7 +465,7 @@ using namespace amrex; // first value in the domain (nodal fields) if (ijk_vec == iv_mirror) { iv_mirror[idim] += (iside == 0) ? 1 : -1; - if (fabbox.contains(iv_mirror)) field(ijk_vec, n) = field(iv_mirror, n); + if (fabbox.contains(iv_mirror)) { field(ijk_vec, n) = field(iv_mirror, n); } } // otherwise set the mirror guard cell equal to the internal cell value else if (fabbox.contains(iv_mirror)) diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index 0b8d1c0fcc8..4692d8e980f 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -18,8 +18,8 @@ using namespace amrex::literals; bool PEC::isAnyBoundaryPEC() { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC) return true; - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC) return true; + if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC) { return true; } + if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC) { return true; } } return false; } @@ -250,8 +250,8 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } // rho values inside guard cells are updated the same as tangential // components of the current density @@ -276,7 +276,7 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& rho_array = rho->array(mfi); @@ -342,8 +342,8 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } for (int icomp=0; icomp < 3; ++icomp) { // Set the psign value correctly for each current component for each @@ -398,7 +398,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jx_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jx_array = Jx->array(mfi); @@ -433,7 +433,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jy_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jy_array = Jy->array(mfi); @@ -468,7 +468,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jz_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jz_array = Jz->array(mfi); @@ -520,8 +520,8 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } mirrorfac[idim][0] = 2*domain_lo[idim] - (1 - Pe_nodal[idim]); mirrorfac[idim][1] = 2*domain_hi[idim] + (1 - Pe_nodal[idim]); @@ -538,7 +538,7 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Pe_array = Pefield->array(mfi); diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp index 9f4976c20cb..8486a487f40 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp @@ -118,7 +118,7 @@ BTDPlotfileHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); @@ -241,7 +241,7 @@ BTDMultiFabHeaderImpl::ReadMultiFabHeader () m_minval[ifab].resize(m_ncomp); for (int icomp = 0; icomp < m_ncomp; ++icomp) { is >> m_minval[ifab][icomp] >> ch; - if( ch != ',' ) amrex::Error("Expected a ',' got something else"); + if( ch != ',' ) { amrex::Error("Expected a ',' got something else"); } } } ablastr::utils::text::goto_next_line(is); @@ -251,7 +251,7 @@ BTDMultiFabHeaderImpl::ReadMultiFabHeader () m_maxval[ifab].resize(m_ncomp); for (int icomp = 0; icomp < m_ncomp; ++icomp) { is >> m_maxval[ifab][icomp] >> ch; - if( ch != ',' ) amrex::Error("Expected a ',' got something else"); + if( ch != ',' ) { amrex::Error("Expected a ',' got something else"); } } } @@ -266,9 +266,9 @@ BTDMultiFabHeaderImpl::WriteMultiFabHeader () } std::ofstream FabHeaderFile; FabHeaderFile.open(m_Header_path.c_str(), std::ofstream::out | - std::ofstream::trunc | - std::ofstream::binary); - if ( !FabHeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + std::ofstream::trunc | + std::ofstream::binary); + if ( !FabHeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } FabHeaderFile.precision(17); @@ -421,7 +421,7 @@ BTDSpeciesHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); @@ -525,7 +525,7 @@ BTDParticleDataHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); m_ba.writeOn(HeaderFile); diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index dd8f17af3ad..4f83863b999 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -112,8 +112,9 @@ void BTDiagnostics::DerivedInitData () const amrex::ParmParse pp_diag_name(m_diag_name); int write_species = 1; pp_diag_name.query("write_species", write_species); - if ((m_output_species_names.empty()) && (write_species == 1)) + if ((m_output_species_names.empty()) && (write_species == 1)) { m_output_species_names = mpc.GetSpeciesNames(); + } m_do_back_transformed_particles = ((!m_output_species_names.empty()) && (write_species == 1)); @@ -233,7 +234,7 @@ BTDiagnostics::ReadParameters () pp_diag_name.query("do_back_transformed_fields", m_do_back_transformed_fields); pp_diag_name.query("do_back_transformed_particles", m_do_back_transformed_particles); AMREX_ALWAYS_ASSERT(m_do_back_transformed_fields or m_do_back_transformed_particles); - if (!m_do_back_transformed_fields) m_varnames.clear(); + if (!m_do_back_transformed_fields) { m_varnames.clear(); } std::vector intervals_string_vec = {"0"}; @@ -424,11 +425,11 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) // computed using the coarsened cell-size in the lab-frame obtained using // the ref_ratio at level, lev-1. auto ref_ratio = amrex::IntVect(1); - if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } // Number of lab-frame cells in z-direction at level, lev const int num_zcells_lab = static_cast( std::floor ( ( zmax_buffer_lab - zmin_buffer_lab) - / dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]) ) ); + / dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]))); // Take the max of 0 and num_zcells_lab const int Nz_lab = std::max( 0, num_zcells_lab ); #if (AMREX_SPACEDIM >= 2) @@ -502,7 +503,7 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) void BTDiagnostics::DefineCellCenteredMultiFab(int lev) { - if (!m_do_back_transformed_fields) return; + if (!m_do_back_transformed_fields) { return; } // Creating MultiFab to store cell-centered data in boosted-frame for the entire-domain // This MultiFab will store all the user-requested fields in the boosted-frame auto & warpx = WarpX::GetInstance(); @@ -524,7 +525,7 @@ void BTDiagnostics::InitializeFieldFunctors (int lev) { // Initialize fields functors only if do_back_transformed_fields is selected - if (!m_do_back_transformed_fields) return; + if (!m_do_back_transformed_fields) { return; } #ifdef WARPX_DIM_RZ // For RZ, initialize field functors RZ for openpmd @@ -611,16 +612,16 @@ BTDiagnostics::UpdateVarnamesForRZopenPMD () const auto m_varnames_fields_size = static_cast(m_varnames_fields.size()); for (int comp=0; comp(m_cellcenter_varnames_fields.size()); for (int comp=0; comp 0 ) ref_ratio = WarpX::RefRatio(lev-1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } const int k_lab = static_cast(std::floor ( ( m_current_z_lab[i_buffer] - (prob_domain_zmin_lab ) ) @@ -884,17 +886,16 @@ BTDiagnostics::k_index_zlab (int i_buffer, int lev) void BTDiagnostics::SetSnapshotFullStatus (const int i_buffer) { - if (m_snapshot_full[i_buffer] == 1) return; + if (m_snapshot_full[i_buffer] == 1) { return; } // if the last valid z-index of the snapshot, which is 0, is filled, then // set the snapshot full integer to 1 - if (m_lastValidZSlice[i_buffer] == 1) m_snapshot_full[i_buffer] = 1; - + if (m_lastValidZSlice[i_buffer] == 1) { m_snapshot_full[i_buffer] = 1; } } void BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) { - if (m_field_buffer_multifab_defined[i_buffer] == 1) return; + if (m_field_buffer_multifab_defined[i_buffer] == 1) { return; } auto & warpx = WarpX::GetInstance(); const int hi_k_lab = m_buffer_k_index_hi[i_buffer]; @@ -911,7 +912,7 @@ BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) m_mf_output[i_buffer][lev].setVal(0.); auto ref_ratio = amrex::IntVect(1); - if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const amrex::Real cellsize = (idim < WARPX_ZINDEX)? warpx.Geom(lev).CellSize(idim): @@ -954,7 +955,7 @@ BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) void BTDiagnostics::DefineSnapshotGeometry (const int i_buffer, const int lev) { - if (m_snapshot_geometry_defined[i_buffer] == 1) return; + if (m_snapshot_geometry_defined[i_buffer] == 1) { return; } if (lev == 0) { // Default non-periodic geometry for diags @@ -1151,17 +1152,20 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) // Create directory only when the first buffer is flushed out. if (m_buffer_flush_counter[i_snapshot] == 0 || m_first_flush_after_restart[i_snapshot] == 1) { // Create Level_0 directory to store all Cell_D and Cell_H files - if (!amrex::UtilCreateDirectory(snapshot_Level0_path, permission_flag_rwxrxrx) ) + if (!amrex::UtilCreateDirectory(snapshot_Level0_path, permission_flag_rwxrxrx) ) { amrex::CreateDirectoryFailed(snapshot_Level0_path); + } // Create directory for each species selected for diagnostic for (int i = 0; i < m_particles_buffer[i_snapshot].size(); ++i) { const std::string snapshot_species_path = snapshot_path + "/" + m_output_species_names[i]; - if ( !amrex::UtilCreateDirectory(snapshot_species_path, permission_flag_rwxrxrx)) + if ( !amrex::UtilCreateDirectory(snapshot_species_path, permission_flag_rwxrxrx)) { amrex::CreateDirectoryFailed(snapshot_species_path); + } // Create Level_0 directory for particles to store Particle_H and DATA files const std::string species_Level0_path = snapshot_species_path + "/Level_0"; - if ( !amrex::UtilCreateDirectory(species_Level0_path, permission_flag_rwxrxrx)) + if ( !amrex::UtilCreateDirectory(species_Level0_path, permission_flag_rwxrxrx)) { amrex::CreateDirectoryFailed(species_Level0_path); + } } const std::string buffer_WarpXHeader_path = recent_Buffer_filepath + "/WarpXHeader"; const std::string snapshot_WarpXHeader_path = snapshot_path + "/WarpXHeader"; @@ -1254,7 +1258,7 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( std::rename(recent_species_Header.c_str(), snapshot_species_Header.c_str()) == 0, std::string("Renaming ").append(recent_species_Header).append(" to ").append(snapshot_species_Header).append(" has failed")); - if (BufferSpeciesHeader.m_total_particles == 0) continue; + if (BufferSpeciesHeader.m_total_particles == 0) { continue; } // if finite number of particles in the output, copy ParticleHdr and Data file WARPX_ALWAYS_ASSERT_WITH_MESSAGE( std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, @@ -1265,7 +1269,7 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) } else { InterleaveSpeciesHeader(recent_species_Header,snapshot_species_Header, m_output_species_names[i], m_buffer_flush_counter[i_snapshot]); - if (BufferSpeciesHeader.m_total_particles == 0) continue; + if (BufferSpeciesHeader.m_total_particles == 0) { continue; } if (m_totalParticles_flushed_already[i_snapshot][i]==0) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index 9797b77ac1d..c85dbd6b226 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -142,7 +142,7 @@ BoundaryScrapingDiagnostics::Flush (int i_buffer, bool /* force_flush */) // If the saving of the particles was not set up for any of the species for this boundary // or if no particles have been lost, then don't write anything out. - if (n_particles == 0) return; + if (n_particles == 0) { return; } // This is not a backtransform diagnostics bool const isBTD = false; diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index f6f2bcd1205..22754f51a53 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -162,7 +162,7 @@ BackTransformFunctor::PrepareFunctorData (int i_buffer, m_current_z_boost[i_buffer] = current_z_boost; m_k_index_zlab[i_buffer] = k_index_zlab; m_perform_backtransform[i_buffer] = 0; - if (z_slice_in_domain && (snapshot_full == 0)) m_perform_backtransform[i_buffer] = 1; + if (z_slice_in_domain && (snapshot_full == 0)) { m_perform_backtransform[i_buffer] = 1; } } void diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp index ce463a11ca3..07804a4fa0c 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp @@ -37,7 +37,7 @@ LorentzTransformParticles::LorentzTransformParticles ( const WarpXParIter& a_pti { using namespace amrex::literals; - if (tmp_particle_data.empty()) return; + if (tmp_particle_data.empty()) { return; } m_get_position = GetParticlePosition(a_pti, a_offset); auto& attribs = a_pti.GetAttribs(); @@ -80,7 +80,7 @@ BackTransformParticleFunctor::BackTransformParticleFunctor ( void BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst, int &totalParticleCounter, int i_buffer) const { - if (m_perform_backtransform[i_buffer] == 0) return; + if (m_perform_backtransform[i_buffer] == 0) { return; } auto &warpx = WarpX::GetInstance(); // get particle slice const int nlevs = std::max(0, m_pc_src->finestLevel()+1); @@ -142,8 +142,9 @@ BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE(int i) { - if (Flag[i] == 1) GetParticleLorentzTransform(dst_data, src_data, i, + if (Flag[i] == 1) { GetParticleLorentzTransform(dst_data, src_data, i, old_size + IndexLocation[i]); + } }); amrex::Gpu::synchronize(); } @@ -171,5 +172,5 @@ BackTransformParticleFunctor::PrepareFunctorData ( int i_buffer, bool z_slice_in m_current_z_boost.at(i_buffer) = current_z_boost; m_t_lab.at(i_buffer) = t_lab; m_perform_backtransform.at(i_buffer) = 0; - if (z_slice_in_domain && (snapshot_full == 0)) m_perform_backtransform.at(i_buffer) = 1; + if (z_slice_in_domain && (snapshot_full == 0)) { m_perform_backtransform.at(i_buffer) = 1; } } diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp index 60b62181c4b..feb25c11183 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp @@ -135,8 +135,8 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, const amrex::ParticleReal uy = p.rdata(PIdx::uy) / PhysConst::c; const amrex::ParticleReal uz = p.rdata(PIdx::uz) / PhysConst::c; amrex::Real filter; - if ((do_filter) && (filter_fn(xw, yw, zw, ux, uy, uz) == 0._rt)) filter = 0._rt; - else filter = 1._rt; + if ((do_filter) && (filter_fn(xw, yw, zw, ux, uy, uz) == 0._rt)) { filter = 0._rt; + } else { filter = 1._rt; } amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 0), (amrex::Real)(p.rdata(PIdx::w) * filter)); }); // Divide value by number of particles for average. Set average to zero if there are no particles @@ -147,8 +147,8 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, amrex::Array4 const& a_ppc = ppc_mf.const_array(mfi); amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - if (a_ppc(i,j,k,0) == 0) a_red(i,j,k,0) = 0; - else a_red(i,j,k,0) = a_red(i,j,k,0) / a_ppc(i,j,k,0); + if (a_ppc(i,j,k,0) == 0) { a_red(i,j,k,0) = 0; + } else { a_red(i,j,k,0) = a_red(i,j,k,0) / a_ppc(i,j,k,0); } }); } } diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index 068a082bce2..3b5daabaffa 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -574,9 +574,7 @@ Diagnostics::FilterComputePackFlush (int step, bool force_flush) } for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { - if ( !DoDump (step, i_buffer, force_flush) ) continue; + if ( !DoDump (step, i_buffer, force_flush) ) { continue; } Flush(i_buffer, force_flush); } - - } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp index e15a7065f3a..3b7006243e7 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp @@ -27,8 +27,9 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) std::string openpmd_backend {"default"}; pp_diag_name.query("openpmd_backend", openpmd_backend); // pick first available backend if default is chosen - if( openpmd_backend == "default" ) + if( openpmd_backend == "default" ) { openpmd_backend = WarpXOpenPMDFileType(); + } pp_diag_name.add("openpmd_backend", openpmd_backend); @@ -37,12 +38,13 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) const bool encodingDefined = pp_diag_name.query("openpmd_encoding", openpmd_encoding); openPMD::IterationEncoding encoding = openPMD::IterationEncoding::groupBased; - if ( openpmd_encoding == "v" ) + if ( openpmd_encoding == "v" ) { encoding = openPMD::IterationEncoding::variableBased; - else if ( openpmd_encoding == "g" ) + } else if ( openpmd_encoding == "g" ) { encoding = openPMD::IterationEncoding::groupBased; - else if ( openpmd_encoding == "f" ) + } else if ( openpmd_encoding == "f" ) { encoding = openPMD::IterationEncoding::fileBased; + } std::string diag_type_str; pp_diag_name.get("diag_type", diag_type_str); @@ -65,8 +67,9 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) { bool openpmd_tspf = false; const bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf); - if ( tspfDefined && openpmd_tspf ) + if ( tspfDefined && openpmd_tspf ) { encoding = openPMD::IterationEncoding::fileBased; + } } // ADIOS2 operator type & parameters @@ -148,8 +151,9 @@ FlushFormatOpenPMD::WriteToFile ( int output_iteration = iteration[0]; // in backtransformed diagnostics (BTD), we dump into a series of labframe // snapshots - if( isBTD ) + if( isBTD ) { output_iteration = snapshotID; + } // Set step and output directory name. m_OpenPMDPlotWriter->SetStep(output_iteration, prefix, file_min_digits, isBTD); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index c2e372b1fb8..2c5edf72b12 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -86,7 +86,7 @@ FlushFormatPlotfile::WriteToFile ( Vector rfs; const VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); VisMF::SetHeaderVersion(amrex::VisMF::Header::Version_v1); - if (plot_raw_fields) rfs.emplace_back("raw_fields"); + if (plot_raw_fields) { rfs.emplace_back("raw_fields"); } amrex::WriteMultiLevelPlotfile(filename, nlev, amrex::GetVecOfConstPtrs(mf), varnames, geom, @@ -245,8 +245,9 @@ FlushFormatPlotfile::WriteWarpXHeader( HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if( ! HeaderFile.good()) + if( ! HeaderFile.good()) { amrex::FileOpenFailed(HeaderFileName); + } HeaderFile.precision(17); @@ -484,7 +485,7 @@ WriteCoarseVector( const std::string field_name, const int lev, const bool plot_guards ) { IntVect ng(0); - if (plot_guards) ng = Fx_fp->nGrowVect(); + if (plot_guards) { ng = Fx_fp->nGrowVect(); } if (lev == 0) { // No coarse field for level 0: instead write a MultiFab @@ -521,7 +522,7 @@ WriteCoarseScalar( const std::string field_name, const int icomp ) { IntVect ng(0); - if (plot_guards) ng = F_fp->nGrowVect(); + if (plot_guards) { ng = F_fp->nGrowVect(); } if (lev == 0) { // No coarse field for level 0: instead write a MultiFab @@ -544,7 +545,7 @@ FlushFormatPlotfile::WriteAllRawFields( const bool plot_raw_fields, const int nlevels, const std::string& plotfilename, const bool plot_raw_fields_guards) const { - if (!plot_raw_fields) return; + if (!plot_raw_fields) { return; } auto & warpx = WarpX::GetInstance(); for (int lev = 0; lev < nlevels; ++lev) { diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index c471beb05b3..4f1e47a2a52 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -147,7 +147,7 @@ FullDiagnostics::FlushRaw () {} bool FullDiagnostics::DoDump (int step, int /*i_buffer*/, bool force_flush) { - if (m_already_done) return false; + if (m_already_done) { return false; } if ( force_flush || (m_intervals.contains(step+1)) ){ m_already_done = true; return true; @@ -360,7 +360,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) { #ifdef WARPX_DIM_RZ - if (!m_dump_rz_modes) return; + if (!m_dump_rz_modes) { return; } auto & warpx = WarpX::GetInstance(); const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); @@ -507,11 +507,13 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { diag_dom.setLo(idim, std::max(m_lo[idim],warpx.Geom(lev).ProbLo(idim)) ); diag_dom.setHi(idim, std::min(m_hi[idim],warpx.Geom(lev).ProbHi(idim)) ); if ( std::fabs(warpx.Geom(lev).ProbLo(idim) - diag_dom.lo(idim)) - > warpx.Geom(lev).CellSize(idim) ) + > warpx.Geom(lev).CellSize(idim) ) { use_warpxba = false; + } if ( std::fabs(warpx.Geom(lev).ProbHi(idim) - diag_dom.hi(idim)) - > warpx.Geom(lev).CellSize(idim) ) + > warpx.Geom(lev).CellSize(idim) ) { use_warpxba = false; + } // User-defined value for coarsening should be an integer divisor of // blocking factor at level, lev. This assert is not relevant and thus @@ -573,7 +575,7 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { ba.coarsen(m_crse_ratio); // Generate a new distribution map if the physical m_lo and m_hi for the output // is different from the lo and hi physical co-ordinates of the simulation domain. - if (!use_warpxba) dmap = amrex::DistributionMapping{ba}; + if (!use_warpxba) { dmap = amrex::DistributionMapping{ba}; } // Allocate output MultiFab for diagnostics. The data will be stored at cell-centers. const int ngrow = (m_format == "sensei" || m_format == "ascent") ? 1 : 0; int const ncomp = static_cast(m_varnames.size()); diff --git a/Source/Diagnostics/MultiDiagnostics.cpp b/Source/Diagnostics/MultiDiagnostics.cpp index 5a50dfb6565..ea14919c713 100644 --- a/Source/Diagnostics/MultiDiagnostics.cpp +++ b/Source/Diagnostics/MultiDiagnostics.cpp @@ -70,9 +70,9 @@ MultiDiagnostics::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( diag_type_str == "Full" || diag_type_str == "BackTransformed" || diag_type_str == "BoundaryScraping", ".diag_type must be Full or BackTransformed or BoundaryScraping"); - if (diag_type_str == "Full") diags_types[i] = DiagTypes::Full; - if (diag_type_str == "BackTransformed") diags_types[i] = DiagTypes::BackTransformed; - if (diag_type_str == "BoundaryScraping") diags_types[i] = DiagTypes::BoundaryScraping; + if (diag_type_str == "Full") { diags_types[i] = DiagTypes::Full; } + if (diag_type_str == "BackTransformed") { diags_types[i] = DiagTypes::BackTransformed; } + if (diag_type_str == "BoundaryScraping") { diags_types[i] = DiagTypes::BoundaryScraping; } } } @@ -82,11 +82,13 @@ MultiDiagnostics::FilterComputePackFlush (int step, bool force_flush, bool BackT int i = 0; for (auto& diag : alldiags){ if (BackTransform) { - if (diags_types[i] == DiagTypes::BackTransformed) + if (diags_types[i] == DiagTypes::BackTransformed) { diag->FilterComputePackFlush (step, force_flush); + } } else { - if (diags_types[i] != DiagTypes::BackTransformed) + if (diags_types[i] != DiagTypes::BackTransformed) { diag->FilterComputePackFlush (step, force_flush); + } } ++i; } diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index 2a29748b3e1..d2a55a4671a 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -263,7 +263,7 @@ void BeamRelevant::ComputeDiags (int step) if (w_sum < std::numeric_limits::min() ) { - for (auto& item: m_data) item = 0.0_rt; + for (auto& item: m_data) { item = 0.0_rt; } return; } diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index b9698d5ffd2..fe2040e50d2 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -94,7 +94,7 @@ FieldMomentum::FieldMomentum (std::string rd_name) void FieldMomentum::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index c041a70aaef..b39b0e89eaf 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -628,14 +628,15 @@ void FieldProbe::ComputeDiags (int step) void FieldProbe::WriteToFile (int step) const { - if (!(ProbeInDomain() && amrex::ParallelDescriptor::IOProcessor())) return; + if (!(ProbeInDomain() && amrex::ParallelDescriptor::IOProcessor())) { return; } // loop over num valid particles to find the lowest particle ID for later sorting auto first_id = static_cast(m_data_out[0]); for (long int i = 0; i < m_valid_particles; i++) { - if (m_data_out[i*noutputs] < first_id) + if (m_data_out[i*noutputs] < first_id) { first_id = static_cast(m_data_out[i*noutputs]); + } } // Create a new array to store probe data ordered by id, which will be printed to file. diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 7843527cbf5..15115a15b08 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -282,7 +282,7 @@ void LoadBalanceCosts::WriteToFile (int step) const // get a reference to WarpX instance auto& warpx = WarpX::GetInstance(); - if (!ParallelDescriptor::IOProcessor()) return; + if (!ParallelDescriptor::IOProcessor()) { return; } // final step is a special case, fill jagged array with NaN if (m_intervals.nextContains(step+1) > warpx.maxStep()) @@ -347,7 +347,7 @@ void LoadBalanceCosts::WriteToFile (int step) const while (std::getline(ss, token, m_sep[0])) { cnt += 1; - if (ss.peek() == m_sep[0]) ss.ignore(); + if (ss.peek() == m_sep[0]) { ss.ignore(); } } // 2 columns for step, time; then nBoxes*nDatafields columns for data; diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp index a132c135471..157b4bfb368 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp @@ -88,7 +88,7 @@ ParticleEnergy::ParticleEnergy (std::string rd_name) void ParticleEnergy::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get MultiParticleContainer class object const auto & mypc = WarpX::GetInstance().GetPartContainer(); diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp index ecefd08eab7..512e6f1394a 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp @@ -158,7 +158,7 @@ ParticleHistogram::ParticleHistogram (std::string rd_name): void ParticleHistogram::ComputeDiags (int step) { // Judge if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); @@ -223,14 +223,16 @@ void ParticleHistogram::ComputeDiags (int step) auto const uz = d_uz[i] / PhysConst::c; // don't count a particle if it is filtered out - if (do_parser_filter) - if (fun_filterparser(t, x, y, z, ux, uy, uz) == 0._rt) + if (do_parser_filter) { + if (fun_filterparser(t, x, y, z, ux, uy, uz) == 0._rt) { return; + } + } // continue function if particle is not filtered out auto const f = fun_partparser(t, x, y, z, ux, uy, uz); // determine particle bin int const bin = int(Math::floor((f-bin_min)/bin_size)); - if ( bin<0 || bin>=num_bins ) return; // discard if out-of-range + if ( bin<0 || bin>=num_bins ) { return; } // discard if out-of-range // add particle to histogram bin //! @todo performance: on CPU, we are probably faster by @@ -260,11 +262,11 @@ void ParticleHistogram::ComputeDiags (int step) Real f_max = 0.0_rt; for ( int i = 0; i < m_bin_num; ++i ) { - if ( m_data[i] > f_max ) f_max = m_data[i]; + if ( m_data[i] > f_max ) { f_max = m_data[i]; } } for ( int i = 0; i < m_bin_num; ++i ) { - if ( f_max > std::numeric_limits::min() ) m_data[i] /= f_max; + if ( f_max > std::numeric_limits::min() ) { m_data[i] /= f_max; } } return; } @@ -279,7 +281,7 @@ void ParticleHistogram::ComputeDiags (int step) } for ( int i = 0; i < m_bin_num; ++i ) { - if ( f_area > std::numeric_limits::min() ) m_data[i] /= f_area; + if ( f_area > std::numeric_limits::min() ) { m_data[i] /= f_area; } } return; } diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp index dfb53b7e2c3..7cac712b4fc 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp @@ -61,8 +61,9 @@ ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) pp_rd_name.query("openpmd_backend", m_openpmd_backend); // pick first available backend if default is chosen - if( m_openpmd_backend == "default" ) + if( m_openpmd_backend == "default" ) { m_openpmd_backend = WarpXOpenPMDFileType(); + } pp_rd_name.add("openpmd_backend", m_openpmd_backend); // read species @@ -130,7 +131,7 @@ ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) void ParticleHistogram2D::ComputeDiags (int step) { // Judge if the diags should be done at this step - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // resize data array Array tlo{0,0}; // lower bounds @@ -215,9 +216,11 @@ void ParticleHistogram2D::ComputeDiags (int step) auto const uz = d_uz[i] / PhysConst::c; // don't count a particle if it is filtered out - if (do_parser_filter) - if(!static_cast(fun_filterparser(t, x, y, z, ux, uy, uz, w))) + if (do_parser_filter) { + if(!static_cast(fun_filterparser(t, x, y, z, ux, uy, uz, w))) { return; + } + } // continue function if particle is not filtered out auto const f_abs = fun_partparser_abs(t, x, y, z, ux, uy, uz, w); @@ -226,10 +229,10 @@ void ParticleHistogram2D::ComputeDiags (int step) // determine particle bin int const bin_abs = int(Math::floor((f_abs-bin_min_abs)/bin_size_abs)); - if ( bin_abs<0 || bin_abs>=num_bins_abs ) return; // discard if out-of-range + if ( bin_abs<0 || bin_abs>=num_bins_abs ) { return; } // discard if out-of-range int const bin_ord = int(Math::floor((f_ord-bin_min_ord)/bin_size_ord)); - if ( bin_ord<0 || bin_ord>=num_bins_ord ) return; // discard if out-of-range + if ( bin_ord<0 || bin_ord>=num_bins_ord ) { return; } // discard if out-of-range amrex::Real &data = d_table(bin_abs, bin_ord); amrex::HostDevice::Atomic::Add(&data, weight); diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp index 5b370d29979..5f278d48f14 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp @@ -116,7 +116,7 @@ ParticleMomentum::ParticleMomentum (std::string rd_name) void ParticleMomentum::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get MultiParticleContainer class object const auto & mypc = WarpX::GetInstance().GetPartContainer(); diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp index 758b48b7262..b4f617a8520 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp @@ -116,7 +116,7 @@ void ReducedDiags::WriteToFile (int step) const ofs << WarpX::GetInstance().gett_new(0); // loop over data size and write - for (const auto& item : m_data) ofs << m_sep << item; + for (const auto& item : m_data) { ofs << m_sep << item; } // end loop over data size diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 70471eb3910..71a585aea35 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -309,7 +309,7 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, slice_cc_nd_box.setLo( idim, slice_realbox.lo(idim) ); slice_cc_nd_box.setHi( idim, slice_realbox.hi(idim) ); - if ( slice_cr_ratio[idim] > 1) slice_cr_ratio[idim] = 1; + if ( slice_cr_ratio[idim] > 1) { slice_cr_ratio[idim] = 1; } // check for interpolation -- compute index lo with floor and ceil if ( slice_cc_nd_box.lo(idim) - real_box.lo(idim) >= fac ) { diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 2ae990851e2..91409bc294d 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -66,7 +66,7 @@ WarpX::GetRestartDMap (const std::string& chkfile, const amrex::BoxArray& ba, in ParallelDescriptor::ReadAndBcastFile(DMFileName, fileCharPtr); const std::string fileCharPtrString(fileCharPtr.dataPtr()); std::istringstream DMFile(fileCharPtrString, std::istringstream::in); - if ( ! DMFile.good()) amrex::FileOpenFailed(DMFileName); + if ( ! DMFile.good()) { amrex::FileOpenFailed(DMFileName); } DMFile.exceptions(std::ios_base::failbit | std::ios_base::badbit); int nprocs_in_checkpoint; @@ -382,11 +382,13 @@ WarpX::InitFromCheckpoint () if (do_pml) { for (int lev = 0; lev < nlevs; ++lev) { - if (pml[lev]) + if (pml[lev]) { pml[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml")); + } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) - if (pml_rz[lev]) + if (pml_rz[lev]) { pml_rz[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml_rz")); + } #endif } } diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index a375266b1d3..6d3b27160d4 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -100,8 +100,9 @@ namespace detail std::string const & engine_type, std::map< std::string, std::string > const & engine_parameters) { - if (operator_type.empty() && engine_type.empty()) + if (operator_type.empty() && engine_type.empty()) { return "{}"; + } std::string options; std::string top_block; @@ -111,7 +112,7 @@ namespace detail std::string op_parameters; for (const auto& kv : operator_parameters) { - if (!op_parameters.empty()) op_parameters.append(",\n"); + if (!op_parameters.empty()) { op_parameters.append(",\n"); } op_parameters.append(std::string(12, ' ')) /* just pretty alignment */ .append("\"").append(kv.first).append("\": ") /* key */ .append("\"").append(kv.second).append("\""); /* value (as string) */ @@ -119,7 +120,7 @@ namespace detail std::string en_parameters; for (const auto& kv : engine_parameters) { - if (!en_parameters.empty()) en_parameters.append(",\n"); + if (!en_parameters.empty()) { en_parameters.append(",\n"); } en_parameters.append(std::string(12, ' ')) /* just pretty alignment */ .append("\"").append(kv.first).append("\": ") /* key */ .append("\"").append(kv.second).append("\""); /* value (as string) */ @@ -154,8 +155,9 @@ namespace detail } ] })END"; - if (!engine_type.empty() || !en_parameters.empty()) + if (!engine_type.empty() || !en_parameters.empty()) { op_block += ","; + } } // end operator string block // add the engine string block @@ -170,8 +172,9 @@ namespace detail "type": ")END"; en_block += engine_type + "\""; - if(!en_parameters.empty()) + if(!en_parameters.empty()) { en_block += ","; + } } // non-default engine parameters @@ -304,33 +307,29 @@ namespace detail getUnitDimension ( std::string const & record_name ) { - if( (record_name == "position") || (record_name == "positionOffset") ) return { - {openPMD::UnitDimension::L, 1.} - }; - else if( record_name == "momentum" ) return { - {openPMD::UnitDimension::L, 1.}, - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::T, -1.} - }; - else if( record_name == "charge" ) return { - {openPMD::UnitDimension::T, 1.}, - {openPMD::UnitDimension::I, 1.} - }; - else if( record_name == "mass" ) return { - {openPMD::UnitDimension::M, 1.} - }; - else if( record_name == "E" ) return { - {openPMD::UnitDimension::L, 1.}, - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::T, -3.}, - {openPMD::UnitDimension::I, -1.}, - }; - else if( record_name == "B" ) return { - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::I, -1.}, - {openPMD::UnitDimension::T, -2.} - }; - else return {}; + if( (record_name == "position") || (record_name == "positionOffset") ) { + return {{openPMD::UnitDimension::L, 1.}}; + } else if( record_name == "momentum" ) { + return {{openPMD::UnitDimension::L, 1.}, + {openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::T, -1.}}; + } else if( record_name == "charge" ) { + return {{openPMD::UnitDimension::T, 1.}, + {openPMD::UnitDimension::I, 1.}}; + } else if( record_name == "mass" ) { + return {{openPMD::UnitDimension::M, 1.}}; + } else if( record_name == "E" ) { + return {{openPMD::UnitDimension::L, 1.}, + {openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::T, -3.}, + {openPMD::UnitDimension::I, -1.}}; + } else if( record_name == "B" ) { + return {{openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::I, -1.}, + {openPMD::UnitDimension::T, -2.}}; + } else { + return {}; + } } /** \brief For a given field that is to be written to an openPMD file, @@ -449,7 +448,7 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush) // default close is true bool callClose = true; // close BTD file only when isLastBTDFlush is true - if (isBTD and !isLastBTDFlush) callClose = false; + if (isBTD and !isLastBTDFlush) { callClose = false; } if (callClose) { if (m_Series) { GetIteration(m_CurrentStep, isBTD).close(); @@ -472,8 +471,9 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush) void WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) { - if( isBTD && m_Series != nullptr ) + if( isBTD && m_Series != nullptr ) { return; // already open for this snapshot (aka timestep in lab frame) + } // either for the next ts file, // or init a single file for all ts @@ -482,10 +482,11 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) // close a previously open series before creating a new one // see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686 - if ( m_Encoding == openPMD::IterationEncoding::fileBased ) + if ( m_Encoding == openPMD::IterationEncoding::fileBased ) { m_Series = nullptr; - else if ( m_Series != nullptr ) + } else if ( m_Series != nullptr ) { return; + } if (amrex::ParallelDescriptor::NProcs() > 1) { #if defined(AMREX_USE_MPI) @@ -504,8 +505,9 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) m_Series->setIterationEncoding( m_Encoding ); // input file / simulation setup author - if( !m_authors.empty()) + if( !m_authors.empty()) { m_Series->setAuthor( m_authors ); + } // more natural naming for PIC m_Series->setMeshesPath( "fields" ); // conform to ED-PIC extension of openPMD @@ -527,9 +529,11 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { WarpXParticleContainer* pc = particle_diags[i].getParticleContainer(); PinnedMemoryParticleContainer* pinned_pc = particle_diags[i].getPinnedParticleContainer(); - if (isBTD || use_pinned_pc) - if (!pinned_pc->isDefined()) + if (isBTD || use_pinned_pc) { + if (!pinned_pc->isDefined()) { continue; // Skip to the next particle container + } + } PinnedMemoryParticleContainer tmp = (isBTD || use_pinned_pc) ? pinned_pc->make_alike() : @@ -682,8 +686,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, // we will set up empty particles unless it's BTD, where we might add some in a following buffer dump // during this setup, we mark some particle properties as constant and potentially zero-sized bool doParticleSetup = true; - if (isBTD) + if (isBTD) { doParticleSetup = is_first_flush_with_particles || is_last_flush_and_never_particles; + } // this setup stage also implicitly calls "makeEmpty" if needed (i.e., is_last_flush_and_never_particles) // for BTD, we call this multiple times as we may resize in subsequent dumps if number of particles in the buffer > 0 @@ -705,7 +710,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++) { auto offset = static_cast( counter.m_ParticleOffsetAtRank[currentLevel] ); // For BTD, the offset include the number of particles already flushed - if (isBTD) offset += ParticleFlushOffset; + if (isBTD) { offset += ParticleFlushOffset; } for (ParticleIter pti(*pc, currentLevel); pti.isValid(); ++pti) { auto const numParticleOnTile = pti.numParticles(); auto const numParticleOnTile64 = static_cast( numParticleOnTile ); @@ -713,7 +718,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, // Do not call storeChunk() with zero-sized particle tiles: // https://github.com/openPMD/openPMD-api/issues/1147 // https://github.com/ECP-WarpX/WarpX/pull/1898#discussion_r745008290 - if (numParticleOnTile == 0) continue; + if (numParticleOnTile == 0) { continue; } contributed_particles = true; @@ -729,8 +734,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, new amrex::ParticleReal[numParticleOnTile], [](amrex::ParticleReal const *p) { delete[] p; } ); - for (auto i = 0; i < numParticleOnTile; i++) + for (auto i = 0; i < numParticleOnTile; i++) { z.get()[i] = aos[i].pos(1); // {0: "r", 1: "z"} + } std::string const positionComponent = "z"; currSpecies["position"]["z"].storeChunk(z, {offset}, {numParticleOnTile64}); } @@ -810,7 +816,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, if (is_resizing_flush && !contributed_particles && isBTD && m_Series->backend() == "ADIOS2") { for( auto & [record_name, record] : currSpecies ) { for( auto & [comp_name, comp] : record ) { - if (comp.constant()) continue; + if (comp.constant()) { continue; } auto dtype = comp.getDatatype(); switch (dtype) { @@ -860,7 +866,7 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, const unsigned long long np, bool const isBTD) const { std::string options = "{}"; - if (isBTD) options = "{ \"resizable\": true }"; + if (isBTD) { options = "{ \"resizable\": true }"; } auto dtype_real = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); auto dtype_int = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); // @@ -897,14 +903,16 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, [[maybe_unused]] const auto [_, newRecord] = addedRecords.insert(record_name); if( newRecord ) { currRecord.setUnitDimension( detail::getUnitDimension(record_name) ); - if( record_name == "weighting" ) + if( record_name == "weighting" ) { currRecord.setAttribute( "macroWeighted", 1u ); - else + } else { currRecord.setAttribute( "macroWeighted", 0u ); - if( record_name == "momentum" || record_name == "weighting" ) + } + if( record_name == "momentum" || record_name == "weighting" ) { currRecord.setAttribute( "weightingPower", 1.0 ); - else + } else { currRecord.setAttribute( "weightingPower", 0.0 ); + } } } } @@ -920,10 +928,11 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, if( newRecord ) { currRecord.setUnitDimension( detail::getUnitDimension(record_name) ); currRecord.setAttribute( "macroWeighted", 0u ); - if( record_name == "momentum" || record_name == "weighting" ) + if( record_name == "momentum" || record_name == "weighting" ) { currRecord.setAttribute( "weightingPower", 1.0 ); - else + } else { currRecord.setAttribute( "weightingPower", 0.0 ); + } } } } @@ -958,8 +967,9 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, [](amrex::ParticleReal const *p){ delete[] p; } ); - for( auto kk=0; kk(), {np}, options); auto idType = openPMD::Dataset(openPMD::determineDatatype< uint64_t >(), {np}, options); @@ -1147,17 +1157,20 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, const auto HalfFieldBoundarySize = static_cast(fieldBoundary.size() / 2u); - for (auto i = 0; i < HalfFieldBoundarySize; ++i) - if (m_fieldPMLdirections.at(i)) + for (auto i = 0; i < HalfFieldBoundarySize; ++i) { + if (m_fieldPMLdirections.at(i)) { fieldBoundary.at(i) = "open"; + } + } - for (int i = 0; i < HalfFieldBoundarySize; ++i) + for (int i = 0; i < HalfFieldBoundarySize; ++i) { if (period.isPeriodic(i)) { fieldBoundary.at(2u * i) = "periodic"; fieldBoundary.at(2u * i + 1u) = "periodic"; particleBoundary.at(2u * i) = "periodic"; particleBoundary.at(2u * i + 1u) = "periodic"; } + } meshes.setAttribute("fieldSolver", []() { switch (WarpX::electromagnetic_solver_id) { @@ -1174,10 +1187,10 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, meshes.setAttribute("fieldBoundary", fieldBoundary); meshes.setAttribute("particleBoundary", particleBoundary); meshes.setAttribute("currentSmoothing", []() { - if (WarpX::use_filter) return "Binomial"; - else return "none"; + if (WarpX::use_filter) { return "Binomial"; } + else { return "none"; } }()); - if (WarpX::use_filter) + if (WarpX::use_filter) { meshes.setAttribute("currentSmoothingParameters", []() { std::stringstream ss; ss << "period=1;compensator=false"; @@ -1195,12 +1208,14 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, std::string currentSmoothingParameters = ss.str(); return currentSmoothingParameters; }()); + } meshes.setAttribute("chargeCorrection", []() { - if (WarpX::do_dive_cleaning) return "hyperbolic"; // TODO or "spectral" or something? double-check - else return "none"; + if (WarpX::do_dive_cleaning) { return "hyperbolic"; // TODO or "spectral" or something? double-check + } else { return "none"; } }()); - if (WarpX::do_dive_cleaning) + if (WarpX::do_dive_cleaning) { meshes.setAttribute("chargeCorrectionParameters", "period=1"); + } } @@ -1279,8 +1294,9 @@ WarpXOpenPMDPlot::GetMeshCompNames (int meshLevel, } } - if ( 0 == meshLevel ) + if ( 0 == meshLevel ) { return; + } field_name += std::string("_lvl").append(std::to_string(meshLevel)); } @@ -1363,19 +1379,21 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename, } // If there are no fields to be written, interrupt the function here - if ( varnames.empty() ) return; + if ( varnames.empty() ) { return; } // loop over levels up to output_levels // note: this is usually the finestLevel, not the maxLevel for (int lev=0; lev < output_levels; lev++) { amrex::Geometry full_geom = geom[lev]; - if( isBTD ) + if( isBTD ) { full_geom = full_BTD_snapshot; + } // setup is called once. So it uses property "period" from first // geometry for field levels. - if ( (0 == lev) && first_write_to_iteration ) + if ( (0 == lev) && first_write_to_iteration ) { SetupFields(meshes, full_geom); + } amrex::Box const & global_box = full_geom.Domain(); @@ -1507,8 +1525,9 @@ WarpXParticleCounter::WarpXParticleCounter (ParticleContainer* pc): m_ParticleSizeAtRank[currentLevel] = numParticles; // adjust offset, it should be numbered after particles from previous levels - for (auto lv=0; lv(result.size()); for (int i=0; iFilterComputePackFlushLastTimestep( istep[0] ); - if (exit_loop_due_to_interrupt_signal) ExecutePythonCallback("onbreaksignal"); + if (exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } } } @@ -439,8 +440,8 @@ WarpX::OneStep_nosub (Real cur_time) // solve. // For extended PML: copy J from regular grid to PML, and damp J in PML - if (do_pml && pml_has_particles) CopyJPML(); - if (do_pml && do_pml_j_damping) DampJPML(); + if (do_pml && pml_has_particles) { CopyJPML(); } + if (do_pml && do_pml_j_damping) { DampJPML(); } ExecutePythonCallback("beforeEsolve"); @@ -467,10 +468,12 @@ WarpX::OneStep_nosub (Real cur_time) else { FillBoundaryE(guard_cells.ng_afterPushPSATD, WarpX::sync_nodal_points); FillBoundaryB(guard_cells.ng_afterPushPSATD, WarpX::sync_nodal_points); - if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) + if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) { FillBoundaryF(guard_cells.ng_alloc_F, WarpX::sync_nodal_points); - if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) + } + if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) { FillBoundaryG(guard_cells.ng_alloc_G, WarpX::sync_nodal_points); + } } } else { EvolveF(0.5_rt * dt[0], DtType::FirstHalf); @@ -506,8 +509,9 @@ WarpX::OneStep_nosub (Real cur_time) // E and B are up-to-date in the domain, but all guard cells are // outdated. - if (safe_guard_cells) + if (safe_guard_cells) { FillBoundaryB(guard_cells.ng_alloc_EB); + } } // !PSATD ExecutePythonCallback("afterEsolve"); @@ -549,7 +553,7 @@ void WarpX::SyncCurrentAndRho () { // TODO This works only without mesh refinement const int lev = 0; - if (use_filter) ApplyFilterJ(current_fp_vay, lev); + if (use_filter) { ApplyFilterJ(current_fp_vay, lev); } } } } @@ -603,11 +607,11 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // 1) Prepare E,B,F,G fields in spectral space PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::do_dive_cleaning) PSATDForwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDForwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } // 2) Set the averaged fields to zero - if (WarpX::fft_do_time_averaging) PSATDEraseAverageFields(); + if (WarpX::fft_do_time_averaging) { PSATDEraseAverageFields(); } // 3) Deposit rho (in rho_new, since it will be moved during the loop) // (after checking that pointer to rho_fp on MR level 0 is not null) @@ -650,7 +654,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) for (int i_deposit = 0; i_deposit < n_loop; i_deposit++) { // Move J from new to old if J is linear in time - if (J_in_time == JInTime::Linear) PSATDMoveJNewToJOld(); + if (J_in_time == JInTime::Linear) { PSATDMoveJNewToJOld(); } const amrex::Real t_deposit_current = (J_in_time == JInTime::Linear) ? (i_deposit-n_deposit+1)*sub_dt : (i_deposit-n_deposit+0.5_rt)*sub_dt; @@ -676,7 +680,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) if (rho_fp[0]) { // Move rho from new to old if rho is linear in time - if (rho_in_time == RhoInTime::Linear) PSATDMoveRhoNewToRhoOld(); + if (rho_in_time == RhoInTime::Linear) { PSATDMoveRhoNewToRhoOld(); } // Deposit rho at relative time t_deposit_charge mypc->DepositCharge(rho_fp, t_deposit_charge); @@ -701,8 +705,8 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) if (i_deposit == n_deposit-1) { PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::do_dive_cleaning) PSATDBackwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDBackwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } } } @@ -722,9 +726,9 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) pml[lev]->PushPSATD(lev); } ApplyEfieldBoundary(lev, PatchType::fine); - if (lev > 0) ApplyEfieldBoundary(lev, PatchType::coarse); + if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf); - if (lev > 0) ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); + if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); } } // Damp fields in PML before exchanging guard cells @@ -736,10 +740,12 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Exchange guard cells and synchronize nodal points FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) + if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) { FillBoundaryF(guard_cells.ng_alloc_F, WarpX::sync_nodal_points); - if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) + } + if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) { FillBoundaryG(guard_cells.ng_alloc_G, WarpX::sync_nodal_points); + } #else amrex::ignore_unused(cur_time); @@ -781,7 +787,7 @@ WarpX::OneStep_sub1 (Real curtime) PushParticlesandDeposit(fine_lev, curtime, DtType::FirstHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); - if (use_filter) ApplyFilterJ(current_fp, fine_lev); + if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, 2*ncomps); @@ -842,7 +848,7 @@ WarpX::OneStep_sub1 (Real curtime) PushParticlesandDeposit(fine_lev, curtime + dt[fine_lev], DtType::SecondHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); - if (use_filter) ApplyFilterJ(current_fp, fine_lev); + if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, ncomps); @@ -863,8 +869,9 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); } - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); + } FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); // v) Push the fields on the coarse patch and mother grid @@ -910,13 +917,15 @@ WarpX::OneStep_sub1 (Real curtime) WarpX::sync_nodal_points); } DampPML(coarse_lev, PatchType::fine); - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); + } } - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); + } } void @@ -1075,8 +1084,8 @@ WarpX::applyMirrors(Real time) NullifyMF(Bz, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_fp[lev]) NullifyMF(*F_fp[lev], lev, z_min, z_max); - if (G_fp[lev]) NullifyMF(*G_fp[lev], lev, z_min, z_max); + if (F_fp[lev]) { NullifyMF(*F_fp[lev], lev, z_min, z_max); } + if (G_fp[lev]) { NullifyMF(*G_fp[lev], lev, z_min, z_max); } if (lev>0) { @@ -1097,8 +1106,8 @@ WarpX::applyMirrors(Real time) NullifyMF(cBz, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_cp[lev]) NullifyMF(*F_cp[lev], lev, z_min, z_max); - if (G_cp[lev]) NullifyMF(*G_cp[lev], lev, z_min, z_max); + if (F_cp[lev]) { NullifyMF(*F_cp[lev], lev, z_min, z_max); } + if (G_cp[lev]) { NullifyMF(*G_cp[lev], lev, z_min, z_max); } } } } diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 6b20d7772d9..bd3086c0438 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -105,8 +105,9 @@ WarpX::AddBoundaryField () // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } // Allocate fields for charge and potential const int num_levels = max_level + 1; @@ -146,8 +147,9 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, @@ -181,7 +183,9 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) bool const local_average = false; // Average across all MPI ranks std::array beta_pr = pc.meanParticleVelocity(local_average); std::array beta; - for (int i=0 ; i < static_cast(beta.size()) ; i++) beta[i] = beta_pr[i]/PhysConst::c; // Normalize + for (int i=0 ; i < static_cast(beta.size()) ; i++) { + beta[i] = beta_pr[i]/PhysConst::c; // Normalize + } // Compute the potential phi, by solving the Poisson equation computePhi( rho, phi, beta, pc.self_fields_required_precision, @@ -201,8 +205,9 @@ WarpX::AddSpaceChargeFieldLabFrame () // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, @@ -381,7 +386,7 @@ void WarpX::setPhiBC ( amrex::Vector>& phi ) const { // check if any dimension has non-periodic boundary conditions - if (!m_poisson_boundary_handler.has_non_periodic) return; + if (!m_poisson_boundary_handler.has_non_periodic) { return; } // get the boundary potentials at the current time amrex::Array phi_bc_values_lo; @@ -417,7 +422,7 @@ WarpX::setPhiBC ( amrex::Vector>& phi ) const // loop over dimensions for (int idim=0; idim, 3> > std::array const beta ) const { // return early if beta is 0 since there will be no B-field - if ((beta[0] == 0._rt) && (beta[1] == 0._rt) && (beta[2] == 0._rt)) return; + if ((beta[0] == 0._rt) && (beta[1] == 0._rt) && (beta[2] == 0._rt)) { return; } for (int lev = 0; lev <= max_level; lev++) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp index b447d4e06a3..36be09696fd 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp @@ -106,13 +106,15 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // At the +z boundary (innermost guard cell) if ( apply_hi_z && (j==domain_box.bigEnd(1)+1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Br(i,j,0,m) = coef1_z*Br(i,j,0,m) - coef2_z*Et(i,j,0,m); + } } // At the -z boundary (innermost guard cell) if ( apply_lo_z && (j==domain_box.smallEnd(1)-1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Br(i,j,0,m) = coef1_z*Br(i,j,0,m) + coef2_z*Et(i,j+1,0,m); + } } }, @@ -120,13 +122,15 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // At the +z boundary (innermost guard cell) if ( apply_hi_z && (j==domain_box.bigEnd(1)+1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Bt(i,j,0,m) = coef1_z*Bt(i,j,0,m) + coef2_z*Er(i,j,0,m); + } } // At the -z boundary (innermost guard cell) if ( apply_lo_z && (j==domain_box.smallEnd(1)-1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Bt(i,j,0,m) = coef1_z*Bt(i,j,0,m) - coef2_z*Er(i,j+1,0,m); + } } // At the +r boundary (innermost guard cell) if ( apply_hi_r && (i==domain_box.bigEnd(0)+1) ){ @@ -233,31 +237,39 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #ifdef WARPX_DIM_3D // At the +y boundary (innermost guard cell) - if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) { Bx(i,j,k) = coef1_y * Bx(i,j,k) + coef2_y * Ez(i,j,k); + } // At the -y boundary (innermost guard cell) - if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) { Bx(i,j,k) = coef1_y * Bx(i,j,k) - coef2_y * Ez(i,j+1,k); + } // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) + if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) + if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i,j,k+1); + } #elif WARPX_DIM_XZ // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i,j+1,k); + } #elif WARPX_DIM_1D_Z // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i+1,j,k); + } #endif }, @@ -266,33 +278,41 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #if (defined WARPX_DIM_3D || WARPX_DIM_XZ) // At the +x boundary (innermost guard cell) - if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) { By(i,j,k) = coef1_x * By(i,j,k) - coef2_x * Ez(i,j,k); + } // At the -x boundary (innermost guard cell) - if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) { By(i,j,k) = coef1_x * By(i,j,k) + coef2_x * Ez(i+1,j,k); + } #endif #ifdef WARPX_DIM_3D // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) + if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) + if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i,j,k+1); + } #elif WARPX_DIM_XZ // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i,j+1,k); + } #elif WARPX_DIM_1D_Z // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i+1,j,k); + } #endif }, @@ -301,19 +321,23 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #if (defined WARPX_DIM_3D || WARPX_DIM_XZ) // At the +x boundary (innermost guard cell) - if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) { Bz(i,j,k) = coef1_x * Bz(i,j,k) + coef2_x * Ey(i,j,k); + } // At the -x boundary (innermost guard cell) - if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) { Bz(i,j,k) = coef1_x * Bz(i,j,k) - coef2_x * Ey(i+1,j,k); + } #endif #ifdef WARPX_DIM_3D // At the +y boundary (innermost guard cell) - if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) { Bz(i,j,k) = coef1_y * Bz(i,j,k) - coef2_y * Ex(i,j,k); + } // At the -y boundary (innermost guard cell) - if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) { Bz(i,j,k) = coef1_y * Bz(i,j,k) + coef2_y * Ex(i,j+1,k); + } #elif WARPX_DIM_1D_Z ignore_unused(i,j,k); #endif diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp index 8923ec1f6a5..1bbc6c9b337 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp @@ -36,8 +36,9 @@ FiniteDifferenceSolver::FiniteDifferenceSolver ( m_grid_type{grid_type} { // return if not FDTD - if (fdtd_algo == ElectromagneticSolverAlgo::None || fdtd_algo == ElectromagneticSolverAlgo::PSATD) + if (fdtd_algo == ElectromagneticSolverAlgo::None || fdtd_algo == ElectromagneticSolverAlgo::PSATD) { return; + } // Calculate coefficients of finite-difference stencil #ifdef WARPX_DIM_RZ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 57999335ae6..5a6f0018b2a 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -242,7 +242,7 @@ void HybridPICModel::InitData () void HybridPICModel::GetCurrentExternal ( amrex::Vector, 3>> const& edge_lengths) { - if (!m_external_field_has_time_dependence) return; + if (!m_external_field_has_time_dependence) { return; } auto& warpx = WarpX::GetInstance(); for (int lev = 0; lev <= warpx.finestLevel(); ++lev) @@ -414,7 +414,7 @@ void HybridPICModel::CalculateCurrentAmpere ( // the boundary correction was already applied to J_i and the B-field // boundary ensures that J itself complies with the boundary conditions, right? // ApplyJfieldBoundary(lev, Jfield[0].get(), Jfield[1].get(), Jfield[2].get()); - for (int i=0; i<3; i++) current_fp_ampere[lev][i]->FillBoundary(warpx.Geom(lev).periodicity()); + for (int i=0; i<3; i++) { current_fp_ampere[lev][i]->FillBoundary(warpx.Geom(lev).periodicity()); } } void HybridPICModel::HybridPICSolveE ( diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index 84d5b773097..1a72fee53c2 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -590,7 +590,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDr(Pe, coefs_r, n_coefs_r, i, j, 0, 0); @@ -601,7 +601,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Er(i, j, 0) = (enE_r - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Er(i, j, 0) += eta(rho_val) * Jr(i, j, 0); + if (include_resistivity_term) { Er(i, j, 0) += eta(rho_val) * Jr(i, j, 0); } }, // Et calculation @@ -623,7 +623,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure // -> d/dt = 0 for m = 0 @@ -635,20 +635,20 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Et(i, j, 0) = (enE_t - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Et(i, j, 0) += eta(rho_val) * Jt(i, j, 0); + if (include_resistivity_term) { Et(i, j, 0) += eta(rho_val) * Jt(i, j, 0); } }, // Ez calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries - if (lz(i,j,0) <= 0) return; + if (lz(i,j,0) <= 0) { return; } #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k, 0); @@ -659,7 +659,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); + if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); } } ); @@ -854,7 +854,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Real rho_val = Interp(rho, nodal, Ex_stag, coarsen, i, j, k, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDx(Pe, coefs_x, n_coefs_x, i, j, k); @@ -865,7 +865,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ex(i, j, k) = (enE_x - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ex(i, j, k) += eta(rho_val) * Jx(i, j, k); + if (include_resistivity_term) { Ex(i, j, k) += eta(rho_val) * Jx(i, j, k); } }, // Ey calculation @@ -873,18 +873,18 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries #ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; + if (ly(i,j,k) <= 0) { return; } #elif defined(WARPX_DIM_XZ) //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) return; + if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) { return; } #endif #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ey_stag, coarsen, i, j, k, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDy(Pe, coefs_y, n_coefs_y, i, j, k); @@ -895,20 +895,20 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ey(i, j, k) = (enE_y - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ey(i, j, k) += eta(rho_val) * Jy(i, j, k); + if (include_resistivity_term) { Ey(i, j, k) += eta(rho_val) * Jy(i, j, k); } }, // Ez calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) return; + if (lz(i,j,k) <= 0) { return; } #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k); @@ -919,7 +919,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); + if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); } } ); diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index bc51d65b536..192017656ce 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -206,7 +206,7 @@ void WarpX::setVectorPotentialBC ( amrex::Vector,3>>& A ) const { // check if any dimension has non-periodic boundary conditions - if (!m_vector_poisson_boundary_handler.has_non_periodic) return; + if (!m_vector_poisson_boundary_handler.has_non_periodic) { return; } auto dirichlet_flag = m_vector_poisson_boundary_handler.dirichlet_flag; @@ -228,7 +228,7 @@ WarpX::setVectorPotentialBC ( amrex::Vector= 2) - if (!is_nodal_x) spectral_field_value *= xshift_arr[i]; + if (!is_nodal_x) { spectral_field_value *= xshift_arr[i]; } #endif #if defined(WARPX_DIM_3D) - if (!is_nodal_y) spectral_field_value *= yshift_arr[j]; - if (!is_nodal_z) spectral_field_value *= zshift_arr[k]; + if (!is_nodal_y) { spectral_field_value *= yshift_arr[j]; } + if (!is_nodal_z) { spectral_field_value *= zshift_arr[k]; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } #elif defined(WARPX_DIM_1D_Z) - if (!is_nodal_z) spectral_field_value *= zshift_arr[i]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[i]; } #endif // Copy field into the right index fields_arr(i,j,k,field_index) = spectral_field_value; @@ -390,15 +390,15 @@ SpectralFieldData::BackwardTransform (const int lev, Complex spectral_field_value = field_arr(i,j,k,field_index); // Apply proper shift in each dimension #if (AMREX_SPACEDIM >= 2) - if (!is_nodal_x) spectral_field_value *= xshift_arr[i]; + if (!is_nodal_x) { spectral_field_value *= xshift_arr[i]; } #endif #if defined(WARPX_DIM_3D) - if (!is_nodal_y) spectral_field_value *= yshift_arr[j]; - if (!is_nodal_z) spectral_field_value *= zshift_arr[k]; + if (!is_nodal_y) { spectral_field_value *= yshift_arr[j]; } + if (!is_nodal_z) { spectral_field_value *= zshift_arr[k]; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } #elif defined(WARPX_DIM_1D_Z) - if (!is_nodal_z) spectral_field_value *= zshift_arr[i]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[i]; } #endif // Copy field into temporary array tmp_arr(i,j,k) = spectral_field_value; @@ -447,7 +447,7 @@ SpectralFieldData::BackwardTransform (const int lev, { for (int dir = 0; dir < AMREX_SPACEDIM; dir++) { - if ((fill_guards[dir]) == 0) mf_box.grow(dir, -mf_ng[dir]); + if ((fill_guards[dir]) == 0) { mf_box.grow(dir, -mf_ng[dir]); } } } diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp index 1268d277e63..c4a693bf25a 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp @@ -331,7 +331,7 @@ SpectralFieldDataRZ::FABZForwardTransform (amrex::MFIter const & mfi, amrex::Box [=] AMREX_GPU_DEVICE(int i, int j, int k, int mode) noexcept { Complex spectral_field_value = tmp_arr(i,j,k,mode); // Apply proper shift. - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } // Copy field into the correct index. int const ic = field_index + mode*n_fields; fields_arr(i,j,k,ic) = spectral_field_value*inv_nz; @@ -369,7 +369,7 @@ SpectralFieldDataRZ::FABZBackwardTransform (amrex::MFIter const & mfi, amrex::Bo int const ic = field_index + mode*n_fields; Complex spectral_field_value = fields_arr(i,j,k,ic); // Apply proper shift. - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } // Copy field into the right index. tmp_arr(i,j,k,mode) = spectral_field_value; }); diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp index c2309512596..39782408eb0 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp @@ -58,7 +58,7 @@ namespace{ q0 = static_cast(jn(n, p0)); q1 = static_cast(jn(n, p1)); for (int it=1; it <= nitmx; it++) { - if (q1 == q0) break; + if (q1 == q0) { break; } p = p1 - q1*(p1 - p0)/(q1 - q0); dp = p - p1; if (it > 1 && std::abs(dp) < tol) { @@ -142,7 +142,7 @@ void GetBesselRoots(int n, int nk, amrex::Vector& roots, amrex::Vec const auto errj = static_cast(std::abs(jn(n, zeroj))); // improve solution using procedure SecantRootFinder - if (errj > tol) ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); + if (errj > tol) { ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); } roots[ik] = zeroj; ier[ik] = ierror; diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index fa9d1860053..bb9ec04f266 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -183,11 +183,11 @@ WarpX::PSATDForwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { - if (F_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *F_fp[lev], Idx.F); + if (F_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *F_fp[lev], Idx.F); } if (spectral_solver_cp[lev]) { - if (F_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *F_cp[lev], Idx.F); + if (F_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *F_cp[lev], Idx.F); } } } } @@ -200,17 +200,17 @@ WarpX::PSATDBackwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { #ifdef WARPX_DIM_RZ - if (F_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F); + if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F); } #else - if (F_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F, m_fill_guards_fields); + if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F, m_fill_guards_fields); } #endif if (spectral_solver_cp[lev]) { #ifdef WARPX_DIM_RZ - if (F_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F); + if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F); } #else - if (F_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F, m_fill_guards_fields); + if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F, m_fill_guards_fields); } #endif } } @@ -229,11 +229,11 @@ WarpX::PSATDForwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (G_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *G_fp[lev], Idx.G); + if (G_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *G_fp[lev], Idx.G); } if (spectral_solver_cp[lev]) { - if (G_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *G_cp[lev], Idx.G); + if (G_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *G_cp[lev], Idx.G); } } } } @@ -246,17 +246,17 @@ WarpX::PSATDBackwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { #ifdef WARPX_DIM_RZ - if (G_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G); + if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G); } #else - if (G_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G, m_fill_guards_fields); + if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G, m_fill_guards_fields); } #endif if (spectral_solver_cp[lev]) { #ifdef WARPX_DIM_RZ - if (G_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G); + if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G); } #else - if (G_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G, m_fill_guards_fields); + if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G, m_fill_guards_fields); } #endif } } @@ -370,15 +370,15 @@ void WarpX::PSATDForwardTransformRho ( const amrex::Vector>& charge_cp, const int icomp, const int dcomp, const bool apply_kspace_filter) { - if (charge_fp[0] == nullptr) return; + if (charge_fp[0] == nullptr) { return; } for (int lev = 0; lev <= finest_level; ++lev) { - if (charge_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *charge_fp[lev], dcomp, icomp); + if (charge_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *charge_fp[lev], dcomp, icomp); } if (spectral_solver_cp[lev]) { - if (charge_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *charge_cp[lev], dcomp, icomp); + if (charge_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *charge_cp[lev], dcomp, icomp); } } } @@ -759,22 +759,23 @@ WarpX::PushPSATD () PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); #ifdef WARPX_DIM_RZ - if (pml_rz[0]) pml_rz[0]->PushPSATD(0); + if (pml_rz[0]) { pml_rz[0]->PushPSATD(0); } #endif // FFT of F and G - if (WarpX::do_dive_cleaning) PSATDForwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDForwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } // Update E, B, F, and G in k-space PSATDPushSpectralFields(); // Inverse FFT of E, B, F, and G PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::fft_do_time_averaging) + if (WarpX::fft_do_time_averaging) { PSATDBackwardTransformEBavg(Efield_avg_fp, Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp); - if (WarpX::do_dive_cleaning) PSATDBackwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDBackwardTransformG(); + } + if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } // Evolve the fields in the PML boxes for (int lev = 0; lev <= finest_level; ++lev) @@ -784,9 +785,9 @@ WarpX::PushPSATD () pml[lev]->PushPSATD(lev); } ApplyEfieldBoundary(lev, PatchType::fine); - if (lev > 0) ApplyEfieldBoundary(lev, PatchType::coarse); + if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf); - if (lev > 0) ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); + if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); } } #endif } @@ -916,7 +917,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) void WarpX::EvolveF (amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } for (int lev = 0; lev <= finest_level; ++lev) { @@ -927,16 +928,16 @@ WarpX::EvolveF (amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveF (int lev, amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } EvolveF(lev, PatchType::fine, a_dt, a_dt_type); - if (lev > 0) EvolveF(lev, PatchType::coarse, a_dt, a_dt_type); + if (lev > 0) { EvolveF(lev, PatchType::coarse, a_dt, a_dt_type); } } void WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } WARPX_PROFILE("WarpX::EvolveF()"); @@ -966,7 +967,7 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ void WarpX::EvolveG (amrex::Real a_dt, DtType a_dt_type) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } for (int lev = 0; lev <= finest_level; ++lev) { @@ -977,7 +978,7 @@ WarpX::EvolveG (amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveG (int lev, amrex::Real a_dt, DtType a_dt_type) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } EvolveG(lev, PatchType::fine, a_dt, a_dt_type); @@ -990,7 +991,7 @@ WarpX::EvolveG (int lev, amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_type*/) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } WARPX_PROFILE("WarpX::EvolveG()"); @@ -1076,8 +1077,8 @@ WarpX::DampFieldsInGuards(const int lev, { // Only apply to damped boundaries - if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) continue; - if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) continue; + if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) { continue; } + if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) { continue; } for ( amrex::MFIter mfi(*Efield[0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) { @@ -1170,8 +1171,8 @@ void WarpX::DampFieldsInGuards(const int lev, std::unique_ptr& for (int iside = 0; iside < 2; iside++) { // Only apply to damped boundaries - if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) continue; - if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) continue; + if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) { continue; } + if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) { continue; } for (amrex::MFIter mfi(*mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 00c2ddab73a..9252671d1ff 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -47,9 +47,11 @@ void WarpX::HybridPICEvolveFields () // SyncCurrent does not include a call to FillBoundary, but it is needed // for the hybrid-PIC solver since current values are interpolated to // a nodal grid - for (int lev = 0; lev <= finest_level; ++lev) - for (int idim = 0; idim < 3; ++idim) + for (int lev = 0; lev <= finest_level; ++lev) { + for (int idim = 0; idim < 3; ++idim) { current_fp[lev][idim]->FillBoundary(Geom(lev).periodicity()); + } + } // Get requested number of substeps to use int sub_steps = m_hybrid_pic_model->m_substeps / 2; diff --git a/Source/Filter/BilinearFilter.cpp b/Source/Filter/BilinearFilter.cpp index ecfee2cd7c9..a095bce6ae3 100644 --- a/Source/Filter/BilinearFilter.cpp +++ b/Source/Filter/BilinearFilter.cpp @@ -35,7 +35,7 @@ namespace { for(int ipass=1; ipass< lastpass; ipass++){ // element 0 has to be treated in its own way new_s.at(0) = 0.5_rt * old_s.at(0); - if (1(el); + } stencil_length_each_dir += 1.; #if defined(WARPX_DIM_3D) // npass_each_dir = npass_x npass_y npass_z diff --git a/Source/Fluids/MusclHancockUtils.H b/Source/Fluids/MusclHancockUtils.H index 764143a7220..2765f28f3b3 100644 --- a/Source/Fluids/MusclHancockUtils.H +++ b/Source/Fluids/MusclHancockUtils.H @@ -44,12 +44,13 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real minmod (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a > 0.0_rt && b > 0.0_rt) + if (a > 0.0_rt && b > 0.0_rt) { return std::min(a, b); - else if (a < 0.0_rt && b < 0.0_rt) + } else if (a < 0.0_rt && b < 0.0_rt) { return std::max(a, b); - else + } else { return 0.0_rt; + } } // Min of 3 inputs AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -68,24 +69,26 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real minmod3 (amrex::Real a, amrex::Real b , amrex::Real c) { using namespace amrex::literals; - if (a > 0.0_rt && b > 0.0_rt && c > 0.0_rt) + if (a > 0.0_rt && b > 0.0_rt && c > 0.0_rt) { return min3(a,b,c); - else if (a < 0.0_rt && b < 0.0_rt && c < 0.0_rt) + } else if (a < 0.0_rt && b < 0.0_rt && c < 0.0_rt) { return max3(a,b,c); - else + } else { return 0.0; + } } //maxmod AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real maxmod (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a > 0.0_rt && b > 0.0_rt) + if (a > 0.0_rt && b > 0.0_rt) { return std::max(a, b); - else if (a < 0.0_rt && b < 0.0_rt) + } else if (a < 0.0_rt && b < 0.0_rt) { return std::min(a, b); - else + } else { return 0.0_rt; + } } // Rusanov Flux (density) AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -132,30 +135,33 @@ amrex::Real ave_adjustable_diff (amrex::Real a, amrex::Real b) { using namespace amrex::literals; constexpr auto sigma = static_cast(2.0*0.732050807568877); - if (a*b > 0.0_rt) + if (a*b > 0.0_rt) { return minmod3( (a+b)/2.0_rt, sigma*a, sigma*b ); - else + } else { return 0.0_rt; + } } // ave_minmod Low diffusivity AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real ave (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a*b > 0.0_rt) + if (a*b > 0.0_rt) { return minmod3( (a+b)/2.0_rt, 2.0_rt*a, 2.0_rt*b ); - else + } else { return 0.0_rt; + } } // ave_superbee AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real ave_superbee (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a*b > 0.0_rt) + if (a*b > 0.0_rt) { return minmod( maxmod(a,b), minmod(2.0_rt*a,2.0_rt*b)); - else + } else { return 0.0_rt; + } } // stage2 slope limiting AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -379,7 +385,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) // U is zero if N is zero, Check positivity before dividing amrex::Real U_m = 0; - if (N(i-1,j,k) > 0) U_m = NU(i-1,j,k)/N(i-1,j,k); + if (N(i-1,j,k) > 0) { U_m = NU(i-1,j,k)/N(i-1,j,k); } return U - U_m; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -396,7 +402,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) // U is zero if N is zero, Check positivity before dividing amrex::Real U_p = 0; - if (N(i+1,j,k) > 0) U_p = NU(i+1,j,k)/N(i+1,j,k); + if (N(i+1,j,k) > 0) { U_p = NU(i+1,j,k)/N(i+1,j,k); } return U_p - U; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -414,7 +420,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) // U is zero if N is zero, Check positivity before dividing amrex::Real U_m = 0; - if (N(i,j-1,k) > 0) U_m = NU(i,j-1,k)/N(i,j-1,k); + if (N(i,j-1,k) > 0) { U_m = NU(i,j-1,k)/N(i,j-1,k); } return U - U_m; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -431,7 +437,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) // U is zero if N is zero, Check positivity before dividing amrex::Real U_p = 0; - if (N(i,j+1,k) > 0) U_p = NU(i,j+1,k)/N(i,j+1,k); + if (N(i,j+1,k) > 0) { U_p = NU(i,j+1,k)/N(i,j+1,k); } return U_p - U; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -450,11 +456,11 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) // U is zero if N is zero, Check positivity before dividing #if defined(WARPX_DIM_3D) - if (N(i,j,k-1) > 0) U_m = NU(i,j,k-1)/N(i,j,k-1); + if (N(i,j,k-1) > 0) { U_m = NU(i,j,k-1)/N(i,j,k-1); } #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - if (N(i,j-1,k) > 0) U_m = NU(i,j-1,k)/N(i,j-1,k); + if (N(i,j-1,k) > 0) { U_m = NU(i,j-1,k)/N(i,j-1,k); } #else - if (N(i-1,j,k) > 0) U_m = NU(i-1,j,k)/N(i-1,j,k); + if (N(i-1,j,k) > 0) { U_m = NU(i-1,j,k)/N(i-1,j,k); } #endif // Return the difference @@ -471,11 +477,11 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) // U is zero if N is zero, Check positivity before dividing #if defined(WARPX_DIM_3D) - if (N(i,j,k+1) > 0) U_p = NU(i,j,k+1)/N(i,j,k+1); + if (N(i,j,k+1) > 0) { U_p = NU(i,j,k+1)/N(i,j,k+1); } #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - if (N(i,j+1,k) > 0) U_p = NU(i,j+1,k)/N(i,j+1,k); + if (N(i,j+1,k) > 0) { U_p = NU(i,j+1,k)/N(i,j+1,k); } #else - if (N(i+1,j,k) > 0) U_p = NU(i+1,j,k)/N(i+1,j,k); + if (N(i+1,j,k) > 0) { U_p = NU(i+1,j,k)/N(i+1,j,k); } #endif // Return the difference diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index 32edcad23e7..1915e4bf772 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -821,10 +821,12 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) // Radial Surfaces amrex::Real S_Ar_plus = 2.0_rt*MathConst::pi*(r + dr/2.0_rt)*dz; amrex::Real S_Ar_minus = 2.0_rt*MathConst::pi*(r - dr/2.0_rt)*dz; - if (i == domain.smallEnd(0)) + if (i == domain.smallEnd(0)) { S_Ar_minus = 0.0_rt; - if (i == domain.bigEnd(0)+1) + } + if (i == domain.bigEnd(0)+1) { S_Ar_plus = 2.0_rt*MathConst::pi*(r)*dz; + } // Impose "none" boundaries // Condition: Vx(r) = 0 at boundaries @@ -1222,7 +1224,7 @@ void WarpXFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho, int icom amrex::ParallelFor(tile_box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - if ( owner_mask_rho_arr(i,j,k) ) rho_arr(i,j,k,icomp) += q*N_arr(i,j,k); + if ( owner_mask_rho_arr(i,j,k) ) { rho_arr(i,j,k,icomp) += q*N_arr(i,j,k); } } ); } @@ -1333,19 +1335,19 @@ void WarpXFluidContainer::DepositCurrent( { amrex::Real jx_tmp = ablastr::coarsen::sample::Interp(tmp_jx_fluid_arr, j_nodal_type, jx_type, coarsening_ratio, i, j, k, 0); - if ( owner_mask_x_arr(i,j,k) ) jx_arr(i, j, k) += jx_tmp; + if ( owner_mask_x_arr(i,j,k) ) { jx_arr(i, j, k) += jx_tmp; } }, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { amrex::Real jy_tmp = ablastr::coarsen::sample::Interp(tmp_jy_fluid_arr, j_nodal_type, jy_type, coarsening_ratio, i, j, k, 0); - if ( owner_mask_y_arr(i,j,k) ) jy_arr(i, j, k) += jy_tmp; + if ( owner_mask_y_arr(i,j,k) ) { jy_arr(i, j, k) += jy_tmp; } }, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { amrex::Real jz_tmp = ablastr::coarsen::sample::Interp(tmp_jz_fluid_arr, j_nodal_type, jz_type, coarsening_ratio, i, j, k, 0); - if ( owner_mask_z_arr(i,j,k) ) jz_arr(i, j, k) += jz_tmp; + if ( owner_mask_z_arr(i,j,k) ) { jz_arr(i, j, k) += jz_tmp; } } ); } diff --git a/Source/Initialization/ExternalField.cpp b/Source/Initialization/ExternalField.cpp index 04d34c59a32..909e5c01977 100644 --- a/Source/Initialization/ExternalField.cpp +++ b/Source/Initialization/ExternalField.cpp @@ -76,15 +76,17 @@ ExternalFieldParams::ExternalFieldParams(const amrex::ParmParse& pp_warpx) // if the input string is "constant", the values for the // external grid must be provided in the input. auto v_B = std::vector(3); - if (B_ext_grid_type == ExternalFieldType::constant) + if (B_ext_grid_type == ExternalFieldType::constant) { utils::parser::getArrWithParser(pp_warpx, "B_external_grid", v_B); + } std::copy(v_B.begin(), v_B.end(), B_external_grid.begin()); // if the input string is "constant", the values for the // external grid must be provided in the input. auto v_E = std::vector(3); - if (E_ext_grid_type == ExternalFieldType::constant) + if (E_ext_grid_type == ExternalFieldType::constant) { utils::parser::getArrWithParser(pp_warpx, "E_external_grid", v_E); + } std::copy(v_E.begin(), v_E.end(), E_external_grid.begin()); //___________________________________________________________________________ diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index 98caae1bb23..f7ee6cff138 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -130,7 +130,7 @@ namespace { u = approx_u_th * std::sqrt(2._rt*std::log(1._rt/xrand)); // Rejection method xrand = amrex::Random(engine); - if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) reject = false; + if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) { reject = false; } } } else { // Mean velocity magnitude is greater than thermal velocity @@ -151,7 +151,7 @@ namespace { } // Rejection method const amrex::Real xrand = amrex::Random(engine); - if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) reject = false; + if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) { reject = false; } } } @@ -197,7 +197,7 @@ struct InjectorMomentumGaussianFlux u_th = m_uz_th; } amrex::Real u = generateGaussianFluxDist(u_m, u_th, engine); - if (m_flux_direction < 0) u = -u; + if (m_flux_direction < 0) { u = -u; } // Note: Here, in RZ geometry, the variables `ux` and `uy` actually // correspond to the radial and azimuthal component of the momentum @@ -338,7 +338,7 @@ struct InjectorMomentumBoltzmann { using namespace amrex::literals; amrex::Real u[3]; - for (auto& el : u) el = 0.0_rt; + for (auto& el : u) { el = 0.0_rt; } const amrex::Real beta = velocity(x,y,z); int const dir = velocity.direction(); const auto gamma = 1._rt/std::sqrt(1._rt-beta*beta); @@ -441,7 +441,7 @@ struct InjectorMomentumJuttner { using namespace amrex::literals; amrex::Real u[3]; - for (auto& el : u) el = 0.0_rt; + for (auto& el : u) { el = 0.0_rt; } amrex::Real const beta = velocity(x,y,z); int const dir = velocity.direction(); auto const gamma = 1._rt/std::sqrt(1._rt-beta*beta); diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index 64cc1a2a620..c86ed432bd1 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -43,19 +43,19 @@ struct InjectorPositionRandomPlane using namespace amrex::literals; #if ((defined WARPX_DIM_3D) || (defined WARPX_DIM_RZ)) // In RZ, the 3 components of the `XDim3` vector below correspond to r, theta, z respectively - if (dir == 0) return amrex::XDim3{0._rt, amrex::Random(engine), amrex::Random(engine)}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), 0._rt, amrex::Random(engine)}; - else return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; + if (dir == 0) { return amrex::XDim3{0._rt, amrex::Random(engine), amrex::Random(engine)}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), 0._rt, amrex::Random(engine)}; } + else { return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; } #elif (defined(WARPX_DIM_XZ)) // In 2D, the 2 first components of the `XDim3` vector below correspond to x and z - if (dir == 0) return amrex::XDim3{0._rt, amrex::Random(engine), 0._rt}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; - else return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; + if (dir == 0) { return amrex::XDim3{0._rt, amrex::Random(engine), 0._rt}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; } + else { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt }; } #elif (defined(WARPX_DIM_1D_Z)) // In 2D, the first components of the `XDim3` vector below correspond to z - if (dir == 0) return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; - else return amrex::XDim3{0._rt, 0._rt, 0._rt}; + if (dir == 0) { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; } + else { return amrex::XDim3{0._rt, 0._rt, 0._rt}; } #endif } private: diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index bd8e6334f48..edf083fd839 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -476,8 +476,9 @@ WarpX::InitData () ExecutePythonCallback("beforeInitEsolve"); ComputeSpaceChargeField(reset_fields); ExecutePythonCallback("afterInitEsolve"); - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) + if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { ComputeMagnetostaticField(); + } // Set up an invariant condition through the rest of // execution, that any code besides the field solver that @@ -552,7 +553,7 @@ WarpX::InitPML () do_pml_Hi[0][idim] = 1; // on level 0 } } - if (finest_level > 0) do_pml = 1; + if (finest_level > 0) { do_pml = 1; } if (do_pml) { #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) @@ -583,10 +584,12 @@ WarpX::InitPML () // Domain box at level, lev const amrex::Box DomainBox = Geom(lev).Domain(); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (levelBox.smallEnd(idim) == DomainBox.smallEnd(idim)) + if (levelBox.smallEnd(idim) == DomainBox.smallEnd(idim)) { do_pml_Lo[lev][idim] = do_pml_Lo[0][idim]; - if (levelBox.bigEnd(idim) == DomainBox.bigEnd(idim)) + } + if (levelBox.bigEnd(idim) == DomainBox.bigEnd(idim)) { do_pml_Hi[lev][idim] = do_pml_Hi[0][idim]; + } } #ifdef WARPX_DIM_RZ @@ -619,8 +622,9 @@ WarpX::ComputePMLFactors () { for (int lev = 0; lev <= finest_level; ++lev) { - if (pml[lev]) + if (pml[lev]) { pml[lev]->ComputePMLFactors(dt[lev]); + } } } } @@ -745,14 +749,16 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ( is_B_ext_const && (lev <= maxlevel_extEMfield_init) ) { Bfield_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); - if (fft_do_time_averaging) + if (fft_do_time_averaging) { Bfield_avg_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + } if (lev > 0) { Bfield_aux[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); Bfield_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); - if (fft_do_time_averaging) + if (fft_do_time_averaging) { Bfield_avg_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + } } } @@ -764,14 +770,16 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ( is_E_ext_const && (lev <= maxlevel_extEMfield_init) ) { Efield_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); - if (fft_do_time_averaging) + if (fft_do_time_averaging) { Efield_avg_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + } if (lev > 0) { Efield_aux[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); Efield_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); - if (fft_do_time_averaging) + if (fft_do_time_averaging) { Efield_avg_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + } } } } diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp index e29439c4bc0..46b47e119af 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp @@ -114,8 +114,9 @@ void WarpXLaserProfiles::FromFileLaserProfile::update (amrex::Real t) { t += m_params.t_min - m_params.t_delay; - if(t >= m_params.t_max) + if(t >= m_params.t_max) { return; + } const auto idx_times = find_left_right_time_indices(t); const auto idx_t_left = idx_times.first; const auto idx_t_right = idx_times.second; @@ -186,8 +187,8 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(std::string lasy_file_ m_params.n_rz_azimuthal_components = static_cast(extent[0]); m_params.nt = static_cast(extent[1]); m_params.nr = static_cast(extent[2]); - if(m_params.nt <= 1) WARPX_ABORT_WITH_MESSAGE("nt in lasy file must be >=2"); - if(m_params.nr <= 1) WARPX_ABORT_WITH_MESSAGE("nr in lasy file must be >=2"); + if(m_params.nt <= 1) { WARPX_ABORT_WITH_MESSAGE("nt in lasy file must be >=2"); } + if(m_params.nr <= 1) { WARPX_ABORT_WITH_MESSAGE("nr in lasy file must be >=2"); } // Calculate the min and max of the grid m_params.t_min = static_cast(offset[0] + position[0]*spacing[0]); m_params.t_max = static_cast(m_params.t_min + (m_params.nt-1)*spacing[0]); @@ -240,24 +241,24 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (std::string binary_ { if(ParallelDescriptor::IOProcessor()){ std::ifstream inp(binary_file_name, std::ios::binary); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); } inp.exceptions(std::ios_base::failbit | std::ios_base::badbit); //Uniform grid flag char flag; inp.read(&flag, 1); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read grid type from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read grid type from binary file"); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flag, "Binary files with non uniform grid are no longer supported"); //Grid points along t, x and y inp.read(reinterpret_cast(&m_params.nt), sizeof(uint32_t)); inp.read(reinterpret_cast(&m_params.nx), sizeof(uint32_t)); inp.read(reinterpret_cast(&m_params.ny), sizeof(uint32_t)); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read sizes from binary file"); - if(m_params.nt <= 1) WARPX_ABORT_WITH_MESSAGE("nt in binary file must be >=2"); - if(m_params.nx <= 1) WARPX_ABORT_WITH_MESSAGE("nx in binary file must be >=2"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read sizes from binary file"); } + if(m_params.nt <= 1) { WARPX_ABORT_WITH_MESSAGE("nt in binary file must be >=2"); } + if(m_params.nx <= 1) { WARPX_ABORT_WITH_MESSAGE("nx in binary file must be >=2"); } #if (defined(WARPX_DIM_3D) || (defined WARPX_DIM_RZ)) - if(m_params.ny <= 1) WARPX_ABORT_WITH_MESSAGE("ny in binary file must be >=2 in 3D"); + if(m_params.ny <= 1) { WARPX_ABORT_WITH_MESSAGE("ny in binary file must be >=2 in 3D"); } #elif defined(WARPX_DIM_XZ) - if(m_params.ny != 1) WARPX_ABORT_WITH_MESSAGE("ny in binary file must be 1 in 2D"); + if(m_params.ny != 1) { WARPX_ABORT_WITH_MESSAGE("ny in binary file must be 1 in 2D"); } #endif //Coordinates Vector dbuf_t, dbuf_x, dbuf_y; @@ -274,7 +275,7 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (std::string binary_ static_cast(dbuf_x.size()*sizeof(double))); inp.read(reinterpret_cast(dbuf_y.dataPtr()), static_cast(dbuf_y.size()*sizeof(double))); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read coords from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read coords from binary file"); } m_params.t_min = static_cast(dbuf_t[0]); m_params.t_max = static_cast(dbuf_t[1]); @@ -383,7 +384,7 @@ WarpXLaserProfiles::FromFileLaserProfile::read_binary_data_t_chunk (int t_begin, if(ParallelDescriptor::IOProcessor()){ //Read data chunk std::ifstream inp(m_params.binary_file_name, std::ios::binary); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); } inp.exceptions(std::ios_base::failbit | std::ios_base::badbit); #if (defined(WARPX_DIM_3D)) auto skip_amount = 1 + @@ -401,12 +402,12 @@ WarpXLaserProfiles::FromFileLaserProfile::read_binary_data_t_chunk (int t_begin, sizeof(double)*t_begin*m_params.nx*m_params.ny; #endif inp.seekg(static_cast(skip_amount)); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); } const int read_size = (i_last - i_first + 1)* m_params.nx*m_params.ny; Vector buf_e(read_size); inp.read(reinterpret_cast(buf_e.dataPtr()), static_cast(read_size*sizeof(double))); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); } std::transform(buf_e.begin(), buf_e.end(), h_E_binary_data.begin(), [](auto x) {return static_cast(x);} ); } diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp index ef00d95fd43..96d61d920f1 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp @@ -73,10 +73,11 @@ WarpXLaserProfiles::GaussianLaserProfile::init ( m_params.stc_direction[1]*m_common_params.p_X[1] + m_params.stc_direction[2]*m_common_params.p_X[2]; - if (arg < -1.0_rt || arg > 1.0_rt) + if (arg < -1.0_rt || arg > 1.0_rt) { m_params.theta_stc = 0._rt; - else + } else { m_params.theta_stc = std::acos(arg); + } #else m_params.theta_stc = 0.; #endif diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index c1ecc5db34c..ae5ab839d10 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -172,7 +172,7 @@ guardCellManager::Init ( // After pushing particle int ng_alloc_F_int = (do_moving_window) ? 2 : 0; // CKC solver requires one additional guard cell - if (electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) ng_alloc_F_int = std::max( ng_alloc_F_int, 1 ); + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) { ng_alloc_F_int = std::max( ng_alloc_F_int, 1 ); } ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int)); // Used if warpx.do_divb_cleaning = 1 @@ -314,8 +314,9 @@ guardCellManager::Init ( // If NCI filter, add guard cells in the z direction IntVect ng_NCIFilter = IntVect::TheZeroVector(); - if (do_fdtd_nci_corr) + if (do_fdtd_nci_corr) { ng_NCIFilter[WARPX_ZINDEX] = NCIGodfreyFilter::m_stencil_width; + } // Note: communications of guard cells for bilinear filter are handled // separately. diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 79c624e212a..7d5b25b1560 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -551,7 +551,7 @@ void WarpX::FillBoundaryE (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryE(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryE(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -608,7 +608,7 @@ void WarpX::FillBoundaryB (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryB(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryB(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -665,7 +665,7 @@ void WarpX::FillBoundaryE_avg(int lev, IntVect ng) { FillBoundaryE_avg(lev, PatchType::fine, ng); - if (lev > 0) FillBoundaryE_avg(lev, PatchType::coarse, ng); + if (lev > 0) { FillBoundaryE_avg(lev, PatchType::coarse, ng); } } void @@ -719,7 +719,7 @@ void WarpX::FillBoundaryB_avg (int lev, IntVect ng) { FillBoundaryB_avg(lev, PatchType::fine, ng); - if (lev > 0) FillBoundaryB_avg(lev, PatchType::coarse, ng); + if (lev > 0) { FillBoundaryB_avg(lev, PatchType::coarse, ng); } } void @@ -770,7 +770,7 @@ void WarpX::FillBoundaryF (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryF(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryF(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryF(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -780,7 +780,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (F_fp[lev]) pml[lev]->ExchangeF(patch_type, F_fp[lev].get(), do_pml_in_domain); + if (F_fp[lev]) { pml[lev]->ExchangeF(patch_type, F_fp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryF(patch_type, nodal_sync); } @@ -795,7 +795,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (F_cp[lev]) pml[lev]->ExchangeF(patch_type, F_cp[lev].get(), do_pml_in_domain); + if (F_cp[lev]) { pml[lev]->ExchangeF(patch_type, F_cp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryF(patch_type, nodal_sync); } @@ -824,7 +824,7 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_fp[lev]) pml[lev]->ExchangeG(patch_type, G_fp[lev].get(), do_pml_in_domain); + if (G_fp[lev]) { pml[lev]->ExchangeG(patch_type, G_fp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } @@ -839,7 +839,7 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_cp[lev]) pml[lev]->ExchangeG(patch_type, G_cp[lev].get(), do_pml_in_domain); + if (G_cp[lev]) { pml[lev]->ExchangeG(patch_type, G_cp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } @@ -1048,7 +1048,7 @@ WarpX::SyncRho ( { WARPX_PROFILE("WarpX::SyncRho()"); - if (!charge_fp[0]) return; + if (!charge_fp[0]) { return; } const int ncomp = charge_fp[0]->nComp(); // See comments in WarpX::SyncCurrent for an explanation of the algorithm. @@ -1330,7 +1330,7 @@ void WarpX::ApplyFilterandSumBoundaryRho ( const int glev = (patch_type == PatchType::fine) ? lev : lev-1; const std::unique_ptr& rho = (patch_type == PatchType::fine) ? charge_fp[lev] : charge_cp[lev]; - if (rho == nullptr) return; + if (rho == nullptr) { return; } ApplyFilterandSumBoundaryRho(lev, glev, *rho, icomp, ncomp); } @@ -1373,7 +1373,7 @@ void WarpX::AddRhoFromFineLevelandSumBoundary ( const int icomp, const int ncomp) { - if (!charge_fp[lev]) return; + if (!charge_fp[lev]) { return; } ApplyFilterandSumBoundaryRho(charge_fp, charge_cp, lev, PatchType::fine, icomp, ncomp); @@ -1452,7 +1452,7 @@ void WarpX::NodalSyncJ ( const int lev, PatchType patch_type) { - if (!override_sync_intervals.contains(istep[0])) return; + if (!override_sync_intervals.contains(istep[0])) { return; } if (patch_type == PatchType::fine) { @@ -1478,7 +1478,7 @@ void WarpX::NodalSyncRho ( const int icomp, const int ncomp) { - if (!override_sync_intervals.contains(istep[0])) return; + if (!override_sync_intervals.contains(istep[0])) { return; } if (patch_type == PatchType::fine && charge_fp[lev]) { diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 5cd5be168a9..dcea3cab58a 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -151,11 +151,11 @@ template void RemakeMultiFab (std::unique_ptr& mf, const DistributionMapping& dm, const bool redistribute, const int lev) { - if (mf == nullptr) return; + if (mf == nullptr) { return; } const IntVect& ng = mf->nGrowVect(); std::unique_ptr pmf; WarpX::AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); - if (redistribute) pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); + if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } mf = std::move(pmf); } @@ -164,7 +164,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi { if (ba == boxArray(lev)) { - if (ParallelDescriptor::NProcs() == 1) return; + if (ParallelDescriptor::NProcs() == 1) { return; } // Fine patch for (int idim=0; idim < 3; ++idim) @@ -335,8 +335,9 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi RemakeMultiFab(current_buffer_masks[lev], dm, false ,lev); RemakeMultiFab(gather_buffer_masks[lev], dm, false ,lev); - if (current_buffer_masks[lev] || gather_buffer_masks[lev]) + if (current_buffer_masks[lev] || gather_buffer_masks[lev]) { BuildBufferMasks(); + } } // Re-initialize the lattice element finder with the new ba and dm. diff --git a/Source/Parallelization/WarpXSumGuardCells.H b/Source/Parallelization/WarpXSumGuardCells.H index 425ce320856..13d6dce4af2 100644 --- a/Source/Parallelization/WarpXSumGuardCells.H +++ b/Source/Parallelization/WarpXSumGuardCells.H @@ -34,10 +34,11 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, amrex::IntVect n_updated_guards; // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { n_updated_guards = mf.nGrowVect(); - else // Update only the valid cells + } else { // Update only the valid cells n_updated_guards = amrex::IntVect::TheZeroVector(); + } ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); } @@ -65,10 +66,11 @@ WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, amrex::IntVect n_updated_guards; // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { n_updated_guards = dst.nGrowVect(); - else // Update only the valid cells + } else { // Update only the valid cells n_updated_guards = amrex::IntVect::TheZeroVector(); + } dst.setVal(0., icomp, ncomp, n_updated_guards); // ablastr::utils::communication::ParallelAdd(dst, src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index ebf8cb47172..84c50aff674 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -352,7 +352,7 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile [=] AMREX_GPU_HOST_DEVICE (long ip, amrex::RandomEngine const& engine) { // determine if this particle should collide - if (amrex::Random(engine) > total_collision_prob) return; + if (amrex::Random(engine) > total_collision_prob) { return; } amrex::ParticleReal x, y, z; GetPosition.AsStored(ip, x, y, z); @@ -398,7 +398,7 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile nu_i += n_a * sigma_E * v_coll / nu_max; // check if this collision should be performed - if (col_select > nu_i) continue; + if (col_select > nu_i) { continue; } // charge exchange is implemented as a simple swap of the projectile // and target velocities which doesn't require any of the Lorentz diff --git a/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H b/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H index cd9b74c72b8..6cd3b0c20f7 100644 --- a/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H +++ b/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H @@ -77,7 +77,7 @@ public: using std::sqrt; // determine if this particle should collide - if (Random(engine) > m_total_collision_prob) return false; + if (Random(engine) > m_total_collision_prob) { return false; } // get references to the particle to get its position const auto& p = ptd.getSuperParticle(i); diff --git a/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp b/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp index bf2138bc78b..458b3c136a3 100644 --- a/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp +++ b/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp @@ -97,14 +97,14 @@ MCCProcess::readCrossSectionFile ( amrex::Gpu::HostVector& sigmas ) { std::ifstream infile(cross_section_file); - if(!infile.is_open()) WARPX_ABORT_WITH_MESSAGE("Failed to open cross-section data file"); + if(!infile.is_open()) { WARPX_ABORT_WITH_MESSAGE("Failed to open cross-section data file"); } amrex::ParticleReal energy, sigma; while (infile >> energy >> sigma) { energies.push_back(energy); sigmas.push_back(sigma); } - if (infile.bad()) WARPX_ABORT_WITH_MESSAGE("Failed to read cross-section data from file."); + if (infile.bad()) { WARPX_ABORT_WITH_MESSAGE("Failed to read cross-section data from file."); } infile.close(); } diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 58fe2c9911a..7a9374ae5f8 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -87,8 +87,9 @@ public: BinaryCollision (std::string collision_name, MultiParticleContainer const * const mypc) : CollisionBase(collision_name) { - if(m_species_names.size() != 2) + if(m_species_names.size() != 2) { WARPX_ABORT_WITH_MESSAGE("Binary collision " + collision_name + " must have exactly two species."); + } m_isSameSpecies = (m_species_names[0] == m_species_names[1]); @@ -166,7 +167,7 @@ public: // Enable tiling amrex::MFItInfo info; - if (amrex::Gpu::notInLaunchRegion()) info.EnableTiling(species1.tile_size); + if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(species1.tile_size); } // Loop over refinement levels for (int lev = 0; lev <= species1.finestLevel(); ++lev){ @@ -341,7 +342,7 @@ public: p_pair_offsets[i_cell] : 0; // Do not collide if there is only one particle in the cell - if ( cell_stop_1 - cell_start_1 <= 1 ) return; + if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } // shuffle ShuffleFisherYates( @@ -447,11 +448,12 @@ public: const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; // Particular case: no pair if a species has no particle in that cell - if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) + if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) { p_n_pairs_in_each_cell[i_cell] = 0; - else + } else { p_n_pairs_in_each_cell[i_cell] = amrex::max(n_part_in_cell_1,n_part_in_cell_2); + } } ); @@ -502,7 +504,7 @@ public: // Do not collide if one species is missing in the cell if ( cell_stop_1 - cell_start_1 < 1 || - cell_stop_2 - cell_start_2 < 1 ) return; + cell_stop_2 - cell_start_2 < 1 ) { return; } // shuffle ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp index 82f263e9daf..430bac5e548 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp @@ -118,16 +118,21 @@ namespace BinaryCollisionUtils{ CollisionType nuclear_fusion_type_to_collision_type (const NuclearFusionType fusion_type) { - if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) + if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) { return CollisionType::DeuteriumTritiumToNeutronHeliumFusion; - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) { return CollisionType::DeuteriumDeuteriumToProtonTritiumFusion; - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) { return CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion; - if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) + } + if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) { return CollisionType::DeuteriumHeliumToProtonHeliumFusion; - if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) + } + if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { return CollisionType::ProtonBoronToAlphasFusion; + } WARPX_ABORT_WITH_MESSAGE("Invalid nuclear fusion type"); return CollisionType::Undefined; } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index b54404d46ba..d7403eeacf9 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -104,7 +104,7 @@ void ElasticCollisionPerez ( n12 = n12 / dV; } // Intra-species: Apply correction in collision rate - if (isSameSpecies) n12 *= T_PR(2.0); + if (isSameSpecies) { n12 *= T_PR(2.0); } // compute Debye length lmdD T_PR lmdD; diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index ca994ed1f56..9ad1b7a6cbf 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -115,7 +115,7 @@ public: { using namespace amrex::literals; - if (n_total_pairs == 0) return amrex::Vector(m_num_product_species, 0); + if (n_total_pairs == 0) { return amrex::Vector(m_num_product_species, 0); } // Compute offset array and allocate memory for the produced species amrex::Gpu::DeviceVector offsets(n_total_pairs); diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index fd74c9029ed..ac377a6799d 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -790,17 +790,17 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // computes min/max positions of current contributions #if !defined(WARPX_DIM_1D_Z) int dil = 1, diu = 1; - if (i_old < i_new) dil = 0; - if (i_old > i_new) diu = 0; + if (i_old < i_new) { dil = 0; } + if (i_old > i_new) { diu = 0; } #endif #if defined(WARPX_DIM_3D) int djl = 1, dju = 1; - if (j_old < j_new) djl = 0; - if (j_old > j_new) dju = 0; + if (j_old < j_new) { djl = 0; } + if (j_old > j_new) { dju = 0; } #endif int dkl = 1, dku = 1; - if (k_old < k_new) dkl = 0; - if (k_old > k_new) dku = 0; + if (k_old < k_new) { dkl = 0; } + if (k_old > k_new) { dku = 0; } #if defined(WARPX_DIM_3D) @@ -1068,7 +1068,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, + uzp[ip] * uzp[ip] * invcsq); // Product of particle charges and weights amrex::Real wq = q * wp[ip]; - if (do_ionization) wq *= ion_lev[ip]; + if (do_ionization) { wq *= ion_lev[ip]; } // Current particle positions (in physical units) amrex::ParticleReal xp, yp, zp; diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H index f80705fd043..3f9ef133477 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H @@ -170,8 +170,9 @@ public: amrex::ParticleReal, pxr_p::unit_system::SI>( px, py, pz); if (gamma_photon < pxr_m::two || - chi_phot < m_bw_minimum_chi_phot) + chi_phot < m_bw_minimum_chi_phot) { return 0; + } const auto is_out = pxr_bw::evolve_optical_depth< amrex::ParticleReal, diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp index 968ab8b85f1..da50d41dc8f 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp +++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp @@ -69,7 +69,7 @@ BreitWheelerEngine::init_lookup_tables_from_raw_data ( { auto raw_iter = raw_data.begin(); const auto size_first = pxr_sr::get_out(raw_iter); - if(size_first <= 0 || size_first >= raw_data.size() ) return false; + if(size_first <= 0 || size_first >= raw_data.size() ) { return false; } const auto raw_dndt_table = vector{ raw_iter, raw_iter+static_cast(size_first)}; @@ -80,8 +80,9 @@ BreitWheelerEngine::init_lookup_tables_from_raw_data ( m_dndt_table = BW_dndt_table{raw_dndt_table}; m_pair_prod_table = BW_pair_prod_table{raw_pair_prod_table}; - if (!m_dndt_table.is_init() || !m_pair_prod_table.is_init()) + if (!m_dndt_table.is_init() || !m_pair_prod_table.is_init()) { return false; + } m_bw_minimum_chi_phot = bw_minimum_chi_phot; @@ -104,8 +105,9 @@ void BreitWheelerEngine::init_builtin_tables( vector BreitWheelerEngine::export_lookup_tables_data () const { - if(!m_lookup_tables_initialized) + if(!m_lookup_tables_initialized) { return vector{}; + } const auto data_dndt = m_dndt_table.serialize(); const auto data_pair_prod = m_pair_prod_table.serialize(); @@ -114,10 +116,12 @@ vector BreitWheelerEngine::export_lookup_tables_data () const vector res{}; pxr_sr::put_in(size_first, res); - for (const auto& tmp : data_dndt) + for (const auto& tmp : data_dndt) { pxr_sr::put_in(tmp, res); - for (const auto& tmp : data_pair_prod) + } + for (const auto& tmp : data_pair_prod) { pxr_sr::put_in(tmp, res); + } return res; } diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H index ceacb2016e8..3ae27e411b6 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H @@ -153,8 +153,9 @@ public: const auto chi_part = QedUtils::chi_ele_pos( m_e*ux, m_e*uy, m_e*uz, ex, ey, ez, bx, by, bz); - if (chi_part < m_qs_minimum_chi_part) + if (chi_part < m_qs_minimum_chi_part) { return 0; + } const auto is_out = pxr_qs::evolve_optical_depth< amrex::ParticleReal, diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp index c6548d91612..59a881814c6 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp +++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp @@ -68,7 +68,7 @@ QuantumSynchrotronEngine::init_lookup_tables_from_raw_data ( { auto raw_iter = raw_data.begin(); const auto size_first = pxr_sr::get_out(raw_iter); - if(size_first <= 0 || size_first >= raw_data.size() ) return false; + if(size_first <= 0 || size_first >= raw_data.size() ) { return false; } const auto raw_dndt_table = vector{ raw_iter, raw_iter+static_cast(size_first)}; @@ -79,8 +79,9 @@ QuantumSynchrotronEngine::init_lookup_tables_from_raw_data ( m_dndt_table = QS_dndt_table{raw_dndt_table}; m_phot_em_table = QS_phot_em_table{raw_phot_em_table}; - if (!m_dndt_table.is_init() || !m_phot_em_table.is_init()) + if (!m_dndt_table.is_init() || !m_phot_em_table.is_init()) { return false; + } m_qs_minimum_chi_part = qs_minimum_chi_part; @@ -103,8 +104,9 @@ void QuantumSynchrotronEngine::init_builtin_tables( vector QuantumSynchrotronEngine::export_lookup_tables_data () const { - if(!m_lookup_tables_initialized) + if(!m_lookup_tables_initialized) { return vector{}; + } const auto data_dndt = m_dndt_table.serialize(); const auto data_phot_em = m_phot_em_table.serialize(); @@ -113,10 +115,12 @@ vector QuantumSynchrotronEngine::export_lookup_tables_data () const vector res{}; pxr_sr::put_in(size_first, res); - for (const auto& tmp : data_dndt) + for (const auto& tmp : data_dndt) { pxr_sr::put_in(tmp, res); - for (const auto& tmp : data_phot_em) + } + for (const auto& tmp : data_phot_em) { pxr_sr::put_in(tmp, res); + } return res; } diff --git a/Source/Particles/Filter/FilterFunctors.H b/Source/Particles/Filter/FilterFunctors.H index 54cf09fb3d7..354d57e2c98 100644 --- a/Source/Particles/Filter/FilterFunctors.H +++ b/Source/Particles/Filter/FilterFunctors.H @@ -164,7 +164,7 @@ struct GeometryFilter AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool operator () (const SuperParticleType& p, const amrex::RandomEngine&) const noexcept { - if ( !m_is_active ) return true; + if ( !m_is_active ) { return true; } return ! (AMREX_D_TERM( (p.pos(0) < m_domain.lo(0)) || (p.pos(0) > m_domain.hi(0) ), || (p.pos(1) < m_domain.lo(1)) || (p.pos(1) > m_domain.hi(1) ), || (p.pos(2) < m_domain.lo(2)) || (p.pos(2) > m_domain.hi(2) ))); diff --git a/Source/Particles/Gather/GetExternalFields.H b/Source/Particles/Gather/GetExternalFields.H index 5e74a409824..9fd534e33d9 100644 --- a/Source/Particles/Gather/GetExternalFields.H +++ b/Source/Particles/Gather/GetExternalFields.H @@ -58,7 +58,7 @@ struct GetExternalEBField std::optional d_lattice_element_finder; - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool isNoOp () const { return (m_Etype == None && m_Btype == None && !d_lattice_element_finder.has_value()); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -78,7 +78,7 @@ struct GetExternalEBField field_Bx, field_By, field_Bz); } - if (m_Etype == None && m_Btype == None) return; + if (m_Etype == None && m_Btype == None) { return; } amrex::ParticleReal Ex = 0._prt; amrex::ParticleReal Ey = 0._prt; diff --git a/Source/Particles/Gather/GetExternalFields.cpp b/Source/Particles/Gather/GetExternalFields.cpp index 4aeb6ce05b9..a4ded3c9f69 100644 --- a/Source/Particles/Gather/GetExternalFields.cpp +++ b/Source/Particles/Gather/GetExternalFields.cpp @@ -31,12 +31,12 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset m_Etype = Unknown; m_Btype = Unknown; - if (mypc.m_E_ext_particle_s == "none") m_Etype = None; - if (mypc.m_B_ext_particle_s == "none") m_Btype = None; + if (mypc.m_E_ext_particle_s == "none") { m_Etype = None; } + if (mypc.m_B_ext_particle_s == "none") { m_Btype = None; } // These lines will be removed once the user interface is redefined and the CI tests updated - if (mypc.m_E_ext_particle_s == "constant") m_Etype = None; - if (mypc.m_B_ext_particle_s == "constant") m_Btype = None; + if (mypc.m_E_ext_particle_s == "constant") { m_Etype = None; } + if (mypc.m_B_ext_particle_s == "constant") { m_Btype = None; } if (mypc.m_E_ext_particle_s == "parse_e_ext_particle_function" || mypc.m_B_ext_particle_s == "parse_b_ext_particle_function" || @@ -68,8 +68,8 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset if (mypc.m_E_ext_particle_s == "repeated_plasma_lens" || mypc.m_B_ext_particle_s == "repeated_plasma_lens") { - if (mypc.m_E_ext_particle_s == "repeated_plasma_lens") m_Etype = RepeatedPlasmaLens; - if (mypc.m_B_ext_particle_s == "repeated_plasma_lens") m_Btype = RepeatedPlasmaLens; + if (mypc.m_E_ext_particle_s == "repeated_plasma_lens") { m_Etype = RepeatedPlasmaLens; } + if (mypc.m_B_ext_particle_s == "repeated_plasma_lens") { m_Btype = RepeatedPlasmaLens; } m_dt = warpx.getdt(a_pti.GetLevel()); auto& attribs = a_pti.GetAttribs(); m_ux = attribs[PIdx::ux].dataPtr() + a_offset; diff --git a/Source/Particles/Gather/ScaleFields.H b/Source/Particles/Gather/ScaleFields.H index 8605868a87e..10a10cfe190 100644 --- a/Source/Particles/Gather/ScaleFields.H +++ b/Source/Particles/Gather/ScaleFields.H @@ -41,7 +41,7 @@ struct ScaleFields { using namespace amrex::literals; - if (!m_do_scale) return; + if (!m_do_scale) { return; } // Scale the fields of particles about to cross the injection plane. // This only approximates what should be happening. The particles diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index 7d918d5dfb5..5c2950759d3 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -279,7 +279,7 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, void LaserParticleContainer::ContinuousInjection (const RealBox& injection_box) { - if (!m_enabled) return; + if (!m_enabled) { return; } // Input parameter injection_box contains small box where injection // should occur. @@ -321,7 +321,7 @@ LaserParticleContainer::ContinuousInjection (const RealBox& injection_box) void LaserParticleContainer::UpdateAntennaPosition (const amrex::Real dt) { - if (!m_enabled) return; + if (!m_enabled) { return; } const int dir = WarpX::moving_window_dir; if (do_continuous_injection and (WarpX::gamma_boost > 1)){ @@ -350,7 +350,7 @@ LaserParticleContainer::UpdateAntennaPosition (const amrex::Real dt) void LaserParticleContainer::InitData () { - if (!m_enabled) return; + if (!m_enabled) { return; } // Call InitData on max level to inject one laser particle per // finest cell. @@ -367,7 +367,7 @@ LaserParticleContainer::InitData () void LaserParticleContainer::InitData (int lev) { - if (!m_enabled) return; + if (!m_enabled) { return; } // spacing of laser particles in the laser plane. // has to be done after geometry is set up. @@ -541,7 +541,7 @@ LaserParticleContainer::InitData (int lev) amrex::Vector particle_uy(np, 0.0); amrex::Vector particle_uz(np, 0.0); - if (Verbose()) amrex::Print() << Utils::TextMsg::Info("Adding laser particles"); + if (Verbose()) { amrex::Print() << Utils::TextMsg::Info("Adding laser particles"); } amrex::Vector> attr; attr.push_back(particle_w); amrex::Vector> attr_int; @@ -566,7 +566,7 @@ LaserParticleContainer::Evolve (int lev, WARPX_PROFILE("LaserParticleContainer::Evolve()"); WARPX_PROFILE_VAR_NS("LaserParticleContainer::Evolve::ParticlePush", blp_pp); - if (!m_enabled) return; + if (!m_enabled) { return; } Real t_lab = t; if (WarpX::gamma_boost > 1) { @@ -701,7 +701,7 @@ LaserParticleContainer::Evolve (int lev, void LaserParticleContainer::PostRestart () { - if (!m_enabled) return; + if (!m_enabled) { return; } Real Sx, Sy; const int lev = finestLevel(); diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 4eaf89c2aa4..369e653ee8b 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -461,11 +461,11 @@ MultiParticleContainer::Evolve (int lev, jx.setVal(0.0); jy.setVal(0.0); jz.setVal(0.0); - if (cjx) cjx->setVal(0.0); - if (cjy) cjy->setVal(0.0); - if (cjz) cjz->setVal(0.0); - if (rho) rho->setVal(0.0); - if (crho) crho->setVal(0.0); + if (cjx) { cjx->setVal(0.0); } + if (cjy) { cjy->setVal(0.0); } + if (cjz) { cjz->setVal(0.0); } + if (rho) { rho->setVal(0.0); } + if (crho) { crho->setVal(0.0); } } for (auto& pc : allcontainers) { pc->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, cjx, cjy, cjz, @@ -507,8 +507,9 @@ MultiParticleContainer::GetZeroChargeDensity (const int lev) const bool is_PSATD_RZ = false; #endif - if( !is_PSATD_RZ ) + if( !is_PSATD_RZ ) { nba.surroundingNodes(); + } auto zero_rho = std::make_unique(nba, dmap, WarpX::ncomps, ng_rho); zero_rho->setVal(amrex::Real(0.0)); @@ -555,7 +556,7 @@ MultiParticleContainer::DepositCharge ( } // Push the particles in time, if needed - if (relative_time != 0.) PushX(relative_time); + if (relative_time != 0.) { PushX(relative_time); } bool const local = true; bool const reset = false; @@ -564,13 +565,13 @@ MultiParticleContainer::DepositCharge ( // Call the deposition kernel for each species for (auto& pc : allcontainers) { - if (pc->do_not_deposit) continue; + if (pc->do_not_deposit) { continue; } pc->DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, interpolate_across_levels); } // Push the particles back in time - if (relative_time != 0.) PushX(-relative_time); + if (relative_time != 0.) { PushX(-relative_time); } #ifdef WARPX_DIM_RZ for (int lev = 0; lev < rho.size(); ++lev) @@ -586,7 +587,7 @@ MultiParticleContainer::GetChargeDensity (int lev, bool local) std::unique_ptr rho = GetZeroChargeDensity(lev); for (auto& container : allcontainers) { - if (container->do_not_deposit) continue; + if (container->do_not_deposit) { continue; } const std::unique_ptr rhoi = container->GetChargeDensity(lev, true); MultiFab::Add(*rho, *rhoi, 0, 0, rho->nComp(), rho->nGrowVect()); } @@ -790,10 +791,10 @@ MultiParticleContainer::mapSpeciesProduct () #ifdef WARPX_QED if (m_do_qed_schwinger) { - m_qed_schwinger_ele_product = - getSpeciesID(m_qed_schwinger_ele_product_name); - m_qed_schwinger_pos_product = - getSpeciesID(m_qed_schwinger_pos_product_name); + m_qed_schwinger_ele_product = + getSpeciesID(m_qed_schwinger_ele_product_name); + m_qed_schwinger_pos_product = + getSpeciesID(m_qed_schwinger_pos_product_name); } #endif } @@ -976,11 +977,13 @@ void MultiParticleContainer::InitQED () } } - if(m_nspecies_quantum_sync != 0) + if(m_nspecies_quantum_sync != 0) { InitQuantumSync(); + } - if(m_nspecies_breit_wheeler !=0) + if(m_nspecies_breit_wheeler !=0) { InitBreitWheeler(); + } } @@ -1065,8 +1068,9 @@ void MultiParticleContainer::InitBreitWheeler () // considered for pair production. If a photon has chi < chi_min, // the optical depth is not evolved and photon generation is ignored amrex::Real bw_minimum_chi_part; - if(!utils::parser::queryWithParser(pp_qed_bw, "chi_min", bw_minimum_chi_part)) + if(!utils::parser::queryWithParser(pp_qed_bw, "chi_min", bw_minimum_chi_part)) { WARPX_ABORT_WITH_MESSAGE("qed_bw.chi_min should be provided!"); + } pp_qed_bw.query("lookup_table_mode", lookup_table_mode); if(lookup_table_mode.empty()){ @@ -1488,7 +1492,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, // in pc_product_ele and positrons in pc_product_pos for (auto& pc_source : allcontainers){ - if(!pc_source->has_breit_wheeler()) continue; + if(!pc_source->has_breit_wheeler()) { continue; } // Get product species auto& pc_product_ele = diff --git a/Source/Particles/ParticleBoundaries_K.H b/Source/Particles/ParticleBoundaries_K.H index 662e1240f8b..bbec1f54e01 100644 --- a/Source/Particles/ParticleBoundaries_K.H +++ b/Source/Particles/ParticleBoundaries_K.H @@ -140,10 +140,10 @@ namespace ApplyParticleBoundaries { uy = ur*std::sin(y) + ut*std::cos(y); } #else - if (change_sign_ux) ux = -ux; - if (change_sign_uy) uy = -uy; + if (change_sign_ux) { ux = -ux; } + if (change_sign_uy) { uy = -uy; } #endif - if (change_sign_uz) uz = -uz; + if (change_sign_uz) { uz = -uz; } } } diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index b6eba51347c..c4024417539 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -51,12 +51,15 @@ struct CopyAndTimestamp { int src_i, int dst_i) const noexcept { dst.m_aos[dst_i] = src.m_aos[src_i]; - for (int j = 0; j < SrcData::NAR; ++j) + for (int j = 0; j < SrcData::NAR; ++j) { dst.m_rdata[j][dst_i] = src.m_rdata[j][src_i]; - for (int j = 0; j < src.m_num_runtime_real; ++j) + } + for (int j = 0; j < src.m_num_runtime_real; ++j) { dst.m_runtime_rdata[j][dst_i] = src.m_runtime_rdata[j][src_i]; - for (int j = 0; j < src.m_num_runtime_int; ++j) + } + for (int j = 0; j < src.m_num_runtime_int; ++j) { dst.m_runtime_idata[j][dst_i] = src.m_runtime_idata[j][src_i]; + } dst.m_runtime_idata[m_index][dst_i] = m_step; } }; @@ -115,7 +118,7 @@ ParticleBoundaryBuffer::ParticleBoundaryBuffer () #endif // Set the flag whether the boundary is active or any species for (int i = 0; i < numBoundaries(); ++i) { - if (m_do_boundary_buffer[i][ispecies]) m_do_any_boundary[i] = 1; + if (m_do_boundary_buffer[i][ispecies]) { m_do_any_boundary[i] = 1; } } } @@ -210,7 +213,7 @@ void ParticleBoundaryBuffer::clearParticles (int const i) { for (int ispecies = 0; ispecies < numSpecies(); ++ispecies) { auto& species_buffer = buffer[ispecies]; - if (species_buffer.isDefined()) species_buffer.clearParticles(); + if (species_buffer.isDefined()) { species_buffer.clearParticles(); } } } @@ -227,13 +230,13 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (geom.isPeriodic(idim)) continue; + if (geom.isPeriodic(idim)) { continue; } for (int iside = 0; iside < 2; ++iside) { auto& buffer = m_particle_containers[2*idim+iside]; for (int i = 0; i < numSpecies(); ++i) { - if (!m_do_boundary_buffer[2*idim+iside][i]) continue; + if (!m_do_boundary_buffer[2*idim+iside][i]) { continue; } const WarpXParticleContainer& pc = mypc.GetParticleContainer(i); if (!buffer[i].isDefined()) { @@ -250,13 +253,13 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, for(PIter pti(pc, lev); pti.isValid(); ++pti) { auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); - if(plevel.find(index) == plevel.end()) continue; + if(plevel.find(index) == plevel.end()) { continue; } auto& ptile_buffer = species_buffer.DefineAndReturnParticleTile( lev, pti.index(), pti.LocalTileIndex()); const auto& ptile = plevel.at(index); auto np = ptile.numParticles(); - if (np == 0) continue; + if (np == 0) { continue; } auto predicate = IsOutsideDomainBoundary{plo, phi, idim, iside}; @@ -324,7 +327,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, pti.LocalTileIndex()); const auto& ptile = plevel.at(index); auto np = ptile.numParticles(); - if (np == 0) continue; + if (np == 0) { continue; } using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; auto predicate = [=] AMREX_GPU_HOST_DEVICE (const SrcData& /*src*/, const int ip) diff --git a/Source/Particles/ParticleCreation/FilterCopyTransform.H b/Source/Particles/ParticleCreation/FilterCopyTransform.H index 7a85f59318f..264b8b86e29 100644 --- a/Source/Particles/ParticleCreation/FilterCopyTransform.H +++ b/Source/Particles/ParticleCreation/FilterCopyTransform.H @@ -56,7 +56,7 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind using namespace amrex; const auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector offsets(np); auto total = amrex::Scan::ExclusiveSum(np, mask, offsets.data()); @@ -129,7 +129,7 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, using namespace amrex; const auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector mask(np); @@ -199,7 +199,7 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, using namespace amrex; auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector offsets(np); auto total = amrex::Scan::ExclusiveSum(np, mask, offsets.data()); @@ -286,7 +286,7 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, using namespace amrex; auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector mask(np); diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 38dc1d6daf2..04973d2d7b8 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -54,7 +54,7 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B using namespace amrex; const auto ncells = box.volume(); - if (ncells == 0) return 0; + if (ncells == 0) { return 0; } auto & warpx = WarpX::GetInstance(); const int level_0 = 0; @@ -184,7 +184,7 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B auto arrNumPartCreation = NumPartCreation.array(); const auto ncells = box.volume(); - if (ncells == 0) return 0; + if (ncells == 0) { return 0; } Gpu::DeviceVector mask(ncells); diff --git a/Source/Particles/ParticleCreation/SmartCopy.H b/Source/Particles/ParticleCreation/SmartCopy.H index da5637a7919..2c04baa18bb 100644 --- a/Source/Particles/ParticleCreation/SmartCopy.H +++ b/Source/Particles/ParticleCreation/SmartCopy.H @@ -52,16 +52,20 @@ struct SmartCopy dst.m_aos[i_dst] = src.m_aos[i_src]; // initialize the real components - for (int j = 0; j < DstData::NAR; ++j) + for (int j = 0; j < DstData::NAR; ++j) { dst.m_rdata[j][i_dst] = initializeRealValue(m_policy_real[j], engine); - for (int j = 0; j < dst.m_num_runtime_real; ++j) + } + for (int j = 0; j < dst.m_num_runtime_real; ++j) { dst.m_runtime_rdata[j][i_dst] = initializeRealValue(m_policy_real[j+DstData::NAR], engine); + } // initialize the int components - for (int j = 0; j < DstData::NAI; ++j) + for (int j = 0; j < DstData::NAI; ++j) { dst.m_idata[j][i_dst] = initializeIntValue(m_policy_int[j]); - for (int j = 0; j < dst.m_num_runtime_int; ++j) + } + for (int j = 0; j < dst.m_num_runtime_int; ++j) { dst.m_runtime_idata[j][i_dst] = initializeIntValue(m_policy_int[j+DstData::NAI]); + } // copy the shared real components for (int j = 0; j < m_num_copy_real; ++j) diff --git a/Source/Particles/ParticleCreation/SmartCreate.H b/Source/Particles/ParticleCreation/SmartCreate.H index 1be3bd79cf3..67d7767a5d3 100644 --- a/Source/Particles/ParticleCreation/SmartCreate.H +++ b/Source/Particles/ParticleCreation/SmartCreate.H @@ -63,16 +63,20 @@ struct SmartCreate prt.m_aos[i_prt].id() = id; // initialize the real components - for (int j = 0; j < PartData::NAR; ++j) + for (int j = 0; j < PartData::NAR; ++j) { prt.m_rdata[j][i_prt] = initializeRealValue(m_policy_real[j], engine); - for (int j = 0; j < prt.m_num_runtime_real; ++j) + } + for (int j = 0; j < prt.m_num_runtime_real; ++j) { prt.m_runtime_rdata[j][i_prt] = initializeRealValue(m_policy_real[j+PartData::NAR], engine); + } // initialize the int components - for (int j = 0; j < PartData::NAI; ++j) + for (int j = 0; j < PartData::NAI; ++j) { prt.m_idata[j][i_prt] = initializeIntValue(m_policy_int[j]); - for (int j = 0; j < prt.m_num_runtime_int; ++j) + } + for (int j = 0; j < prt.m_num_runtime_int; ++j) { prt.m_runtime_idata[j][i_prt] = initializeIntValue(m_policy_int[j+PartData::NAI]); + } } }; diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index 28dfcd8e280..4dc442bdf95 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -185,7 +185,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, np_to_push, [=] AMREX_GPU_DEVICE (long i, auto exteb_control, auto qed_control) { - if (do_copy) copyAttribs(i); + if (do_copy) { copyAttribs(i); } ParticleReal x, y, z; GetPosition(i, x, y, z); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 182577a81cf..f21e3d32c08 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -343,7 +343,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp_species_name.query("do_field_ionization", do_field_ionization); pp_species_name.query("do_resampling", do_resampling); - if (do_resampling) m_resampler = Resampling(species_name); + if (do_resampling) { m_resampler = Resampling(species_name); } //check if Radiation Reaction is enabled and do consistency checks pp_species_name.query("do_classical_radiation_reaction", do_classical_radiation_reaction); @@ -365,12 +365,14 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp #ifdef WARPX_QED pp_species_name.query("do_qed_quantum_sync", m_do_qed_quantum_sync); - if (m_do_qed_quantum_sync) + if (m_do_qed_quantum_sync) { AddRealComp("opticalDepthQSR"); + } pp_species_name.query("do_qed_breit_wheeler", m_do_qed_breit_wheeler); - if (m_do_qed_breit_wheeler) + if (m_do_qed_breit_wheeler) { AddRealComp("opticalDepthBW"); + } if(m_do_qed_quantum_sync){ pp_species_name.get("qed_quantum_sync_phot_product_species", @@ -999,7 +1001,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // If no part_realbox is provided, initialize particles in the whole domain const Geometry& geom = Geom(lev); - if (!part_realbox.ok()) part_realbox = geom.ProbDomain(); + if (!part_realbox.ok()) { part_realbox = geom.ProbDomain(); } const int num_ppc = plasma_injector.num_particles_per_cell; #ifdef WARPX_DIM_RZ @@ -1135,12 +1137,15 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const auto zlim = GpuArray{lo.z,(lo.z+hi.z)/2._rt,hi.z}; const auto checker = [&](){ - for (const auto& x : xlim) - for (const auto& y : ylim) - for (const auto& z : zlim) + for (const auto& x : xlim) { + for (const auto& y : ylim) { + for (const auto& z : zlim) { if (inj_pos->insideBounds(x,y,z) and (inj_rho->getDensity(x,y,z) > 0) ) { return 1; } + } + } + } return 0; }; const int flag_pcount = checker(); @@ -1248,12 +1253,14 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // has to be initialized const bool loc_has_quantum_sync = has_quantum_sync(); const bool loc_has_breit_wheeler = has_breit_wheeler(); - if (loc_has_quantum_sync) + if (loc_has_quantum_sync) { p_optical_depth_QSR = soa.GetRealData( particle_comps["opticalDepthQSR"]).data() + old_size; - if(loc_has_breit_wheeler) + } + if(loc_has_breit_wheeler) { p_optical_depth_BW = soa.GetRealData( particle_comps["opticalDepthBW"]).data() + old_size; + } //If needed, get the appropriate functors from the engines QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt; @@ -1286,7 +1293,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const auto index = overlap_box.index(iv); #ifdef WARPX_DIM_RZ Real theta_offset = 0._rt; - if (rz_random_theta) theta_offset = amrex::Random(engine) * 2._rt * MathConst::pi; + if (rz_random_theta) { theta_offset = amrex::Random(engine) * 2._rt * MathConst::pi; } #endif Real scale_fac = 0.0_rt; @@ -1538,15 +1545,15 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, scale_fac = dx[0]*dx[1]/num_ppc_real; // When emission is in the r direction, the emitting surface is a cylinder. // The factor 2*pi*r is added later below. - if (plasma_injector.flux_normal_axis == 0) scale_fac /= dx[0]; + if (plasma_injector.flux_normal_axis == 0) { scale_fac /= dx[0]; } // When emission is in the z direction, the emitting surface is an annulus // The factor 2*pi*r is added later below. - if (plasma_injector.flux_normal_axis == 2) scale_fac /= dx[1]; + if (plasma_injector.flux_normal_axis == 2) { scale_fac /= dx[1]; } // When emission is in the theta direction (flux_normal_axis == 1), // the emitting surface is a rectangle, within the plane of the simulation #elif defined(WARPX_DIM_1D_Z) scale_fac = dx[0]/num_ppc_real; - if (plasma_injector.flux_normal_axis == 2) scale_fac /= dx[0]; + if (plasma_injector.flux_normal_axis == 2) { scale_fac /= dx[0]; } #endif amrex::LayoutData* cost = WarpX::getCosts(0); @@ -1798,12 +1805,14 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // has to be initialized const bool loc_has_quantum_sync = has_quantum_sync(); const bool loc_has_breit_wheeler = has_breit_wheeler(); - if (loc_has_quantum_sync) + if (loc_has_quantum_sync) { p_optical_depth_QSR = soa.GetRealData( particle_comps["opticalDepthQSR"]).data() + old_size; - if(loc_has_breit_wheeler) + } + if(loc_has_breit_wheeler) { p_optical_depth_BW = soa.GetRealData( particle_comps["opticalDepthBW"]).data() + old_size; + } //If needed, get the appropriate functors from the engines QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt; @@ -2061,8 +2070,9 @@ PhysicalParticleContainer::Evolve (int lev, const auto t_lev = pti.GetLevel(); const auto index = pti.GetPairIndex(); tmp_particle_data.resize(finestLevel()+1); - for (int i = 0; i < TmpIdx::nattribs; ++i) + for (int i = 0; i < TmpIdx::nattribs; ++i) { tmp_particle_data[t_lev][index][i].resize(np); + } } } @@ -2547,7 +2557,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, { WARPX_PROFILE("PhysicalParticleContainer::PushP()"); - if (do_not_push) return; + if (do_not_push) { return; } const std::array& dx = WarpX::CellSize(std::max(lev,0)); @@ -2743,7 +2753,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, (gather_lev==(lev )), "Gather buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_push == 0) return; + if (np_to_push == 0) { return; } // Get cell size on gather_lev const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); @@ -2842,7 +2852,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, #ifdef WARPX_QED const auto do_sync = m_do_qed_quantum_sync; amrex::Real t_chi_max = 0.0; - if (do_sync) t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); + if (do_sync) { t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); } QuantumSynchrotronEvolveOpticalDepth evolve_opt; amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR = nullptr; @@ -2958,7 +2968,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, void PhysicalParticleContainer::InitIonizationModule () { - if (!do_field_ionization) return; + if (!do_field_ionization) { return; } const ParmParse pp_species_name(species_name); if (charge != PhysConst::q_e){ ablastr::warn_manager::WMRecordWarning("Species", diff --git a/Source/Particles/Pusher/CopyParticleAttribs.H b/Source/Particles/Pusher/CopyParticleAttribs.H index 1f0cee67421..d20626a53a7 100644 --- a/Source/Particles/Pusher/CopyParticleAttribs.H +++ b/Source/Particles/Pusher/CopyParticleAttribs.H @@ -49,7 +49,7 @@ struct CopyParticleAttribs CopyParticleAttribs (const WarpXParIter& a_pti, TmpParticles& tmp_particle_data, long a_offset = 0) noexcept { - if (tmp_particle_data.empty()) return; + if (tmp_particle_data.empty()) { return; } auto& attribs = a_pti.GetAttribs(); diff --git a/Source/Particles/Pusher/PushSelector.H b/Source/Particles/Pusher/PushSelector.H index 637d2ac74db..fd75f97f7c3 100644 --- a/Source/Particles/Pusher/PushSelector.H +++ b/Source/Particles/Pusher/PushSelector.H @@ -70,7 +70,7 @@ void doParticlePush(const GetParticlePosition& GetPosition, amrex::ParticleReal qp = a_q; if (ion_lev) { qp *= ion_lev; } - if (do_copy) copyAttribs(i); + if (do_copy) { copyAttribs(i); } if (do_crr) { #ifdef WARPX_QED amrex::ignore_unused(t_chi_max); diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index e3910286c91..e48c478a6f7 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -89,8 +89,9 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, // do nothing for cells with less particles than min_ppc // (this intentionally includes skipping empty cells, too) - if (cell_numparts < min_ppc) + if (cell_numparts < min_ppc) { return; + } amrex::Real average_weight = 0._rt; // First loop over cell particles to compute average particle weight in the cell diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index d0e19af155a..ff7dbf48166 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -334,7 +334,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, { WARPX_PROFILE("RigidInjectedParticleContainer::PushP"); - if (do_not_push) return; + if (do_not_push) { return; } const std::array& dx = WarpX::CellSize(std::max(lev,0)); diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index d44bc8ad9c3..a1dc8d14f4a 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -330,10 +330,10 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, "Deposition buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_deposit == 0) return; + if (np_to_deposit == 0) { return; } // If user decides not to deposit - if (do_not_deposit) return; + if (do_not_deposit) { return; } // Number of guard cells for local deposition of J const WarpX& warpx = WarpX::GetInstance(); @@ -693,7 +693,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, "Deposition buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_deposit == 0) return; + if (np_to_deposit == 0) { return; } // Number of guard cells for local deposition of rho const WarpX& warpx = WarpX::GetInstance(); @@ -1014,7 +1014,7 @@ WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, { // Reset the rho array if reset is True int const nc = WarpX::ncomps; - if (reset) rho->setVal(0., icomp*nc, nc, rho->nGrowVect()); + if (reset) { rho->setVal(0., icomp*nc, nc, rho->nGrowVect()); } // Loop over particle tiles and deposit charge on each level #ifdef AMREX_USE_OMP @@ -1081,8 +1081,9 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) #else const bool is_PSATD_RZ = false; #endif - if( !is_PSATD_RZ ) + if( !is_PSATD_RZ ) { nba.surroundingNodes(); + } // Number of guard cells for local deposition of rho const WarpX& warpx = WarpX::GetInstance(); @@ -1117,7 +1118,7 @@ amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { total_charge = get<0>(reduce_data.value()); - if (!local) ParallelDescriptor::ReduceRealSum(total_charge); + if (!local) { ParallelDescriptor::ReduceRealSum(total_charge); } total_charge *= this->charge; return total_charge; } @@ -1230,7 +1231,7 @@ amrex::ParticleReal WarpXParticleContainer::maxParticleVelocity(bool local) { } } - if (!local) ParallelAllReduce::Max(max_v, ParallelDescriptor::Communicator()); + if (!local) { ParallelAllReduce::Max(max_v, ParallelDescriptor::Communicator()); } return max_v; } @@ -1248,7 +1249,7 @@ WarpXParticleContainer::PushX (int lev, amrex::Real dt) { WARPX_PROFILE("WarpXParticleContainer::PushX()"); - if (do_not_push) return; + if (do_not_push) { return; } amrex::LayoutData* costs = WarpX::getCosts(lev); @@ -1323,7 +1324,7 @@ WarpXParticleContainer::particlePostLocate(ParticleType& p, const ParticleLocData& pld, const int lev) { - if (not do_splitting) return; + if (not do_splitting) { return; } // Tag particle if goes to higher level. // It will be split later in the loop @@ -1345,7 +1346,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ WARPX_PROFILE("WarpXParticleContainer::ApplyBoundaryConditions()"); // Periodic boundaries are handled in AMReX code - if (m_boundary_conditions.CheckAll(ParticleBoundaryType::Periodic)) return; + if (m_boundary_conditions.CheckAll(ParticleBoundaryType::Periodic)) { return; } auto boundary_conditions = m_boundary_conditions.data; @@ -1384,7 +1385,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ ParticleType& p = pp[i]; // skip particles that are already flagged for removal - if (p.id() < 0) return; + if (p.id() < 0) { return; } ParticleReal x, y, z; GetPosition.AsStored(i, x, y, z); diff --git a/Source/Utils/Parser/IntervalsParser.cpp b/Source/Utils/Parser/IntervalsParser.cpp index 6af4ce6a8c3..6564003abdf 100644 --- a/Source/Utils/Parser/IntervalsParser.cpp +++ b/Source/Utils/Parser/IntervalsParser.cpp @@ -92,7 +92,7 @@ utils::parser::IntervalsParser::IntervalsParser ( const std::vector& instr_vec) { std::string inconcatenated; - for (const auto& instr_element : instr_vec) inconcatenated +=instr_element; + for (const auto& instr_element : instr_vec) { inconcatenated +=instr_element; } auto insplit = ablastr::utils::text::split_string>( inconcatenated, m_separator); @@ -102,7 +102,7 @@ utils::parser::IntervalsParser::IntervalsParser ( const SliceParser temp_slice(inslc); m_slices.push_back(temp_slice); if ((temp_slice.getPeriod() > 0) && - (temp_slice.getStop() >= temp_slice.getStart())) m_activated = true; + (temp_slice.getStop() >= temp_slice.getStart())) { m_activated = true; } } } @@ -155,7 +155,7 @@ utils::parser::BTDIntervalsParser::BTDIntervalsParser ( const std::vector& instr_vec) { std::string inconcatenated; - for (const auto& instr_element : instr_vec) inconcatenated +=instr_element; + for (const auto& instr_element : instr_vec) { inconcatenated +=instr_element; } auto const insplit = ablastr::utils::text::split_string>( inconcatenated, std::string(1,m_separator)); diff --git a/Source/Utils/Parser/ParserUtils.cpp b/Source/Utils/Parser/ParserUtils.cpp index 6d430d4a619..c2da9577947 100644 --- a/Source/Utils/Parser/ParserUtils.cpp +++ b/Source/Utils/Parser/ParserUtils.cpp @@ -135,7 +135,7 @@ amrex::Parser utils::parser::makeParser ( parser.registerVariables(varnames); std::set symbols = parser.symbols(); - for (auto const& v : varnames) symbols.erase(v); + for (auto const& v : varnames) { symbols.erase(v); } // User can provide inputs under this name, through which expressions // can be provided for arbitrary variables. PICMI inputs are aware of diff --git a/Source/Utils/RelativeCellPosition.cpp b/Source/Utils/RelativeCellPosition.cpp index 1a108b54b56..d30da1841cb 100644 --- a/Source/Utils/RelativeCellPosition.cpp +++ b/Source/Utils/RelativeCellPosition.cpp @@ -22,8 +22,9 @@ utils::getRelativeCellPosition(amrex::MultiFab const& mf) // WarpX::grid_type==GridType::Collocated means: all indices/directions on CellIndex::NODE for (int d = 0; d < AMREX_SPACEDIM; d++) { - if (idx_type.cellCentered(d)) + if (idx_type.cellCentered(d)) { relative_position.at(d) = 0.5; + } } return relative_position; diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index b361aab14e9..2123a65627b 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -159,8 +159,9 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ algo_to_int = current_deposition_algo_to_int; if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD || WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC || - WarpX::electrostatic_solver_id != ElectrostaticSolverAlgo::None) + WarpX::electrostatic_solver_id != ElectrostaticSolverAlgo::None) { algo_to_int["default"] = CurrentDepositionAlgo::Direct; + } } else if (0 == std::strcmp(pp_search_key, "charge_deposition")) { algo_to_int = charge_deposition_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "field_gathering")) { diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 6f2a97f5550..fdcb7cb8da4 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -144,7 +144,7 @@ WarpX::MoveWindow (const int step, bool move_j) if (step == end_moving_window_step) { amrex::Print() << Utils::TextMsg::Info("Stopping moving window"); } - if (!moving_window_active(step)) return 0; + if (!moving_window_active(step)) { return 0; } // Update the continuous position of the moving window, // and of the plasma injection @@ -165,7 +165,7 @@ WarpX::MoveWindow (const int step, bool move_j) const amrex::Real* cdx = geom[0].CellSize(); const int num_shift_base = static_cast((moving_window_x - current_lo[dir]) / cdx[dir]); - if (num_shift_base == 0) return 0; + if (num_shift_base == 0) { return 0; } // update the problem domain. Note the we only do this on the base level because // amrex::Geometry objects share the same, static RealBox. @@ -222,16 +222,16 @@ WarpX::MoveWindow (const int step, bool move_j) if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { use_Bparser = true; - if (dim == 0) Bfield_parser = m_p_ext_field_params->Bxfield_parser->compile<3>(); - if (dim == 1) Bfield_parser = m_p_ext_field_params->Byfield_parser->compile<3>(); - if (dim == 2) Bfield_parser = m_p_ext_field_params->Bzfield_parser->compile<3>(); + if (dim == 0) { Bfield_parser = m_p_ext_field_params->Bxfield_parser->compile<3>(); } + if (dim == 1) { Bfield_parser = m_p_ext_field_params->Byfield_parser->compile<3>(); } + if (dim == 2) { Bfield_parser = m_p_ext_field_params->Bzfield_parser->compile<3>(); } } if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { use_Eparser = true; - if (dim == 0) Efield_parser = m_p_ext_field_params->Exfield_parser->compile<3>(); - if (dim == 1) Efield_parser = m_p_ext_field_params->Eyfield_parser->compile<3>(); - if (dim == 2) Efield_parser = m_p_ext_field_params->Ezfield_parser->compile<3>(); + if (dim == 0) { Efield_parser = m_p_ext_field_params->Exfield_parser->compile<3>(); } + if (dim == 1) { Efield_parser = m_p_ext_field_params->Eyfield_parser->compile<3>(); } + if (dim == 2) { Efield_parser = m_p_ext_field_params->Ezfield_parser->compile<3>(); } } shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); diff --git a/Source/Utils/WarpXTagging.cpp b/Source/Utils/WarpXTagging.cpp index 8d87dfccc9b..192fc9e9152 100644 --- a/Source/Utils/WarpXTagging.cpp +++ b/Source/Utils/WarpXTagging.cpp @@ -32,7 +32,7 @@ WarpX::ErrorEst (int lev, TagBoxArray& tags, Real /*time*/, int /*ngrow*/) const auto dx = Geom(lev).CellSizeArray(); amrex::ParserExecutor<3> ref_parser; - if (ref_patch_parser) ref_parser = ref_patch_parser->compile<3>(); + if (ref_patch_parser) { ref_parser = ref_patch_parser->compile<3>(); } const auto ftlo = fine_tag_lo; const auto fthi = fine_tag_hi; #ifdef AMREX_USE_OMP diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index 327998a61f3..5b917a81fdf 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -146,7 +146,7 @@ void ConvertLabParamsToBoost() ReadBoostedFrameParameters(gamma_boost, beta_boost, boost_direction); - if (gamma_boost <= 1.) return; + if (gamma_boost <= 1.) { return; } Vector prob_lo(AMREX_SPACEDIM); Vector prob_hi(AMREX_SPACEDIM); @@ -313,8 +313,9 @@ void CheckGriddingForRZSpectral () const int electromagnetic_solver_id = GetAlgorithmInteger(pp_algo, "maxwell_solver"); // only check for PSATD in RZ - if (electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { return; + } int max_level; Vector n_cell(AMREX_SPACEDIM, -1); @@ -419,10 +420,12 @@ void ReadBCParams () const ParmParse pp_boundary("boundary"); pp_boundary.queryarr("field_lo", field_BC_lo, 0, AMREX_SPACEDIM); pp_boundary.queryarr("field_hi", field_BC_hi, 0, AMREX_SPACEDIM); - if (pp_boundary.queryarr("particle_lo", particle_BC_lo, 0, AMREX_SPACEDIM)) + if (pp_boundary.queryarr("particle_lo", particle_BC_lo, 0, AMREX_SPACEDIM)) { particle_boundary_specified = true; - if (pp_boundary.queryarr("particle_hi", particle_BC_hi, 0, AMREX_SPACEDIM)) + } + if (pp_boundary.queryarr("particle_hi", particle_BC_hi, 0, AMREX_SPACEDIM)) { particle_boundary_specified = true; + } AMREX_ALWAYS_ASSERT(field_BC_lo.size() == AMREX_SPACEDIM); AMREX_ALWAYS_ASSERT(field_BC_hi.size() == AMREX_SPACEDIM); AMREX_ALWAYS_ASSERT(particle_BC_lo.size() == AMREX_SPACEDIM); diff --git a/Source/Utils/WarpXVersion.cpp b/Source/Utils/WarpXVersion.cpp index 43978a4795a..41abfebb38c 100644 --- a/Source/Utils/WarpXVersion.cpp +++ b/Source/Utils/WarpXVersion.cpp @@ -17,10 +17,11 @@ WarpX::Version () #ifdef WARPX_GIT_VERSION version = std::string(WARPX_GIT_VERSION); #endif - if( version.empty() ) + if( version.empty() ) { return {"Unknown"}; - else + } else { return version; + } } std::string @@ -30,8 +31,9 @@ WarpX::PicsarVersion () #ifdef PICSAR_GIT_VERSION version = std::string(PICSAR_GIT_VERSION); #endif - if( version.empty() ) + if( version.empty() ) { return {"Unknown"}; - else + } else { return version; + } } diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 597a561ae2c..e862391383e 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -540,13 +540,13 @@ WarpX::ReadParameters () if(std::string str_abort_on_warning_threshold; pp_warpx.query("abort_on_warning_threshold", str_abort_on_warning_threshold)){ std::optional abort_on_warning_threshold = std::nullopt; - if (str_abort_on_warning_threshold == "high") + if (str_abort_on_warning_threshold == "high") { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::high; - else if (str_abort_on_warning_threshold == "medium" ) + } else if (str_abort_on_warning_threshold == "medium" ) { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::medium; - else if (str_abort_on_warning_threshold == "low") + } else if (str_abort_on_warning_threshold == "low") { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::low; - else { + } else { WARPX_ABORT_WITH_MESSAGE(str_abort_on_warning_threshold +"is not a valid option for warpx.abort_on_warning_threshold (use: low, medium or high)"); } @@ -771,7 +771,7 @@ WarpX::ReadParameters () // Filter currently not working with FDTD solver in RZ geometry: turn OFF by default // (see https://github.com/ECP-WarpX/WarpX/issues/1943) #ifdef WARPX_DIM_RZ - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) WarpX::use_filter = false; + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { WarpX::use_filter = false; } #endif // Read filter and fill IntVect filter_npass_each_dir with @@ -843,8 +843,9 @@ WarpX::ReadParameters () const bool shared_tilesize_is_specified = utils::parser::queryArrWithParser(pp_warpx, "shared_tilesize", vect_shared_tilesize, 0, AMREX_SPACEDIM); if (shared_tilesize_is_specified){ - for (int i=0; i(GetAlgorithmInteger(pp_warpx, "grid_type")); // Use same shape factors in all directions, for gathering - if (grid_type == GridType::Collocated) galerkin_interpolation = false; + if (grid_type == GridType::Collocated) { galerkin_interpolation = false; } #ifdef WARPX_DIM_RZ // Only needs to be set with WARPX_DIM_RZ, otherwise defaults to 1 @@ -1202,7 +1203,7 @@ WarpX::ReadParameters () } // Use same shape factors in all directions, for gathering - if (field_gathering_algo == GatheringAlgo::MomentumConserving) galerkin_interpolation = false; + if (field_gathering_algo == GatheringAlgo::MomentumConserving) { galerkin_interpolation = false; } // With the PSATD solver, momentum-conserving field gathering // combined with mesh refinement does not seem to work correctly @@ -1228,8 +1229,9 @@ WarpX::ReadParameters () load_balance_intervals_string_vec); pp_algo.query("load_balance_with_sfc", load_balance_with_sfc); // Knapsack factor only used with non-SFC strategy - if (!load_balance_with_sfc) + if (!load_balance_with_sfc) { pp_algo.query("load_balance_knapsack_factor", load_balance_knapsack_factor); + } utils::parser::queryWithParser(pp_algo, "load_balance_efficiency_ratio_threshold", load_balance_efficiency_ratio_threshold); load_balance_costs_update_algo = static_cast(GetAlgorithmInteger(pp_algo, "load_balance_costs_update")); @@ -1313,8 +1315,9 @@ WarpX::ReadParameters () pp_warpx, "sort_bin_size", vect_sort_bin_size, 0, AMREX_SPACEDIM); if (sort_bin_size_is_specified){ - for (int i=0; i(WarpX::do_multi_J_n_depositions); + if (WarpX::do_multi_J) { solver_dt /= static_cast(WarpX::do_multi_J_n_depositions); } auto pss = std::make_unique(lev, realspace_ba, @@ -2704,7 +2708,7 @@ void WarpX::AllocLevelSpectralSolver (amrex::Vector(WarpX::do_multi_J_n_depositions); + if (WarpX::do_multi_J) { solver_dt /= static_cast(WarpX::do_multi_J_n_depositions); } auto pss = std::make_unique(lev, realspace_ba, @@ -3191,8 +3195,8 @@ bool WarpX::isAnyBoundaryPML() { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PML) return true; - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PML) return true; + if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PML) { return true; } + if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PML) { return true; } } return false; } diff --git a/Source/ablastr/coarsen/average.H b/Source/ablastr/coarsen/average.H index 315938acb48..621b4cb54e2 100644 --- a/Source/ablastr/coarsen/average.H +++ b/Source/ablastr/coarsen/average.H @@ -72,20 +72,22 @@ namespace ablastr::coarsen::average // Compute number of points for (int l = 0; l < 3; ++l) { - if (cr[l] == 1) + if (cr[l] == 1) { np[l] = 1; // no coarsening - else + } else { np[l] = cr[l] * (1 - sf[l]) * (1 - sc[l]) // cell-centered + (2 * (cr[l] - 1) + 1) * sf[l] * sc[l]; // nodal + } } // Compute starting indices of source array (fine) for (int l = 0; l < 3; ++l) { - if (cr[l] == 1) + if (cr[l] == 1) { idx_min[l] = ic[l]; // no coarsening - else + } else { idx_min[l] = ic[l] * cr[l] * (1 - sf[l]) * (1 - sc[l]) // cell-centered + (ic[l] * cr[l] - cr[l] + 1) * sf[l] * sc[l]; // nodal + } } // Auxiliary integer variables diff --git a/Source/ablastr/coarsen/sample.H b/Source/ablastr/coarsen/sample.H index 6ef0962168a..80eac14e833 100644 --- a/Source/ablastr/coarsen/sample.H +++ b/Source/ablastr/coarsen/sample.H @@ -67,14 +67,14 @@ namespace ablastr::coarsen::sample // Compute number of points for ( int l = 0; l < 3; ++l ) { - if ( cr[l] == 1 ) np[l] = 1+amrex::Math::abs(sf[l]-sc[l]); // no coarsening - else np[l] = 2-sf[l]; + if ( cr[l] == 1 ) { np[l] = 1+amrex::Math::abs(sf[l]-sc[l]); // no coarsening + } else { np[l] = 2-sf[l]; } } // Compute starting indices of source array (fine) for ( int l = 0; l < 3; ++l ) { - if ( cr[l] == 1 ) idx_min[l] = ic[l]-sc[l]*(1-sf[l]); // no coarsening - else idx_min[l] = ic[l]*cr[l]+static_cast(cr[l]/2)*(1-sc[l])-(1-sf[l]); + if ( cr[l] == 1 ) { idx_min[l] = ic[l]-sc[l]*(1-sf[l]); // no coarsening + } else { idx_min[l] = ic[l]*cr[l]+static_cast(cr[l]/2)*(1-sc[l])-(1-sf[l]); } } // Auxiliary integer variables diff --git a/Source/ablastr/coarsen/sample.cpp b/Source/ablastr/coarsen/sample.cpp index ab5b135309b..b5661037c7e 100644 --- a/Source/ablastr/coarsen/sample.cpp +++ b/Source/ablastr/coarsen/sample.cpp @@ -41,9 +41,10 @@ namespace ablastr::coarsen::sample const amrex::IntVect stag_src = mf_src.boxArray().ixType().toIntVect(); const amrex::IntVect stag_dst = mf_dst.boxArray().ixType().toIntVect(); - if ( crse_ratio > amrex::IntVect(1) ) + if ( crse_ratio > amrex::IntVect(1) ) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( ngrowvect == amrex::IntVect(0), "option of filling guard cells of destination MultiFab with coarsening not supported for this interpolation" ); + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( mf_src.nGrowVect() >= stag_dst-stag_src+ngrowvect, "source fine MultiFab does not have enough guard cells for this interpolation" ); @@ -118,9 +119,9 @@ namespace ablastr::coarsen::sample "source MultiFab converted to staggering of destination MultiFab is not coarsenable" ); ba_tmp.coarsen( crse_ratio ); - if ( ba_tmp == mf_dst.boxArray() and mf_src.DistributionMap() == mf_dst.DistributionMap() ) + if ( ba_tmp == mf_dst.boxArray() and mf_src.DistributionMap() == mf_dst.DistributionMap() ) { Loop( mf_dst, mf_src, dcomp, scomp, ncomp, ngrowvect, crse_ratio ); - else + } else { // Cannot coarsen into MultiFab with different BoxArray or DistributionMapping: // 1) create temporary MultiFab on coarsened version of source BoxArray with same DistributionMapping diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 06416168e88..eaeadfbb2fb 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -162,7 +162,7 @@ computePhi (amrex::Vector const & rho, const bool always_use_bnorm = (max_norm_b > 0); if (!always_use_bnorm) { - if (absolute_tolerance == 0.0) absolute_tolerance = amrex::Real(1e-6); + if (absolute_tolerance == 0.0) { absolute_tolerance = amrex::Real(1e-6); } ablastr::warn_manager::WMRecordWarning( "ElectrostaticSolver", "Max norm of rho is 0", @@ -306,9 +306,11 @@ computePhi (amrex::Vector const & rho, } // Run additional operations, such as calculation of the E field for embedded boundaries - if constexpr (!std::is_same::value) - if (post_phi_calculation.has_value()) + if constexpr (!std::is_same::value) { + if (post_phi_calculation.has_value()) { post_phi_calculation.value()(mlmg, lev); + } + } } // loop over lev(els) } diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index 6f824c3be34..a8833b070b8 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -122,7 +122,7 @@ computeVectorPotential ( amrex::Vector > co const bool always_use_bnorm = (max_comp_J > 0); if (!always_use_bnorm) { - if (absolute_tolerance == 0.0) absolute_tolerance = amrex::Real(1e-6); + if (absolute_tolerance == 0.0) { absolute_tolerance = amrex::Real(1e-6); } ablastr::warn_manager::WMRecordWarning( "MagnetostaticSolver", "Max norm of J is 0", @@ -242,9 +242,11 @@ computeVectorPotential ( amrex::Vector > co curr[lev][adim]->mult(-1._rt/ablastr::constant::SI::mu0); } // Loop over adim // Run additional operations, such as calculation of the B fields for embedded boundaries - if constexpr (!std::is_same::value) - if (post_A_calculation.has_value()) + if constexpr (!std::is_same::value) { + if (post_A_calculation.has_value()) { post_A_calculation.value()(mlmg, lev); + } + } } // loop over lev(els) } } // namepace Magnetostatic diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index 821a3e0d9a9..da24af94dfb 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -73,8 +73,9 @@ deposit_charge (typename T_PC::ParIterType& pti, { // deposition guards amrex::IntVect ng_rho = rho->nGrowVect(); - if (num_rho_deposition_guards.has_value()) + if (num_rho_deposition_guards.has_value()) { ng_rho = num_rho_deposition_guards.value(); + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(ng_rho <= rho->nGrowVect(), "num_rho_deposition_guards are larger than allocated!"); // particle shape @@ -82,14 +83,16 @@ deposit_charge (typename T_PC::ParIterType& pti, // used for MR when we want to deposit for a subset of the particles on the level in the // current box; with offset, we start at a later particle index - if (!np_to_deposit.has_value()) + if (!np_to_deposit.has_value()) { np_to_deposit = pti.numParticles(); + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(np_to_deposit.value() + offset <= pti.numParticles(), "np_to_deposit + offset are out-of-bounds for particle iterator"); int const lev = pti.GetLevel(); - if (!depos_lev.has_value()) + if (!depos_lev.has_value()) { depos_lev = lev; + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev.value() == (lev-1)) || (depos_lev.value() == (lev )), "Deposition buffers only work for lev or lev-1"); @@ -100,7 +103,7 @@ deposit_charge (typename T_PC::ParIterType& pti, } // If no particles, do not do anything - if (np_to_deposit == 0) return; + if (np_to_deposit == 0) { return; } // Extract deposition order and check that particles shape fits within the guard cells. // NOTE: In specific situations where the staggering of rho and the charge deposition algorithm diff --git a/Source/ablastr/profiler/ProfilerWrapper.H b/Source/ablastr/profiler/ProfilerWrapper.H index f23f13773c1..6b476d78114 100644 --- a/Source/ablastr/profiler/ProfilerWrapper.H +++ b/Source/ablastr/profiler/ProfilerWrapper.H @@ -21,8 +21,9 @@ namespace ablastr::profiler AMREX_FORCE_INLINE void device_synchronize(bool const do_device_synchronize = false) { - if (do_device_synchronize) + if (do_device_synchronize) { amrex::Gpu::synchronize(); + } } /** An object that conditionally calls device_synchronize() on destruction diff --git a/Source/ablastr/utils/Communication.cpp b/Source/ablastr/utils/Communication.cpp index c33e6a1d154..a3cd9dec98a 100644 --- a/Source/ablastr/utils/Communication.cpp +++ b/Source/ablastr/utils/Communication.cpp @@ -180,7 +180,7 @@ void OverrideSync (amrex::MultiFab &mf, { BL_PROFILE("ablastr::utils::communication::OverrideSync"); - if (mf.ixType().cellCentered()) return; + if (mf.ixType().cellCentered()) { return; } if (do_single_precision_comms) { diff --git a/Source/ablastr/utils/Serialization.H b/Source/ablastr/utils/Serialization.H index 29173a8eeec..b4dd2efa8ac 100644 --- a/Source/ablastr/utils/Serialization.H +++ b/Source/ablastr/utils/Serialization.H @@ -73,8 +73,9 @@ namespace ablastr::utils::serialization ", except vectors of std::string."); put_in(static_cast(val.size()), vec); - for (const auto &el : val) + for (const auto &el : val) { put_in(el, vec); + } } } @@ -145,8 +146,9 @@ namespace ablastr::utils::serialization const auto length = get_out(it); std::vector res(length); - for (int i = 0; i < length; ++i) + for (int i = 0; i < length; ++i) { res[i] = get_out(it); + } return res; } diff --git a/Source/ablastr/utils/SignalHandling.cpp b/Source/ablastr/utils/SignalHandling.cpp index 18d43409ec3..3681eb0e171 100644 --- a/Source/ablastr/utils/SignalHandling.cpp +++ b/Source/ablastr/utils/SignalHandling.cpp @@ -146,8 +146,9 @@ SignalHandling::CheckSignals () { // Is any signal handling action configured? // If not, we can skip all handling and the MPI communication as well. - if (!m_any_signal_action_active) + if (!m_any_signal_action_active) { return; + } // We assume that signals will definitely be delivered to rank 0, // and may be delivered to other ranks as well. For coordination, @@ -185,8 +186,9 @@ SignalHandling::WaitSignals () { // Is any signal handling action configured? // If not, we can skip all handling and the MPI communication as well. - if (!m_any_signal_action_active) + if (!m_any_signal_action_active) { return; + } #if defined(AMREX_USE_MPI) BL_MPI_REQUIRE(MPI_Wait(&signal_mpi_ibcast_request, MPI_STATUS_IGNORE)); diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.cpp b/Source/ablastr/utils/msg_logger/MsgLogger.cpp index 4ba36ee82e9..629eab8c25d 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.cpp +++ b/Source/ablastr/utils/msg_logger/MsgLogger.cpp @@ -98,25 +98,27 @@ namespace std::string abl_msg_logger::PriorityToString(const Priority& priority) { - if(priority == Priority::high) + if(priority == Priority::high) { return "high"; - else if (priority == Priority::medium) + } else if (priority == Priority::medium) { return "medium"; - else + } else { return "low"; + } } Priority abl_msg_logger::StringToPriority(const std::string& priority_string) { - if(priority_string == "high") + if(priority_string == "high") { return Priority::high; - else if (priority_string == "medium") + } else if (priority_string == "medium") { return Priority::medium; - else if (priority_string == "low") + } else if (priority_string == "low") { return Priority::low; - else + } else { ABLASTR_ABORT_WITH_MESSAGE( "Priority string '" + priority_string + "' not recognized"); + } //this silences a "non-void function does not return a value in all control paths" warning return Priority::low; @@ -223,8 +225,9 @@ std::vector Logger::get_msgs() const { auto res = std::vector{}; - for (const auto& msg_w_counter : m_messages) + for (const auto& msg_w_counter : m_messages) { res.emplace_back(msg_w_counter.first); + } return res; } @@ -233,8 +236,9 @@ std::vector Logger::get_msgs_with_counter() const { auto res = std::vector{}; - for (const auto& msg : m_messages) + for (const auto& msg : m_messages) { res.emplace_back(MsgWithCounter{msg.first, msg.second}); + } return res; } @@ -246,8 +250,9 @@ Logger::collective_gather_msgs_with_counter_and_ranks() const #ifdef AMREX_USE_MPI // Trivial case of only one rank - if (m_num_procs == 1) + if (m_num_procs == 1) { return one_rank_gather_msgs_with_counter_and_ranks(); + } // Find out who is the "gather rank" and how many messages it has const auto my_msgs = get_msgs(); @@ -256,8 +261,9 @@ Logger::collective_gather_msgs_with_counter_and_ranks() const find_gather_rank_and_its_msgs(how_many_msgs); // If the "gather rank" has zero messages there are no messages at all - if(gather_rank_how_many_msgs == 0) + if(gather_rank_how_many_msgs == 0) { return std::vector{}; + } // All the ranks receive the msgs of the "gather rank" as a byte array const auto serialized_gather_rank_msgs = @@ -347,7 +353,7 @@ Logger::compute_msgs_with_counter_and_ranks( const std::vector& displacements, const int gather_rank) const { - if(m_rank != gather_rank) return std::vector{}; + if(m_rank != gather_rank) { return std::vector{}; } std::vector msgs_with_counter_and_ranks; @@ -369,8 +375,9 @@ Logger::compute_msgs_with_counter_and_ranks( #pragma omp parallel for #endif for(int rr = 0; rr < m_num_procs; ++rr){ //for each rank - if(rr == gather_rank) // (skip gather_rank) + if(rr == gather_rank) { // (skip gather_rank) continue; + } // get counters generated by rank rr auto it = all_data.begin() + displacements[rr]; @@ -461,8 +468,9 @@ void Logger::swap_with_io_rank( if (gather_rank != m_io_rank){ if(m_rank == gather_rank){ auto package = std::vector{}; - for (const auto& el: msgs_with_counter_and_ranks) + for (const auto& el: msgs_with_counter_and_ranks) { abl_ser::put_in_vec(el.serialize(), package); + } auto package_size = static_cast(package.size()); amrex::ParallelDescriptor::Send(&package_size, 1, m_io_rank, 0); @@ -510,9 +518,10 @@ get_serialized_gather_rank_msgs( amrex::ParallelDescriptor::Bcast( &size_serialized_gather_rank_msgs, 1, gather_rank); - if (!is_gather_rank) + if (!is_gather_rank) { serialized_gather_rank_msgs.resize( size_serialized_gather_rank_msgs); + } amrex::ParallelDescriptor::Bcast( serialized_gather_rank_msgs.data(), @@ -554,9 +563,10 @@ compute_package_for_gather_rank( // Add the additional messages seen by the current rank to the package abl_ser::put_in(static_cast(msgs_to_send.size()), package); - for (const auto& el : msgs_to_send) + for (const auto& el : msgs_to_send) { abl_ser::put_in_vec( MsgWithCounter{el.first, el.second}.serialize(), package); + } return package; } diff --git a/Source/ablastr/warn_manager/WarnManager.cpp b/Source/ablastr/warn_manager/WarnManager.cpp index 889f2df848d..ee052ded299 100644 --- a/Source/ablastr/warn_manager/WarnManager.cpp +++ b/Source/ablastr/warn_manager/WarnManager.cpp @@ -29,15 +29,16 @@ namespace const abl_msg_logger::Priority& priority) { using namespace abl_msg_logger; - if (priority == Priority::low) + if (priority == Priority::low) { return WarnPriority::low; - else if (priority == Priority::medium) + } else if (priority == Priority::medium) { return WarnPriority::medium; - else if (priority == Priority::high) + } else if (priority == Priority::high) { return WarnPriority::high; - else + } else { ABLASTR_ABORT_WITH_MESSAGE( "Parsing Priority to WarnPriority has failed"); + } return WarnPriority::high; } @@ -59,10 +60,11 @@ void WarnManager::RecordWarning( WarnPriority priority) { auto msg_priority = abl_msg_logger::Priority::high; - if(priority == WarnPriority::low) + if(priority == WarnPriority::low) { msg_priority = abl_msg_logger::Priority::low; - else if(priority == WarnPriority::medium) + } else if(priority == WarnPriority::medium) { msg_priority = abl_msg_logger::Priority::medium; + } if(m_always_warn_immediately){ @@ -86,10 +88,11 @@ void WarnManager::RecordWarning( if(m_abort_on_warning_threshold){ auto abort_priority = abl_msg_logger::Priority::high; - if(m_abort_on_warning_threshold == WarnPriority::low) + if(m_abort_on_warning_threshold == WarnPriority::low) { abort_priority = abl_msg_logger::Priority::low; - else if(m_abort_on_warning_threshold == WarnPriority::medium) + } else if(m_abort_on_warning_threshold == WarnPriority::medium) { abort_priority = abl_msg_logger::Priority::medium; + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( msg_priority < abort_priority, @@ -130,8 +133,9 @@ std::string WarnManager::PrintGlobalWarnings(const std::string& when) const auto all_warnings = m_p_logger->collective_gather_msgs_with_counter_and_ranks(); - if(m_rank != amrex::ParallelDescriptor::IOProcessorNumber()) + if(m_rank != amrex::ParallelDescriptor::IOProcessorNumber()) { return "[see I/O rank message]"; + } std::sort(all_warnings.begin(), all_warnings.end(), [](const auto& a, const auto& b){ @@ -217,23 +221,25 @@ std::string WarnManager::PrintWarnMsg( { std::stringstream ss; ss << "* --> "; - if (msg_with_counter.msg.priority == abl_msg_logger::Priority::high) + if (msg_with_counter.msg.priority == abl_msg_logger::Priority::high) { ss << "[!!!]"; - else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::medium) + } else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::medium) { ss << "[!! ]"; - else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::low) + } else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::low) { ss << "[! ]"; - else + } else { ss << "[???]"; + } ss << " [" + msg_with_counter.msg.topic << "] "; - if(msg_with_counter.counter == 2) + if(msg_with_counter.counter == 2) { ss << "[raised twice]\n"; - else if(msg_with_counter.counter == 1) + } else if(msg_with_counter.counter == 1) { ss << "[raised once]\n"; - else + } else { ss << "[raised " << msg_with_counter.counter << " times]\n"; + } ss << MsgFormatter(msg_with_counter.msg.text, warn_line_size, warn_tab_size); @@ -248,8 +254,9 @@ std::string WarnManager::PrintWarnMsg( std::string raised_by = "@ Raised by: "; if (!msg_with_counter_and_ranks.all_ranks){ - for (const auto rr : msg_with_counter_and_ranks.ranks) + for (const auto rr : msg_with_counter_and_ranks.ranks) { raised_by += " " + std::to_string(rr); + } } else{ raised_by += "ALL\n"; @@ -296,8 +303,9 @@ WarnManager::MsgFormatter( msg, line_size-prefix_length); std::stringstream ss_out; - for (const auto& line : wrapped_text) + for (const auto& line : wrapped_text) { ss_out << prefix << line << "\n"; + } return ss_out.str(); } From aab5c894ef852297eae24b707a0b9c16cee9d7bc Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Thu, 14 Dec 2023 01:36:26 -0800 Subject: [PATCH 041/176] Clang-Tidy: readability-inconsistent-declaration-parameter-name (#4514) --- .clang-tidy | 1 - Source/Diagnostics/BTDiagnostics.H | 4 ++-- Source/Diagnostics/BTDiagnostics.cpp | 6 +++--- .../FlushFormats/FlushFormatPlotfile.H | 4 ++-- Source/Diagnostics/WarpXOpenPMD.H | 6 +++--- Source/Evolve/WarpXEvolve.cpp | 8 ++++---- .../FiniteDifferenceSolver.H | 8 ++++---- .../HybridPICModel/HybridPICModel.H | 8 ++++---- Source/Fluids/WarpXFluidContainer.H | 4 ++-- Source/Particles/PhysicalParticleContainer.H | 18 +++++++++--------- Source/WarpX.H | 14 +++++++------- 11 files changed, 40 insertions(+), 41 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 1fb2f6e93c1..4e69b12d905 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -41,7 +41,6 @@ Checks: ' -readability-function-cognitive-complexity, -readability-identifier-length, -readability-implicit-bool-conversion, - -readability-inconsistent-declaration-parameter-name, -readability-isolate-declaration, -readability-magic-numbers, -readability-make-member-function-const, diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index f72e7aa8b07..8940a89711a 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -380,8 +380,8 @@ private: /** Interleave meta-data of the buffer multifabs to be consistent * with the merged plotfile lab-frame data. */ - void InterleaveFabArrayHeader( std::string Buffer_FabHeaderFilename, - std::string snapshot_FabHeaderFilename, + void InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, + std::string snapshot_FabHeader_path, std::string newsnapshot_FabFilename); /** Interleave lab-frame metadata of the species header file in the buffers to * be consistent with the merged plotfile lab-frame data diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 4f83863b999..fd939b0370b 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -1336,9 +1336,9 @@ BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_pat void -BTDiagnostics::InterleaveFabArrayHeader(std::string Buffer_FabHeader_path, - std::string snapshot_FabHeader_path, - std::string newsnapshot_FabFilename) +BTDiagnostics::InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, + std::string snapshot_FabHeader_path, + std::string newsnapshot_FabFilename) { BTDMultiFabHeaderImpl snapshot_FabHeader(snapshot_FabHeader_path); snapshot_FabHeader.ReadMultiFabHeader(); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H index 5e7d9b7b8e5..486dcc3b5ee 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H @@ -46,12 +46,12 @@ public: const std::string& plotfilename, bool plot_raw_fields_guards) const; /** \brief Write particles data to file. - * \param[in] filename name of output directory + * \param[in] dir name of output directory * \param[in] particle_diags Each element of this vector handles output of 1 species. * \param[in] time the simulation time on the coarsest level * \param[in] isBTD whether this is a back-transformed diagnostic */ - void WriteParticles(const std::string& filename, + void WriteParticles(const std::string& dir, const amrex::Vector& particle_diags, amrex::Real time, bool isBTD = false) const; diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 4675e219fdd..bf83c1ebb0b 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -205,15 +205,15 @@ private: * @param[in] varname name from WarpX * @param[out] field_name field name for openPMD-api output * @param[in] comp_name comp name for openPMD-api output - * @param[in] is_theta_mode indicate if this field will be output with theta - * modes (instead of a reconstructed 2D slice) + * @param[in] var_in_theta_mode indicate if this field will be output with theta + * modes (instead of a reconstructed 2D slice) */ void GetMeshCompNames ( int meshLevel, const std::string& varname, std::string& field_name, std::string& comp_name, - bool is_theta_mode + bool var_in_theta_mode ) const; /** This function sets up the entries for storing the particle positions and global IDs diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index e57eeaf0e0a..119dfab5d99 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -770,7 +770,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) * */ void -WarpX::OneStep_sub1 (Real curtime) +WarpX::OneStep_sub1 (Real cur_time) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electrostatic_solver_id == ElectrostaticSolverAlgo::None, @@ -784,7 +784,7 @@ WarpX::OneStep_sub1 (Real curtime) const int coarse_lev = 0; // i) Push particles and fields on the fine patch (first fine step) - PushParticlesandDeposit(fine_lev, curtime, DtType::FirstHalf); + PushParticlesandDeposit(fine_lev, cur_time, DtType::FirstHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } @@ -815,7 +815,7 @@ WarpX::OneStep_sub1 (Real curtime) // ii) Push particles on the coarse patch and mother grid. // Push the fields on the coarse patch and mother grid // by only half a coarse step (first half) - PushParticlesandDeposit(coarse_lev, curtime, DtType::Full); + PushParticlesandDeposit(coarse_lev, cur_time, DtType::Full); StoreCurrent(coarse_lev); AddCurrentFromFineLevelandSumBoundary(current_fp, current_cp, current_buf, coarse_lev); AddRhoFromFineLevelandSumBoundary(rho_fp, rho_cp, charge_buf, coarse_lev, 0, ncomps); @@ -845,7 +845,7 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryAux(guard_cells.ng_UpdateAux); // iv) Push particles and fields on the fine patch (second fine step) - PushParticlesandDeposit(fine_lev, curtime + dt[fine_lev], DtType::SecondHalf); + PushParticlesandDeposit(fine_lev, cur_time + dt[fine_lev], DtType::SecondHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index 612267b50a3..861b2648c1e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -145,7 +145,7 @@ class FiniteDifferenceSolver * \param[in] Pefield scalar electron pressure MultiFab at a given level * \param[in] edge_lengths length of edges along embedded boundaries * \param[in] lev level number for the calculation - * \param[in] hybrid_pic_model instance of the hybrid-PIC model + * \param[in] hybrid_model instance of the hybrid-PIC model * \param[in] include_resistivity_term boolean flag for whether to include resistivity */ void HybridPICSolveE ( std::array< std::unique_ptr, 3>& Efield, @@ -156,7 +156,7 @@ class FiniteDifferenceSolver std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); /** @@ -240,7 +240,7 @@ class FiniteDifferenceSolver std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); template @@ -345,7 +345,7 @@ class FiniteDifferenceSolver std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); template diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 22f6e130486..c7ea9b46458 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -121,12 +121,12 @@ public: * charge density (and assumption of quasi-neutrality) using the user * specified electron equation of state. * - * \param[out] Pe scalar electron pressure MultiFab at a given level - * \param[in] rhofield scalar ion charge density Multifab at a given level + * \param[out] Pe_filed scalar electron pressure MultiFab at a given level + * \param[in] rho_field scalar ion charge density Multifab at a given level */ void FillElectronPressureMF ( - std::unique_ptr const& Pe, - amrex::MultiFab* const& rhofield ); + std::unique_ptr const& Pe_field, + amrex::MultiFab* const& rho_field ); // Declare variables to hold hybrid-PIC model parameters /** Number of substeps to take when evolving B */ diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H index 17d541f180f..04ec4d9e80d 100644 --- a/Source/Fluids/WarpXFluidContainer.H +++ b/Source/Fluids/WarpXFluidContainer.H @@ -99,12 +99,12 @@ public: * \param[in] Bx Yee magnetic field (x) * \param[in] By Yee magnetic field (y) * \param[in] Bz Yee magnetic field (z) - * \param[in] cur_time Current time + * \param[in] t Current time */ void GatherAndPush (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::Real cur_time); + amrex::Real t); /** * DepositCurrent interpolates the fluid current density comps. onto the Yee grid and diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index ef642fc718a..04c9682486c 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -286,12 +286,12 @@ public: * \param Bx Field array before filtering (not modified) * \param By Field array before filtering (not modified) * \param Bz Field array before filtering (not modified) - * \param exfab pointer to the Ex field (modified) - * \param eyfab pointer to the Ey field (modified) - * \param ezfab pointer to the Ez field (modified) - * \param bxfab pointer to the Bx field (modified) - * \param byfab pointer to the By field (modified) - * \param bzfab pointer to the Bz field (modified) + * \param ex_ptr pointer to the Ex field (modified) + * \param ey_ptr pointer to the Ey field (modified) + * \param ez_ptr pointer to the Ez field (modified) + * \param bx_ptr pointer to the Bx field (modified) + * \param by_ptr pointer to the By field (modified) + * \param bz_ptr pointer to the Bz field (modified) * * The NCI Godfrey filter is applied on Ex, the result is stored in filtered_Ex * and the pointer exfab is modified (before this function is called, it points to Ex @@ -307,9 +307,9 @@ public: const amrex::FArrayBox& Ex, const amrex::FArrayBox& Ey, const amrex::FArrayBox& Ez, const amrex::FArrayBox& Bx, const amrex::FArrayBox& By, const amrex::FArrayBox& Bz, - amrex::FArrayBox const * & exfab, amrex::FArrayBox const * & eyfab, - amrex::FArrayBox const * & ezfab, amrex::FArrayBox const * & bxfab, - amrex::FArrayBox const * & byfab, amrex::FArrayBox const * & bzfab); + amrex::FArrayBox const * & ex_ptr, amrex::FArrayBox const * & ey_ptr, + amrex::FArrayBox const * & ez_ptr, amrex::FArrayBox const * & bx_ptr, + amrex::FArrayBox const * & by_ptr, amrex::FArrayBox const * & bz_ptr); /** * \brief This function determines if resampling should be done for the current species, and diff --git a/Source/WarpX.H b/Source/WarpX.H index bf39d9275bf..c644aa67982 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1173,8 +1173,8 @@ protected: //! Make a new level from scratch using provided BoxArray and //! DistributionMapping. Only used during initialization. Called //! by AmrCoreInitFromScratch. - void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm) final; + void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& new_grids, + const amrex::DistributionMapping& new_dmap) final; //! Make a new level using provided BoxArray and //! DistributionMapping and fill with interpolated coarse level @@ -1223,13 +1223,13 @@ private: void AddExternalFields (); - void OneStep_nosub (amrex::Real t); - void OneStep_sub1 (amrex::Real t); + void OneStep_nosub (amrex::Real cur_time); + void OneStep_sub1 (amrex::Real cur_time); /** * \brief Perform one PIC iteration, with the multiple J deposition per time step */ - void OneStep_multiJ (amrex::Real t); + void OneStep_multiJ (amrex::Real cur_time); void RestrictCurrentFromFineToCoarsePatch ( const amrex::Vector,3>>& J_fp, @@ -1298,8 +1298,8 @@ private: void InitFromScratch (); - void AllocLevelData (int lev, const amrex::BoxArray& new_grids, - const amrex::DistributionMapping& new_dmap); + void AllocLevelData (int lev, const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm); [[nodiscard]] amrex::DistributionMapping GetRestartDMap (const std::string& chkfile, const amrex::BoxArray& ba, int lev) const; From af5079073e3941c6c2efe6b4b88b6097a2df310b Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 14 Dec 2023 16:25:33 +0100 Subject: [PATCH 042/176] Clang-tidy: enable no-malloc check in clang-tidy CI test (#4518) --- .clang-tidy | 1 - 1 file changed, 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 4e69b12d905..831f16e1322 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -14,7 +14,6 @@ Checks: ' -cppcoreguidelines-avoid-non-const-global-variables, -cppcoreguidelines-init-variables, -cppcoreguidelines-macro-usage, - -cppcoreguidelines-no-malloc, -cppcoreguidelines-narrowing-conversions, -cppcoreguidelines-non-private-member-variables-in-classes, -cppcoreguidelines-owning-memory, From e97330368d3edfb5ff1afaf6c9478c19016f6449 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 14 Dec 2023 13:36:06 -0800 Subject: [PATCH 043/176] Update descriptions of hybrid-PIC examples (#4479) * update descriptions of hybrid-PIC examples * requested changes as well as updates to the magnetic reconnection example README --- Docs/source/usage/examples.rst | 13 ++- Examples/Tests/ohm_solver_EM_modes/README.rst | 106 +++++++++++++++--- .../ohm_solver_ion_Landau_damping/README.rst | 51 ++++++--- .../README.rst | 65 ++++++++--- .../README.rst | 33 +++--- 5 files changed, 207 insertions(+), 61 deletions(-) diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index eeee6ddf092..5490068e448 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -113,9 +113,16 @@ Coming soon: Kinetic-fluid Hybrid Models ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Several examples and benchmarks of the kinetic-fluid hybrid model are shown below. -The first few examples are replications of the verification tests described in :cite:t:`ex-MUNOZ2018`. -The hybrid-PIC model was added to WarpX in `PR #3665 `_ - the figures in the examples below were generated at that time. +WarpX includes a reduced plasma model in which electrons are treated as a massless +fluid while ions are kinetically evolved, and Ohm's law is used to calculate +the electric field. This model is appropriate for problems in which ion kinetics +dominate (ion cyclotron waves, for instance). See the +:ref:`theory section ` for more details. Several +examples and benchmarks of this kinetic-fluid hybrid model are provided below. +A few of the examples are replications of the verification tests described in +:cite:t:`ex-MUNOZ2018`. The hybrid-PIC model was added to WarpX in +`PR #3665 `_ - the figures in the +examples below were generated at that time. .. toctree:: :maxdepth: 1 diff --git a/Examples/Tests/ohm_solver_EM_modes/README.rst b/Examples/Tests/ohm_solver_EM_modes/README.rst index a45cecb3c88..034ee5815f0 100644 --- a/Examples/Tests/ohm_solver_EM_modes/README.rst +++ b/Examples/Tests/ohm_solver_EM_modes/README.rst @@ -1,11 +1,56 @@ .. _examples-ohm-solver-em-modes: -Ohm Solver: Electromagnetic modes +Ohm solver: Electromagnetic modes ================================= In this example a simulation is seeded with a thermal plasma while an initial magnetic field is applied in either the :math:`z` or :math:`x` direction. The simulation is progressed for a large number of steps and the resulting fields are -analyzed for mode excitations. +Fourier analyzed for Alfvén mode excitations. + +Run +--- + +The same input script can be used for 1d, 2d or 3d Cartesian simulations as well +as replicating either the parallel propagating or ion-Bernstein modes as indicated below. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Parallel propagating waves + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --bdir z + + .. tab-item:: Perpendicular propagating waves + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --bdir {x/y} + +Analyze +------- + +The following script reads the simulation output from the above example, performs +Fourier transforms of the field data and compares the calculated spectrum +to the theoretical dispersions. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis.py``. Right and left circularly polarized electromagnetic waves are supported through the cyclotron motion of the ions, except in a region of thermal resonances as indicated on the plot below. @@ -14,32 +59,61 @@ in a region of thermal resonances as indicated on the plot below. :alt: Parallel EM modes in thermal ion plasma :width: 70% + Calculated Alvén waves spectrum with the theoretical dispersions overlaid. + Perpendicularly propagating modes are also supported, commonly referred to as ion-Bernstein modes. .. figure:: https://user-images.githubusercontent.com/40245517/231217944-7d12b8d4-af4b-44f8-a1b9-a2b59ce3a1c2.png - :alt: Perpendicular EM modes in thermal ion plasma + :alt: Perpendicular modes in thermal ion plasma :width: 50% -The input file for these examples and the corresponding analysis can be found at: + Calculated ion Bernstein waves spectrum with the theoretical dispersion overlaid. + +Ohm solver: Cylindrical normal modes +==================================== + +A RZ-geometry example case for normal modes propagating along an applied magnetic +field in a cylinder is also available. The analytical solution for these modes +are described in :cite:t:`ex-Stix1992` Chapter 6, Sec. 2. + +Run +--- -* :download:`EM modes input ` -* :download:`Analysis script ` +The following script initializes a thermal plasma in a metallic cylinder with +periodic boundaries at the cylinder ends. -The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the parallel propagating or -ion-Bernstein modes as indicated below. +.. dropdown:: Script ``PICMI_inputs_rz.py`` - .. code-block:: bash + .. literalinclude:: PICMI_inputs_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py``. - python3 PICMI_inputs.py -dim {1/2/3} --bdir {x/y/z} +The example can be executed using: -A RZ-geometry example case for normal modes propagating along an applied magnetic field in a cylinder is also available. -The analytical solution for these modes are described in :cite:t:`ex-Stix1992` Chapter 6, Sec. 2. +.. code-block:: bash + + python3 PICMI_inputs_rz.py + +Analyze +------- + +After the simulation completes the following script can be used to analyze the +field evolution and extract the normal mode dispersion relation. It performs a +standard Fourier transform along the cylinder axis and a Hankel transform in the +radial direction. + +.. dropdown:: Script ``analysis_rz.py`` + + .. literalinclude:: analysis_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis_rz.py``. + +The following figure was produced with the above analysis script, showing excellent +agreement between the calculated and theoretical dispersion relations. .. figure:: https://user-images.githubusercontent.com/40245517/259251824-33e78375-81d8-410d-a147-3fa0498c66be.png :alt: Normal EM modes in a metallic cylinder :width: 90% -The input file for this example and corresponding analysis can be found at: - -* :download:`Cylindrical modes input ` -* :download:`Analysis script ` + Cylindrical normal mode dispersion comparing the calculated spectrum with the + theoretical one. diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst index 7e1fc21d935..dd4f94b4edf 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst @@ -1,25 +1,50 @@ .. _examples-ohm-solver-ion-landau-damping: -Ohm Solver: Ion Landau Damping +Ohm solver: Ion Landau Damping ============================== -Landau damping is a well known process in which electrostatic (acoustic) waves are damped by transferring energy to particles satisfying a resonance condition. -The process can be simulated by seeding a plasma with a specific acoustic mode (density perturbation) and tracking the strength of the mode as a function of time. -The figure below shows a set of such simulations with parameters matching those described in section 4.5 of :cite:t:`ex-MUNOZ2018`. -The straight lines show the theoretical damping rate for the given temperature ratios. - -.. figure:: https://user-images.githubusercontent.com/40245517/230523935-3c8d63bd-ee69-4639-b111-f06dad5587f6.png - :alt: Ion Landau damping - :width: 70% +Landau damping is a well known process in which electrostatic (acoustic) waves are +damped by transferring energy to particles satisfying a resonance condition. +The process can be simulated by seeding a plasma with a specific acoustic mode +(density perturbation) and tracking the strength of the mode as a function of time. -The input file for these examples and the corresponding analysis can be found at: - -* :download:`Ion Landau damping input ` -* :download:`Analysis script ` +Run +--- The same input script can be used for 1d, 2d or 3d simulations and to sweep different temperature ratios. +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + .. code-block:: bash python3 PICMI_inputs.py -dim {1/2/3} --temp_ratio {value} + +Analyze +------- + +The following script extracts the amplitude of the seeded mode as a function +of time and compares it to the theoretical damping rate. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py``. + +The figure below shows a set of such simulations with parameters matching those +described in section 4.5 of :cite:t:`ex-MUNOZ2018`. +The straight lines show the theoretical damping rate for the given temperature ratios. + +.. figure:: https://user-images.githubusercontent.com/40245517/230523935-3c8d63bd-ee69-4639-b111-f06dad5587f6.png + :alt: Ion Landau damping + :width: 70% + + Decay of seeded modes as a function of time for different electron-ion temperature ratios. + The theoretical damping of the given modes are shown in dashed lines. diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/README.rst b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst index ae98787b799..59469cf4aa9 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/README.rst +++ b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst @@ -1,11 +1,57 @@ .. _examples-ohm-solver-ion-beam-instability: -Ohm Solver: Ion Beam R Instability +Ohm solver: Ion Beam R Instability ================================== In this example a low density ion beam interacts with a "core" plasma population which induces an instability. Based on the relative density between the beam and the core plasma a resonant or non-resonant condition can -be accessed. The figures below show the evolution of the y-component of the magnetic field as the beam and +be accessed. + +Run +--- + +The same input script can be used for 1d, 2d or 3d simulations as well as +replicating either the resonant or non-resonant condition as indicated below. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Resonant case + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --resonant + + .. tab-item:: Non-resonant case + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} + +Analyze +------- + +The following script reads the simulation output from the above example, performs +Fourier transforms of the field data and outputs the figures shown below. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_beam_instability/analysis.py``. + +The figures below show the evolution of the y-component of the magnetic field as the beam and core plasma interact. .. figure:: https://user-images.githubusercontent.com/40245517/217923933-6bdb65cb-7d26-40d8-8687-7dd75274bd48.png @@ -16,6 +62,8 @@ core plasma interact. :alt: Non-resonant ion beam R instability :width: 70% + Evolution of :math:`B_y` for resonant (top) and non-resonant (bottom) conditions. + The growth rates of the strongest growing modes for the resonant case are compared to theory (dashed lines) in the figure below. @@ -23,14 +71,5 @@ to theory (dashed lines) in the figure below. :alt: Resonant ion beam R instability growth rates :width: 50% -The input file for these examples and the corresponding analysis can be found at: - -* :download:`Ion beam R instability input ` -* :download:`Analysis script ` - -The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the resonant or non-resonant -condition as indicated below. - - .. code-block:: bash - - python3 PICMI_inputs.py -dim {1/2/3} --resonant + Time series of the mode amplitudes for m = 4, 5, 6 from simulation. The + theoretical growth for these modes are also shown as dashed lines. diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst index c31f9915abd..943b5bd0248 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst @@ -4,32 +4,33 @@ Ohm Solver: Magnetic Reconnection ================================= Hybrid-PIC codes are often used to simulate magnetic reconnection in space plasmas. -An example of magnetic reconnection from a force-free sheet is provided, based on the simulation described in :cite:t:`ex-Le2016`. +An example of magnetic reconnection from a force-free sheet is provided, based on +the simulation described in :cite:t:`ex-Le2016`. Run --- -This example can be run as a **Python** script: ``python3 PICMI_inputs.py``. +The following **Python** script configures and launches the simulation. -.. literalinclude:: PICMI_inputs.py - :language: python3 - :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py``. +.. dropdown:: Script ``PICMI_inputs.py`` + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py``. -Analyze -------- - -We run the following script to analyze correctness: - -.. note:: +Running the full simulation should take about 4 hours if executed on 1 V100 GPU. +For `MPI-parallel `__ runs, prefix these lines with +``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. - This section is TODO. + .. code-block:: bash + python3 PICMI_inputs.py -Visualize ---------- +Analyze +------- -You can run the following script to visualize the B-field evolution over time: +The following script extracts the reconnection rate as a function of time and +animates the evolution of the magnetic field (as shown below). .. dropdown:: Script ``analysis.py`` @@ -41,4 +42,4 @@ You can run the following script to visualize the B-field evolution over time: :alt: Magnetic reconnection. :width: 70% - Magnetic reconnection. + Magnetic reconnection from a force-free sheet. From 38794d198abc1a692f02c12b9279c8cac4a68fa8 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Thu, 14 Dec 2023 13:38:58 -0800 Subject: [PATCH 044/176] Bibliography: improved citations (#4513) * Fix up months and whitespace alignment * Replace arxiv citations with published versions * Author initials, Physics of Plasmas, delete empty lines, replace et al * Comment urlretrieve in conf.py * VayJCP13 redundant * refs.bib author touch ups * Uncomment conf.py * alphabetize refs.bib entries * Remove more "et al" from bib files * Whitespaces and page numbers --- .../Boosted_frame/Boosted_frame.tex | 8 +- Docs/source/latex_theory/allbibs.bib | 568 ++++++++---------- Docs/source/refs.bib | 344 +++++------ Docs/source/theory/boosted_frame.rst | 4 +- Docs/source/theory/picsar_theory.rst | 4 +- 5 files changed, 446 insertions(+), 482 deletions(-) diff --git a/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex b/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex index 471863ac3ee..fe8365d15dd 100644 --- a/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex +++ b/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex @@ -175,7 +175,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} the simulation, and the artificial ``bump'' is again an arbitrary correction that departs from the underlying physics. -A new scheme was recently proposed, in \cite{KirchenARXIV2016,LeheARXIV2016}, which +A new scheme was recently proposed, in \cite{KirchenPOP2016,LehePRE2016}, which completely eliminates the NCI for a plasma drifting at a uniform relativistic velocity -- with no arbitrary correction -- by simply integrating the PIC equations in \emph{Galilean coordinates} (also known as @@ -217,7 +217,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} \emph{themselves} are not only translated but in this case, the physical equations are modified accordingly. Most importantly, the assumed time evolution of the current $\vec{J}$ within one timestep is different in a standard PSATD scheme with moving -window and in a Galilean PSATD scheme \cite{LeheARXIV2016}. +window and in a Galilean PSATD scheme \cite{LehePRE2016}. In the Galilean coordinates $\vec{x}'$, the equations of particle motion and the Maxwell equations take the form @@ -235,7 +235,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} Integrating these equations from $t=n\Delta t$ to $t=(n+1)\Delta t$ results in the following update equations (see -\cite{LeheARXIV2016} for the details of the derivation): +\cite{LehePRE2016} for the details of the derivation): % \begin{subequations} \begin{align} @@ -271,5 +271,5 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} Note that, in the limit $\vgal=\vec{0}$, (\ref{eq:disc-maxwell1}) and (\ref{eq:disc-maxwell2}) reduce to the standard PSATD equations \cite{Habericnsp73}, as expected. -As shown in \cite{KirchenARXIV2016,LeheARXIV2016}, +As shown in \cite{KirchenPOP2016,LehePRE2016}, the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. diff --git a/Docs/source/latex_theory/allbibs.bib b/Docs/source/latex_theory/allbibs.bib index 32ede278416..a3c355aae25 100644 --- a/Docs/source/latex_theory/allbibs.bib +++ b/Docs/source/latex_theory/allbibs.bib @@ -4,11 +4,11 @@ BibTeX export options can be customized via Preferences -> BibTeX in Mendeley Desktop @article{QuickpicParallel, -author = {Feng, B. and Huang, C. and Decyk, V. and Mori, W.B. and Muggli, P. and Katsouleas, T.}, +author = {Feng, B. and Huang, C. and Decyk, V. and Mori, W. B. and Muggli, P. and Katsouleas, T.}, doi = {10.1016/j.jcp.2009.04.019}, issn = {00219991}, journal = {Journal of Computational Physics}, -month = {aug}, +month = {Aug}, number = {15}, pages = {5340--5348}, title = {{Enhancing parallel quasi-static particle-in-cell simulations with a pipelining algorithm}}, @@ -26,11 +26,11 @@ @book{HockneyEastwoodBook year = {1988} } @article{Parkerjcp1991, -author = {Parker, Se and Birdsall, Ck}, +author = {Parker, S E and Birdsall, C K}, doi = {10.1016/0021-9991(91)90040-R}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {nov}, +month = {Nov}, number = {1}, pages = {91--102}, title = {{Numerical Error In Electron Orbits With Large Omega-Ce Delta-T}}, @@ -45,7 +45,7 @@ @inproceedings{Fawleyipac10 year = {2010} } @article{Godfreyjcp74, -author = {Godfrey, Bb}, +author = {Godfrey, B B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {4}, @@ -59,7 +59,7 @@ @article{Quickpic doi = {10.1016/J.Jcp.2006.01.039}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {2}, pages = {658--679}, title = {{Quickpic: A Highly Efficient Particle-In-Cell Code For Modeling Wakefield Acceleration In Plasmas}}, @@ -71,7 +71,7 @@ @article{Lundprstab2009 doi = {10.1103/Physrevstab.12.114801}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {nov}, +month = {Nov}, number = {11}, title = {{Generation Of Initial Kinetic Distributions For Simulation Of Long-Pulse Charged Particle Beams With High Space-Charge Intensity}}, volume = {12}, @@ -82,7 +82,7 @@ @article{CowanPRSTAB13 doi = {10.1103/PhysRevSTAB.16.041303}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {apr}, +month = {Apr}, number = {4}, title = {{Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations}}, volume = {16}, @@ -103,7 +103,7 @@ @article{YuPRL2014 author = {Yu, L.-L. and Esarey, E and Schroeder, C B and Vay, J.-L. and Benedetti, C and Geddes, C G R and Chen, M and Leemans, W P}, doi = {10.1103/PhysRevLett.112.125001}, journal = {Phys. Rev. Lett.}, -month = {mar}, +month = {Mar}, number = {12}, pages = {125001}, publisher = {American Physical Society}, @@ -119,7 +119,7 @@ @article{Vaynim2007 institution = {Univ Paris Sud Xi}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {65--69}, title = {{Self-Consistent Simulations Of Heavy-Ion Beams Interacting With Electron-Clouds}}, @@ -142,7 +142,7 @@ @article{Vayjcp2011 author = {Vay, J-L and Geddes, C G R and Cormier-Michel, E and Grote, D P}, doi = {10.1016/J.Jcp.2011.04.003}, journal = {Journal of Computational Physics}, -month = {jul}, +month = {Jul}, number = {15}, pages = {5908--5929}, title = {{Numerical Methods For Instability Mitigation In The Modeling Of Laser Wakefield Accelerators In A Lorentz-Boosted Frame}}, @@ -153,21 +153,15 @@ @article{Krallpre1993 author = {Krall, J and Ting, A and Esarey, E and Sprangle, P}, doi = {10.1103/Physreve.48.2157}, journal = {Physical Review E}, -month = {sep}, +month = {Sep}, number = {3}, pages = {2157--2161}, title = {{Enhanced Acceleration In A Self-Modulated-Laser Wake-Field Accelerator}}, volume = {48}, year = {1993} } -@article{Vayarxiv10_2, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:1011.0917V2}, -title = {{Effects Of Hyperbolic Rotation In Minkowski Space On The Modeling Of Plasma Accelerators In A Lorentz Boosted Frame}}, -year = {2010} -} @article{Dennisw1997585, -author = {W., Dennis and Hewett}, +author = {Dennis W. Hewett}, doi = {10.1006/Jcph.1997.5835}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -179,12 +173,11 @@ @article{Dennisw1997585 year = {1997} } @article{Habib2016, - archivePrefix = {arXiv}, arxivId = {1603.09303}, -author = {Habib, Salman and Roser, Robert and Gerber, Richard and Antypas, Katie and Riley, Katherine and Williams, Tim and Wells, Jack and Straatsma, Tjerk and Almgren, A. and Amundson, J. and Bailey, S. and Bard, D. and Bloom, K. and Bockelman, B. and Borgland, A. and Borrill, J. and Boughezal, R. and Brower, R. and Cowan, B. and Finkel, H. and Frontiere, N. and Fuess, S. and Ge, L. and Gnedin, N. and Gottlieb, S. and Gutsche, O. and Han, T. and Heitmann, K. and Hoeche, S. and Ko, K. and Kononenko, O. and LeCompte, T. and Li, Z. and Lukic, Z. and Mori, W. and Nugent, P. and Ng, C. -K. and Oleynik, G. and O'Shea, B. and Padmanabhan, N. and Petravick, D. and Petriello, F. J. and Power, J. and Qiang, J. and Reina, L. and Rizzo, T. J. and Ryne, R. and Schram, M. and Spentzouris, P. and Toussaint, D. and Vay, J. -L. and Viren, B. and Wurthwein, F. and Xiao, L.}, +author = {Habib, Salman and Roser, Robert and Gerber, Richard and Antypas, Katie and Riley, Katherine and Williams, Tim and Wells, Jack and Straatsma, Tjerk and Almgren, A. and Amundson, J. and Bailey, S. and Bard, D. and Bloom, K. and Bockelman, B. and Borgland, A. and Borrill, J. and Boughezal, R. and Brower, R. and Cowan, B. and Finkel, H. and Frontiere, N. and Fuess, S. and Ge, L. and Gnedin, N. and Gottlieb, S. and Gutsche, O. and Han, T. and Heitmann, K. and Hoeche, S. and Ko, K. and Kononenko, O. and LeCompte, T. and Li, Z. and Lukic, Z. and Mori, W. and Nugent, P. and Ng, C.-K. and Oleynik, G. and O'Shea, B. and Padmanabhan, N. and Petravick, D. and Petriello, F. J. and Power, J. and Qiang, J. and Reina, L. and Rizzo, T. J. and Ryne, R. and Schram, M. and Spentzouris, P. and Toussaint, D. and Vay, J.-L. and Viren, B. and Wurthwein, F. and Xiao, L.}, eprint = {1603.09303}, -month = {mar}, +month = {Mar}, title = {{ASCR/HEP Exascale Requirements Review Report}}, url = {http://arxiv.org/abs/1603.09303}, year = {2016} @@ -193,8 +186,8 @@ @article{Shadwickpop09 author = {Shadwick, B A and Schroeder, C B and Esarey, E}, doi = {10.1063/1.3124185}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56704}, title = {{Nonlinear Laser Energy Depletion In Laser-Plasma Accelerators}}, @@ -207,7 +200,7 @@ @article{Leemansaac2010 journal = {Aip Conference Proceedings}, keywords = {[978-0-7354-0853-1/10/{\$}30.00]}, pages = {3--11}, -title = {{The Berkeley Lab Laser Accelerator (Bella): A 10 Gev Laser Plasma Accelerator}}, +title = {{The Berkeley Lab Laser Accelerator (Bella): A 10 GeV Laser Plasma Accelerator}}, volume = {1299}, year = {2010} } @@ -215,7 +208,7 @@ @article{Marderjcp87 author = {Marder, B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {jan}, +month = {Jan}, number = {1}, pages = {48--55}, title = {{A Method For Incorporating Gauss Law Into Electromagnetic Pic Codes}}, @@ -232,7 +225,7 @@ @article{Ohmurapiers2010 year = {2010} } @article{Adamjcp1982, -author = {Adam, Jc and Serveniere, Ag and Langdon, Ab}, +author = {Adam, J C and Serveniere, Ag and Langdon, A B}, doi = {10.1016/0021-9991(82)90076-6}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -243,7 +236,7 @@ @article{Adamjcp1982 year = {1982} } @article{Tajimaprl79, -author = {Tajima, T and Dawson, Jm}, +author = {Tajima, T and Dawson, J M}, issn = {0031-9007}, journal = {Physical Review Letters}, number = {4}, @@ -252,13 +245,19 @@ @article{Tajimaprl79 volume = {43}, year = {1979} } -@article{Benedetti08, -author = {{Benedetti et al.}, C}, -journal = {Nuclear Inst. And Methods In Physics Research A}, -pages = {94--98}, -title = {{No Title}}, +@article{Benedetti09, +author = {C. Benedetti and P. Londrillo and V. Petrillo and L. Serafini and A. Sgattoni and P. Tomassini and G. Turchetti}, +doi = {https://doi.org/10.1016/j.nima.2009.05.064}, +issn = {0168-9002}, +journal = {Nuclear Instruments and Methods in Physics Research Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, +keywords = {PIC simulation, Laser–plasma interaction, FEL}, +note = {Compton sources for X/γ rays: Physics and applications}, +number = {1, Supplement }, +pages = {S94-S98}, +title = {PIC simulations of the production of high-quality electron beams via laser–plasma interaction}, +url = {https://www.sciencedirect.com/science/article/pii/S0168900209009784}, volume = {608}, -year = {2009} +year = {2009}, } @inproceedings{Godfreyicnsp80, author = {Godfrey, B B}, @@ -271,7 +270,7 @@ @article{Blumenfeld2007 author = {Blumenfeld, Ian and Clayton, Christopher E and Decker, Franz-Josef and Hogan, Mark J and Huang, Chengkun and Ischebeck, Rasmus and Iverson, Richard and Joshi, Chandrashekhar and Katsouleas, Thomas and Kirby, Neil and Lu, Wei and Marsh, Kenneth A and Mori, Warren B and Muggli, Patric and Oz, Erdem and Siemann, Robert H and Walz, Dieter and Zhou, Miaomiao}, issn = {0028-0836}, journal = {Nature}, -month = {feb}, +month = {Feb}, number = {7129}, pages = {741--744}, title = {{Energy doubling of 42[thinsp]GeV electrons in a metre-scale plasma wakefield accelerator}}, @@ -281,7 +280,7 @@ @article{Blumenfeld2007 } @inproceedings{Vayicap2002, annote = {7Th International Conference On Computational Accelerator Physics, Michigan State Univ, E Lansing, Mi, Oct 15-18, 2002}, -author = {Vay, Jl and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Friedman, A and Grote, D P}, booktitle = {Computational Accelerator Physics 2002}, editor = {Berz, M and Makino, K}, isbn = {0-7503-0939-3}, @@ -340,7 +339,7 @@ @inproceedings{Qiang @inproceedings{Geddespac09, address = {Vancouver, Canada}, annote = {We6Rfp075}, -author = {{Geddes et al.}, C G R}, +author = {C. G. R. Geddes and E. Cormier-Michel and E. Esarey and C. B. Schroeder and W. P. Leemans}, booktitle = {Proc. Particle Accelerator Conference}, title = {{Scaled Simulation Design Of High Quality Laser Wakefield Accelerator Stages}}, year = {2009} @@ -348,10 +347,9 @@ @inproceedings{Geddespac09 @article{LotovPRSTAB2003, author = {Lotov, K. V.}, doi = {10.1103/PhysRevSTAB.6.061301}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Lotov - 2003 - Fine wakefield structure in the blowout regime of plasma wakefield accelerators.pdf:pdf}, issn = {1098-4402}, journal = {Physical Review Special Topics - Accelerators and Beams}, -month = {jun}, +month = {Jun}, number = {6}, pages = {061301}, publisher = {American Physical Society}, @@ -361,7 +359,7 @@ @article{LotovPRSTAB2003 year = {2003} } @article{Godfreyjcp75, -author = {Godfrey, Bb}, +author = {Godfrey, B B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {1}, @@ -374,7 +372,7 @@ @article{Kishekprl2012 author = {Kishek, R A}, doi = {10.1103/Physrevlett.108.035003}, journal = {Phys. Rev. Lett.}, -month = {jan}, +month = {Jan}, number = {3}, pages = {35003}, publisher = {American Physical Society}, @@ -392,12 +390,12 @@ @article{Zhang2016 } @article{Cohennim2009, annote = {17Th International Symposium On Heavy Ion Inertial Fusion, Tokyo, Japan, Aug 04-08, 2008}, -author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J -L.}, +author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J-L}, doi = {10.1016/J.Nima.2009.03.083}, institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {53--55}, title = {{An Implicit ``Drift-Lorentz{\{}''{\}} Mover For Plasma And Beam Simulations}}, @@ -405,10 +403,11 @@ @article{Cohennim2009 year = {2009} } @article{Osiris, -author = {{Fonseca et al.}, R A}, -journal = {Lec. Notes In Comp. Sci.}, -pages = {342}, -title = {{No Title}}, +author = {Fonseca, R. A. and Silva, L. O. and Tsung, F. S. and Decyk, V. K. and Lu, W. and Ren, C. and Mori, W. B. and Deng, S. and Lee, S. and Katsouleas, T. and Adam, J. C.}, +booktitle = {Computational Science --- ICCS 2002}, +pages = {342--351}, +publisher = {Springer Berlin Heidelberg}, +title = {{OSIRIS: A Three-Dimensional, Fully Relativistic Particle in Cell Code for Modeling Plasma Based Accelerators}}, volume = {2329}, year = {2002} } @@ -439,7 +438,7 @@ @article{GeddesPRL2008 author = {Geddes, C G R and Nakamura, K and Plateau, G R and Toth, Cs. and Cormier-Michel, E and Esarey, E and Schroeder, C B and Cary, J R and Leemans, W P}, doi = {10.1103/PhysRevLett.100.215004}, journal = {Phys. Rev. Lett.}, -month = {may}, +month = {May}, number = {21}, pages = {215004}, publisher = {American Physical Society}, @@ -465,12 +464,6 @@ @misc{Huebl2015 url = {http://dx.doi.org/10.5281/zenodo.591699}, year = {2015} } -@article{Vayarxiv10_1, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:1009.2727V2}, -title = {{Modeling Laser Wakefield Accelerators In A Lorentz Boosted Frame}}, -year = {2010} -} @article{Huangscidac09, author = {Huang, C and An, W and Decyk, V K and Lu, W and Mori, W B and Tsung, F S and Tzoufras, M and Morshed, S and Antonsen, T and Feng, B and Katsouleas, T and Fonseca, R A and Martins, S F and Vieira, J and Silva, L O and Esarey, E and Geddes, C G R and Leemans, W P and Cormier-Michel, E and Vay, J.-L. and Bruhwiler, D L and Cowan, B and Cary, J R and Paul, K}, issn = {1742-6596}, @@ -494,7 +487,7 @@ @article{Cormierprstab10 title = {{Propagation Of Higher Order Modes In Plasma Channels And Shaping Of The Transverse Field In Laser Plasma Accelerators}} } @article{Yee, -author = {Yee, Ks}, +author = {Yee, K S}, issn = {0018-926X}, journal = {Ieee Transactions On Antennas And Propagation}, number = {3}, @@ -521,7 +514,7 @@ @inproceedings{Cormieraac08 booktitle = {Aip Conference Proceedings}, issn = {0094-243X}, pages = {297--302}, -title = {{Scaled Simulations Of A 10 Gev Accelerator}}, +title = {{Scaled Simulations Of A 10 GeV Accelerator}}, volume = {1086}, year = {2009} } @@ -536,11 +529,11 @@ @inproceedings{Bruhwileraac08 } @article{Vaynim2005, annote = {15Th International Symposium On Heavy Ion Inertial Fusion, Princeton, Nj, Jun 07-11, 2004}, -author = {Vay, Jl and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Friedman, A and Grote, D P}, doi = {10.1016/J.Nima.2005.01.232}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {may}, +month = {May}, number = {1-2}, pages = {347--352}, title = {{Application Of Adaptive Mesh Refinement To Pic Simulations In Heavy Ion Fusion}}, @@ -560,11 +553,11 @@ @article{BulanovSV2014 year = {2014} } @article{Furmanprstab2002, -author = {Furman, Ma and Pivi, Mtf}, +author = {Furman, M A and Pivi, M T F}, doi = {10.1103/Physrevstab.5.124404}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {dec}, +month = {Dec}, number = {12}, title = {{Probabilistic Model For The Simulation Of Secondary Electron Emission}}, volume = {5}, @@ -573,10 +566,9 @@ @article{Furmanprstab2002 @article{Qiang2014, author = {Qiang, J. and Corlett, J. and Mitchell, C. E. and Papadopoulos, C. F. and Penn, G. and Placidi, M. and Reinsch, M. and Ryne, R. D. and Sannibale, F. and Sun, C. and Venturini, M. and Emma, P. and Reiche, S.}, doi = {10.1103/PhysRevSTAB.17.030701}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Qiang et al. - 2014 - Start-to-end simulation of x-ray radiation of a next generation light source using the real number of electrons.pdf:pdf}, issn = {1098-4402}, journal = {Physical Review Special Topics - Accelerators and Beams}, -month = {mar}, +month = {Mar}, number = {3}, pages = {030701}, publisher = {American Physical Society}, @@ -601,13 +593,12 @@ @misc{Bruhwilerpc08 year = {2008} } @article{Yu2014, - author = {Yu, Peicheng and Xu, Xinlu and Decyk, Viktor K. and An, Weiming and Vieira, Jorge and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.jcp.2014.02.016}, issn = {00219991}, journal = {Journal of Computational Physics}, keywords = {Boosted frame simulation,IN-CELL CODE,INSTABILITIES,Laser wakefield accelerator,Numerical Cerenkov instability,PARTICLE SIMULATION,PLASMAS,Particle-in-cell,Plasma simulation,Spectral solver}, -month = {jun}, +month = {Jun}, pages = {124--138}, publisher = {ACADEMIC PRESS INC ELSEVIER SCIENCE, 525 B ST, STE 1900, SAN DIEGO, CA 92101-4495 USA}, title = {{Modeling of laser wakefield acceleration in Lorentz boosted frame using EM-PIC code with spectral solver}}, @@ -619,10 +610,10 @@ @article{Vaypop2011 author = {Vay, J.-L. and Geddes, C G R and Esarey, E and Schroeder, C B and Leemans, W P and Cormier-Michel, E and Grote, D P}, doi = {10.1063/1.3663841}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {dec}, +journal = {Physics of Plasmas}, +month = {Dec}, number = {12}, -title = {{Modeling Of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations}}, +title = {{Modeling Of 10 GeV-1 TeV Laser-Plasma Accelerators Using Lorentz Boosted Simulations}}, volume = {18}, year = {2011} } @@ -631,7 +622,7 @@ @article{LehePRSTAB13 doi = {10.1103/PhysRevSTAB.16.021301}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {feb}, +month = {Feb}, number = {2}, title = {{Numerical growth of emittance in simulations of laser-wakefield acceleration}}, volume = {16}, @@ -641,7 +632,7 @@ @article{Langdoncpc92 author = {Langdon, A B}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {jul}, +month = {Jul}, number = {3}, pages = {447--450}, title = {{On Enforcing Gauss Law In Electromagnetic Particle-In-Cell Codes}}, @@ -653,31 +644,29 @@ @article{Colellajcp2010 doi = {10.1016/J.Jcp.2009.07.004}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {feb}, +month = {Feb}, number = {4}, pages = {947--957}, title = {{Controlling Self-Force Errors At Refinement Boundaries For Amr-Pic}}, volume = {229}, year = {2010} } - @article{Coleieee1997, author = {Cole, J. B.}, issn = {0018-9480}, journal = {Ieee Transactions On Microwave Theory And Techniques}, -month = {jun}, +month = {Jun}, number = {6}, pages = {991--996}, title = {{A High-Accuracy Realization Of The Yee Algorithm Using Non-Standard Finite Differences}}, volume = {45}, year = {1997} } - @article{Coleieee2002, author = {Cole, J. B.}, issn = {0018-926X}, journal = {Ieee Transactions On Antennas And Propagation}, -month = {sep}, +month = {Sep}, number = {9}, pages = {1185--1191}, title = {{High-Accuracy Yee Algorithm Based On Nonstandard Finite Differences: New Developments And Verifications}}, @@ -685,7 +674,6 @@ @article{Coleieee2002 year = {2002}, doi = {10.1109/Tap.2002.801268}, } - @article{HajimaNIM09, annote = {Workshop on Compton Sources for X-gamma Rays, Porto Conte, ITALY, SEP 07-12, 2008}, author = {Hajima, R and Kikuzawa, N and Nishimori, N and Hayakawa, T and Shizuma, T and Kawase, K and Kando, M and Minehara, E and Toyokawa, H and Ohgaki, H}, @@ -693,7 +681,7 @@ @article{HajimaNIM09 institution = {Ist Nazl Fis Nucl; ICFA}, issn = {0168-9002}, journal = {NUCLEAR INSTRUMENTS {\&} METHODS IN PHYSICS RESEARCH SECTION A-ACCELERATORS SPECTROMETERS DETECTORS AND ASSOCIATED EQUIPMENT}, -month = {sep}, +month = {Sep}, number = {1}, pages = {S57--S61}, title = {{Detection of radioactive isotopes by using laser Compton scattered gamma-ray beams}}, @@ -705,7 +693,7 @@ @article{MatlisJOSA11 doi = {10.1364/JOSAB.28.000023}, issn = {0740-3224}, journal = {JOURNAL OF THE OPTICAL SOCIETY OF AMERICA B-OPTICAL PHYSICS}, -month = {jan}, +month = {Jan}, number = {1}, pages = {23--27}, title = {{Single-shot spatiotemporal measurements of ultrashort THz waveforms using temporal electric-field cross correlation}}, @@ -713,7 +701,6 @@ @article{MatlisJOSA11 year = {2011} } @article{Quickpic2, - author = {An, Weiming and Decyk, Viktor K. and Mori, Warren B. and Antonsen, Thomas M.}, doi = {10.1016/j.jcp.2013.05.020}, issn = {00219991}, @@ -730,29 +717,32 @@ @misc{Cowanpriv2010 year = {2010} } @inproceedings{Geddesscidac09, -author = {{Geddes et al.}, Cgr}, +author = {Geddes, Cameron G.R. and Cormier-Michel, Estelle and Esarey, Eric H and Schroeder, Carl B and Vay, Jean-Luc and Leemans, Wim P and Bruhwiler, David L and Cary, John R and Cowan, Ben and Durant, Marc and Hamill, Paul and Messmer, Peter and Mullowney, Paul and Nieter, Chet and Paul, Kevin and Shasharina, Svetlana and Veitzer, Seth and Weber, Gunther and Rubel, Oliver and Ushizima, Daniela and Bethel, Wes and Wu, John}, booktitle = {Scidac Review 13}, -pages = {13}, +journal = {SciDAC Review}, +number = {13}, +pages = {13--21}, title = {{Laser Plasma Particle Accelerators: Large Fields For Smaller Facility Sources}}, +url = {https://www.osti.gov/biblio/971264}, year = {2009} } @article{Gomberoffpop2007, author = {Gomberoff, K and Fajans, J and Friedman, A and Grote, D and Vay, J.-L. and Wurtele, J S}, doi = {10.1063/1.2778420}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {oct}, +journal = {Physics of Plasmas}, +month = {Oct}, number = {10}, title = {{Simulations Of Plasma Confinement In An Antihydrogen Trap}}, volume = {14}, year = {2007} } @article{Morapop1997, -author = {Mora, P and Antonsen, Tm}, +author = {Mora, P and Antonsen, T M}, doi = {10.1063/1.872134}, issn = {1070-664X}, journal = {Phys. Plasmas}, -month = {jan}, +month = {Jan}, number = {1}, pages = {217--229}, title = {{Kinetic Modeling Of Intense, Short Laser Pulses Propagating In Tenuous Plasmas}}, @@ -770,12 +760,12 @@ @inproceedings{Cowanaac08 } @article{Cohenprstab2009, annote = {17Th International Symposium On Heavy Ion Inertial Fusion, Tokyo, Japan, Aug 04-08, 2008}, -author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J -L.}, +author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J-L}, doi = {10.1016/J.Nima.2009.03.083}, institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {53--55}, title = {{An Implicit ``Drift-Lorentz{\{}''{\}} Mover For Plasma And Beam Simulations}}, @@ -783,7 +773,6 @@ @article{Cohenprstab2009 year = {2009} } @article{Kaganovich2012, - author = {Kaganovich, Igor D. and Massidda, Scott and Startsev, Edward A. and Davidson, Ronald C. and Vay, Jean-Luc and Friedman, Alex}, journal = {Nuclear Instruments and Methods in Physics Research, Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, keywords = {Beam dynamics,Longitudinal compression,Voltage errors}, @@ -792,20 +781,22 @@ @article{Kaganovich2012 volume = {678}, year = {2012} } -@article{Vincenti2016, -author = {Vincenti, H. and Lehe, R. and Sasanka, R. and Vay, J.-L.}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Vincenti et al. - 2016 - An efficient and portable SIMD algorithm for chargecurrent deposition in Particle-In-Cell codes.pdf:pdf}, -journal = {Computer Programs in Physics}, -pages = {To appear}, -title = {{An efficient and portable SIMD algorithm for charge/current deposition in Particle-In-Cell codes}}, -url = {https://arxiv.org/abs/1601.02056}, -year = {2016} +@article{VincentiCPC2017, +author = {H. Vincenti and M. Lobet and R. Lehe and R. Sasanka and J.-L. Vay}, +doi = {https://doi.org/10.1016/j.cpc.2016.08.023}, +issn = {0010-4655}, +journal = {Computer Physics Communications}, +pages = {145-154}, +title = {An efficient and portable SIMD algorithm for charge/current deposition in Particle-In-Cell codes}, +url = {https://www.sciencedirect.com/science/article/pii/S0010465516302764}, +volume = {210}, +year = {2017}, } @article{Vaypop2008, author = {Vay, J-L}, doi = {10.1063/1.2837054}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56701}, title = {{Simulation Of Beams Or Plasmas Crossing At Relativistic Velocity}}, @@ -817,7 +808,7 @@ @article{Wieland2016 doi = {10.3847/0004-637X/820/1/62}, issn = {1538-4357}, journal = {The Astrophysical Journal}, -month = {mar}, +month = {Mar}, number = {1}, pages = {62}, title = {{NONRELATIVISTIC PERPENDICULAR SHOCKS MODELING YOUNG SUPERNOVA REMNANTS: NONSTATIONARY DYNAMICS AND PARTICLE ACCELERATION AT FORWARD AND REVERSE SHOCKS}}, @@ -831,11 +822,11 @@ @misc{Vay url = {http://www.nersc.gov/assets/Uploads/DOEExascaleReviewVay.pdf} } @article{Faurenature04, -author = {Faure, J and Glinec, Y and Pukhov, A and Kiselev, S and Gordienko, S and Lefebvre, E and Rousseau, Jp and Burgy, F and Malka, V}, +author = {Faure, J and Glinec, Y and Pukhov, A and Kiselev, S and Gordienko, S and Lefebvre, E and Rousseau, J P and Burgy, F and Malka, V}, doi = {10.1038/Nature02963}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {541--544}, title = {{A Laser-Plasma Accelerator Producing Monoenergetic Electron Beams}}, @@ -843,11 +834,11 @@ @article{Faurenature04 year = {2004} } @article{Greenwoodjcp04, -author = {Greenwood, Ad and Cartwright, Kl and Luginsland, Jw and Baca, Ea}, +author = {Greenwood, A D and Cartwright, K L and Luginsland, J W and Baca, E A}, doi = {10.1016/J.Jcp.2004.06.021}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {dec}, +month = {Dec}, number = {2}, pages = {665--684}, title = {{On The Elimination Of Numerical Cerenkov Radiation In Pic Simulations}}, @@ -855,7 +846,6 @@ @article{Greenwoodjcp04 year = {2004} } @article{GodfreyIEEE2014, - author = {Godfrey, Brendan B. and Vay, Jean-Luc and Haber, Irving}, journal = {IEEE Transactions on Plasma Science}, keywords = {Accelerators,numerical stability,particle beams,particle-in-cell (PIC),relativistic effects,simulation,spectral methods}, @@ -874,23 +864,28 @@ @inproceedings{Cowanicap09 year = {2009} } @article{Wuprstab2011, -author = {Wu, H -C. and Meyer-Ter-Vehn, J and Hegelich, B M and Fernandez, J C}, +author = {Wu, H-C and Meyer-Ter-Vehn, J and Hegelich, B M and Fernandez, J C}, doi = {10.1103/Physrevstab.14.070702}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {jul}, +month = {Jul}, number = {7}, title = {{Nonlinear Coherent Thomson Scattering From Relativistic Electron Sheets As A Means To Produce Isolated Ultrabright Attosecond X-Ray Pulses}}, volume = {14}, year = {2011} } @article{VayAAC2010, -author = {Vay, J-L and Geddes, C G R and Benedetti, C and Bruhwiler, D L and Cormier-Michel, E and Cowan, B M and Cary, J R and Grote, D P}, +author = {Vay, J.‐L. and Geddes, C. G. R. and Benedetti, C. and Bruhwiler, D. L. and Cormier‐Michel, E. and Cowan, B. M. and Cary, J. R. and Grote, D. P.}, doi = {10.1063/1.3520322}, -journal = {Aip Conference Proceedings}, +eprint = {https://pubs.aip.org/aip/acp/article-pdf/1299/1/244/11928106/244\_1\_online.pdf}, +issn = {0094-243X}, +journal = {AIP Conference Proceedings}, keywords = {[978-0-7354-0853-1/10/{\$}30.00]}, +month = {Nov}, +number = {1}, pages = {244--249}, title = {{Modeling Laser Wakefield Accelerators In A Lorentz Boosted Frame}}, +url = {https://doi.org/10.1063/1.3520322}, volume = {1299}, year = {2010} } @@ -905,21 +900,23 @@ @article{Vayprl07 year = {2007} } @article{VayJCP2013, - author = {Vay, Jean-Luc and Haber, Irving and Godfrey, Brendan B.}, +doi = {10.1016/j.jcp.2013.03.010}, +issn = {0021-9991}, journal = {Journal of Computational Physics}, keywords = {Domain decomposition,Electromagnetic,FFT,Fast fourier transform,Parallel,Particle-In-Cell,Spectral}, +month = {Jun}, pages = {260--268}, title = {{A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas}}, volume = {243}, year = {2013} } @article{Kaganovichpop2004, -author = {Kaganovich, Id and Startsev, Ea and Davidson, Rc}, +author = {Kaganovich, I D and Startsev, E A and Davidson, R C}, doi = {10.1063/1.1758945}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {jul}, +journal = {Physics of Plasmas}, +month = {Jul}, number = {7}, pages = {3546--3552}, title = {{Nonlinear Plasma Waves Excitation By Intense Ion Beams In Background Plasma}}, @@ -928,12 +925,12 @@ @article{Kaganovichpop2004 } @article{Cohenpop2005, annote = {46Th Annual Meeting Of The Division Of Plasma Physics Of The American-Physical-Society, Savannah, Ga, Nov 15-19, 2004}, -author = {Cohen, Rh and Friedman, A and Covo, Mk and Lund, Sm and Molvik, Aw and Bieniosek, Fm and Seidl, Pa and Vay, Jl and Stoltz, P and Veitzer, S}, +author = {Cohen, R H and Friedman, A and Covo, M K and Lund, S M and Molvik, A W and Bieniosek, F M and Seidl, P A and Vay, J-L and Stoltz, P and Veitzer, S}, doi = {10.1063/1.1882292}, institution = {Amer Phys Soc}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Simulating Electron Clouds In Heavy-Ion Accelerators}}, volume = {12}, @@ -941,11 +938,11 @@ @article{Cohenpop2005 } @article{Vayfed1996, annote = {7Th International Symposium On Heavy Ion Inertial Fusion, Princeton Plasma Phys Lab, Princeton, Nj, Sep 06-09, 1995}, -author = {Vay, Jl and Deutsch, C}, +author = {Vay, J-L and Deutsch, C}, doi = {10.1016/S0920-3796(96)00502-9}, issn = {0920-3796}, journal = {Fusion Engineering And Design}, -month = {nov}, +month = {Nov}, pages = {467--476}, title = {{A Three-Dimensional Electromagnetic Particle-In-Cell Code To Simulate Heavy Ion Beam Propagation In The Reaction Chamber}}, volume = {32-33}, @@ -962,10 +959,8 @@ @article{Fengjcp09 year = {2009} } @article{BESAC2013, - author = {BESAC}, doi = {10.1037/a0034573}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/582d1799e353fccb9458ff0ccb827db4b01d7308.pdf:pdf}, issn = {1935-990X}, journal = {The American psychologist}, number = {7}, @@ -977,11 +972,11 @@ @article{BESAC2013 year = {2013} } @article{Prostprstab2005, -author = {Prost, Lr and Seidl, Pa and Bieniosek, Fm and Celata, Cm and Faltens, A and Baca, D and Henestroza, E and Kwan, Jw and Leitner, M and Waldron, Wl and Cohen, R and Friedman, A and Grote, D and Lund, Sm and Molvik, Aw and Morse, E}, +author = {Prost, L R and Seidl, P A and Bieniosek, F M and Celata, C M and Faltens, A and Baca, D and Henestroza, E and Kwan, J W and Leitner, M and Waldron, W L and Cohen, R and Friedman, A and Grote, D and Lund, S M and Molvik, A W and Morse, E}, doi = {10.1103/Physrevstab.8.020101}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {feb}, +month = {Feb}, number = {2}, title = {{High Current Transport Experiment For Heavy Ion Inertial Fusion}}, volume = {8}, @@ -1000,13 +995,12 @@ @inproceedings{Godfrey2013PPPS title = {{Numerical Stability of the Pseudo-Spectral EM PIC Algorithm}}, year = {2013} } - @article{Vaypop04, author = {Vay, J.-L. and Colella, P and Kwan, J W and Mccorquodale, P and Serafini, D B and Friedman, A and Grote, D P and Westenskow, G and Adam, J.-C. and Heron, A and Haber, I}, doi = {10.1063/1.1689669}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {2928--2934}, title = {{Application Of Adaptive Mesh Refinement To Particle-In-Cell Simulations Of Plasmas And Beams}}, @@ -1017,7 +1011,7 @@ @article{Friedmanjcp90 author = {Friedman, A}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {oct}, +month = {Oct}, number = {2}, pages = {292--312}, title = {{A 2Nd-Order Implicit Particle Mover With Adjustable Damping}}, @@ -1027,8 +1021,8 @@ @article{Friedmanjcp90 @article{Rittershoferpop2010, author = {Rittershofer, W and Schroeder, C B and Esarey, E and Gruner, F J and Leemans, W P}, doi = {10.1063/1.3430638}, -journal = {Physics Of Plasmas}, -month = {jun}, +journal = {Physics of Plasmas}, +month = {Jun}, number = {6}, pages = {63104}, title = {{Tapered Plasma Channels To Phase-Lock Accelerating And Focusing Forces In Laser-Plasma Accelerators}}, @@ -1043,16 +1037,17 @@ @article{Vorpal volume = {196}, year = {2004} } -@article{LiARXIV2016, - -archivePrefix = {arXiv}, -arxivId = {1605.01496}, -author = {Li, Fei and Yu, Peicheng and Xu, Xinlu and Fiuza, Frederico and Decyk, Viktor K. and Dalichaouch, Thamine and Davidson, Asher and Tableman, Adam and An, Weiming and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Mori, Warren B.}, -eprint = {1605.01496}, -month = {may}, -title = {{Controlling the Numerical Cerenkov Instability in PIC simulations using a customized finite difference Maxwell solver and a local FFT based current correction}}, -url = {http://arxiv.org/abs/1605.01496}, -year = {2016} +@article{LiCPC2017, +author = {Fei Li and Peicheng Yu and Xinlu Xu and Frederico Fiuza and Viktor K. Decyk and Thamine Dalichaouch and Asher Davidson and Adam Tableman and Weiming An and Frank S. Tsung and Ricardo A. Fonseca and Wei Lu and Warren B. Mori}, +doi = {https://doi.org/10.1016/j.cpc.2017.01.001}, +issn = {0010-4655}, +journal = {Computer Physics Communications}, +keywords = {PIC simulation, Hybrid Maxwell solver, Relativistic plasma drift, Numerical Cerenkov instability, Lorentz boosted frame}, +pages = {6-17}, +title = {{Controlling the numerical Cerenkov instability in PIC simulations using a customized finite difference Maxwell solver and a local FFT based current correction}}, +url = {https://www.sciencedirect.com/science/article/pii/S0010465517300012}, +volume = {214}, +year = {2017}, } @article{XuJCP2013, author = {Xu, Xinlu and Yu, Peicheng and Martins, Samual F and Tsung, Frank S and Decyk, Viktor K and Vieira, Jorge and Fonseca, Ricardo A and Lu, Wei and Silva, Luis O and Mori, Warren B}, @@ -1072,17 +1067,17 @@ @article{Cormierpre08 doi = {10.1103/Physreve.78.016404}, issn = {1539-3755}, journal = {Physical Review E}, -month = {jul}, +month = {Jul}, number = {1, Part 2}, title = {{Unphysical Kinetic Effects In Particle-In-Cell Modeling Of Laser Wakefield Accelerators}}, volume = {78}, year = {2008} } @article{Berengerjcp96, -author = {Berenger, Jp}, +author = {Berenger, J P}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {2}, pages = {363--379}, title = {{Three-Dimensional Perfectly Matched Layer For The Absorption Of Electromagnetic Waves}}, @@ -1090,15 +1085,20 @@ @article{Berengerjcp96 year = {1996} } @article{Kalmykovprl09, -author = {{Kalmykov et al.}, S}, +author = {Kalmykov, S. and Yi, S. A. and Khudik, V. and Shvets, G.}, +doi = {10.1103/PhysRevLett.103.135004}, +issue = {13}, journal = {Phys. Rev. Lett.}, +month = {Sep}, +numpages = {4}, pages = {135004}, -title = {{No Title}}, +publisher = {American Physical Society}, +title = {{Electron Self-Injection and Trapping into an Evolving Plasma Bubble}}, +url = {https://link.aps.org/doi/10.1103/PhysRevLett.103.135004}, volume = {103}, year = {2009} } @article{Geddes2015, - author = {Geddes, Cameron G R and Rykovanov, Sergey and Matlis, Nicholas H. and Steinke, Sven and Vay, Jean-Luc and Esarey, Eric H. and Ludewigt, Bernhard and Nakamura, Kei and Quiter, Brian J. and Schroeder, Carl B. and Toth, Csaba and Leemans, Wim P.}, journal = {Nuclear Instruments and Methods in Physics Research, Section B: Beam Interactions with Materials and Atoms}, keywords = {Active interrogation,Homeland security,Laser plasma accelerator,Monoenergetic photon source,Nonproliferation}, @@ -1112,7 +1112,7 @@ @article{Antonsenprl1992 author = {Antonsen, T M and Mora, P}, doi = {10.1103/Physrevlett.69.2204}, journal = {Physical Review Letters}, -month = {oct}, +month = {Oct}, number = {15}, pages = {2204--2207}, title = {{Self-Focusing And Raman-Scattering Of Laser-Pulses In Tenuous Plasmas}}, @@ -1120,7 +1120,6 @@ @article{Antonsenprl1992 year = {1992} } @article{GodfreyCPC2015, - author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Computer Physics Communications}, keywords = {Numerical stability,Particle-in-cell,Pseudo-Spectral Time-Domain,Relativistic beam}, @@ -1142,10 +1141,10 @@ @article{VayCSD12 year = {2012} } @article{Esirkepovcpc01, -author = {Esirkepov, Tz}, +author = {Esirkepov, T Z}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {apr}, +month = {Apr}, number = {2}, pages = {144--153}, title = {{Exact Charge Conservation Scheme For Particle-In-Cell Simulation With An Arbitrary Form-Factor}}, @@ -1156,8 +1155,8 @@ @article{Martinspop10 author = {Martins, S F and Fonseca, R A and Vieira, J and Silva, L O and Lu, W and Mori, W B}, doi = {10.1063/1.3358139}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56705}, title = {{Modeling Laser Wakefield Accelerator Experiments With Ultrafast Particle-In-Cell Simulations In Boosted Frames}}, @@ -1166,7 +1165,7 @@ @article{Martinspop10 } @inproceedings{Habericnsp73, address = {Berkeley, Ca}, -author = {Haber, I and Lee, R and Klein, Hh and Boris, Jp}, +author = {Haber, I and Lee, R and Klein, H H and Boris, J P}, booktitle = {Proc. Sixth Conf. Num. Sim. Plasmas}, pages = {46--48}, title = {{Advances In Electromagnetic Simulation Techniques}}, @@ -1177,7 +1176,7 @@ @article{Martinscpc10 doi = {10.1016/J.Cpc.2009.12.023}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {may}, +month = {May}, number = {5}, pages = {869--875}, title = {{Numerical Simulations Of Laser Wakefield Accelerators In Optimal Lorentz Frames}}, @@ -1189,7 +1188,7 @@ @article{Vaycpc04 doi = {10.1016/J.Cpc.2004.06.026}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {dec}, +month = {Dec}, number = {1-3}, pages = {171--177}, title = {{Asymmetric Pml For The Absorption Of Waves. Application To Mesh Refinement In Electromagnetic Particle-In-Cell Plasma Simulations}}, @@ -1197,7 +1196,6 @@ @article{Vaycpc04 year = {2004} } @article{GodfreyJCP2014_PSATD, - author = {Godfrey, Brendan B. and Vay, Jean-Luc and Haber, Irving}, journal = {Journal of Computational Physics}, keywords = {Numerical stability,Particle-in-cell,Pseudo-spectral,Relativistic beam}, @@ -1211,20 +1209,19 @@ @article{PruetJAP06 doi = {10.1063/1.2202005}, issn = {0021-8979}, journal = {JOURNAL OF APPLIED PHYSICS}, -month = {jun}, +month = {Jun}, number = {12}, title = {{Detecting clandestine material with nuclear resonance fluorescence}}, volume = {99}, year = {2006} } @article{YuCPC2015-Circ, - author = {Yu, Peicheng and Xu, Xinlu and Tableman, Adam and Decyk, Viktor K. and Tsung, Frank S. and Fiuza, Frederico and Davidson, Asher and Vieira, Jorge and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.cpc.2015.08.026}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {ALGORITHM,CODES,Hybrid Maxwell solver,IN-CELL SIMULATION,LASER WAKEFIELD ACCELERATORS,LORENTZ-BOOSTED FRAME,Numerical Cerenkov instability,OSIRIS,PARTICLE SIMULATION,PIC SIMULATIONS,PIC simulation,PLASMAS,Quasi-3D algorithm,Relativistic plasma drift,STABILITY}, -month = {dec}, +month = {Dec}, pages = {144--152}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Mitigation of numerical Cerenkov radiation and instability using a hybrid finite difference-FFT Maxwell solver and a local charge conserving current deposit}}, @@ -1233,10 +1230,10 @@ @article{YuCPC2015-Circ year = {2015} } @article{Berengerjcp94, -author = {Berenger, Jp}, +author = {Berenger, J P}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {oct}, +month = {Oct}, number = {2}, pages = {185--200}, title = {{A Perfectly Matched Layer For The Absorption Of Electromagnetic-Waves}}, @@ -1253,11 +1250,11 @@ @article{Rubel2016 year = {2016} } @article{Geddesnature04, -author = {Geddes, Cgr and Toth, C and {Van Tilborg}, J and Esarey, E and Schroeder, Cb and Bruhwiler, D and Nieter, C and Cary, J and Leemans, Wp}, +author = {Geddes, C G R and Toth, C and {Van Tilborg}, J and Esarey, E and Schroeder, C B and Bruhwiler, D and Nieter, C and Cary, J and Leemans, W P}, doi = {10.1038/Nature02900}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {538--541}, title = {{High-Quality Electron Beams From A Laser Wakefield Accelerator Using Plasma-Channel Guiding}}, @@ -1265,11 +1262,11 @@ @article{Geddesnature04 year = {2004} } @article{Munzjcp2000, -author = {Munz, Cd and Omnes, P and Schneider, R and Sonnendrucker, E and Voss, U}, +author = {Munz, C D and Omnes, P and Schneider, R and Sonnendrucker, E and Voss, U}, doi = {10.1006/Jcph.2000.6507}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {jul}, +month = {Jul}, number = {2}, pages = {484--511}, title = {{Divergence Correction Techniques For Maxwell Solvers Based On A Hyperbolic Model}}, @@ -1277,8 +1274,7 @@ @article{Munzjcp2000 year = {2000} } @article{DavidsonJCP2015, - -author = {Davidson, A. and Tableman, A. and An, W. and Tsung, F.S. and Lu, W. and Vieira, J. and Fonseca, R.A. and Silva, L.O. and Mori, W.B.}, +author = {Davidson, A. and Tableman, A. and An, W. and Tsung, F. S. and Lu, W. and Vieira, J. and Fonseca, R. A. and Silva, L. O. and Mori, W. B.}, doi = {10.1016/j.jcp.2014.10.064}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -1301,7 +1297,6 @@ @article{GodfreyJCP2014 year = {2014} } @article{Godfrey2013, - author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Journal of Computational Physics}, keywords = {Esirkepov algorithm,Numerical stability,Particle-in-cell,Relativistic beam}, @@ -1316,8 +1311,8 @@ @article{Gilsonpop2010 doi = {10.1063/1.3354109}, institution = {Amer Phys Soc, Div Plasma Phys}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Studies Of Emittance Growth And Halo Particle Production In Intense Charged Particle Beams Using The Paul Trap Simulator Experiment}}, volume = {17}, @@ -1348,7 +1343,7 @@ @article{Steinke2016 author = {Steinke, S and van Tilborg, J and Benedetti, C and Geddes, C G R and Schroeder, C B and Daniels, J and Swanson, K K and Gonsalves, A J and Nakamura, K and Matlis, N H and Shaw, B H and Esarey, E and Leemans, W P}, issn = {0028-0836}, journal = {Nature}, -month = {feb}, +month = {Feb}, number = {7589}, pages = {190--193}, publisher = {Nature Publishing Group, a division of Macmillan Publishers Limited. All Rights Reserved.}, @@ -1376,7 +1371,7 @@ @article{Genonioppj2010 year = {2010} } @article{LewisJCP1972, -author = {Lewis, H.Ralph}, +author = {Lewis, H. Ralph}, doi = {http://dx.doi.org/10.1016/0021-9991(72)90044-7}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -1400,11 +1395,11 @@ @article{Vay2002 year = {2002} } @article{Manglesnature04, -author = {Mangles, Spd and Murphy, Cd and Najmudin, Z and Thomas, Agr and Collier, Jl and Dangor, Ae and Divall, Ej and Foster, Ps and Gallacher, Jg and Hooker, Cj and Jaroszynski, Da and Langley, Aj and Mori, Wb and Norreys, Pa and Tsung, Fs and Viskup, R and Walton, Br and Krushelnick, K}, +author = {Mangles, S P D and Murphy, C D and Najmudin, Z and Thomas, A G R and Collier, J L and Dangor, A E and Divall, E J and Foster, P S and Gallacher, J G and Hooker, C J and Jaroszynski, D A and Langley, A J and Mori, W B and Norreys, P A and Tsung, F S and Viskup, R and Walton, B R and Krushelnick, K}, doi = {10.1038/Nature02939}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {535--538}, title = {{Monoenergetic Beams Of Relativistic Electrons From Intense Laser-Plasma Interactions}}, @@ -1423,8 +1418,8 @@ @inproceedings{Schroederaac08 @article{Vaypop98, author = {Vay, J.-L. and Deutsch, C}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {apr}, +journal = {Physics of Plasmas}, +month = {Apr}, number = {4}, pages = {1190--1197}, title = {{Charge Compensated Ion Beam Propagation In A Reactor Sized Chamber}}, @@ -1432,11 +1427,11 @@ @article{Vaypop98 year = {1998} } @article{Friedmanjcp1991, -author = {Friedman, A and Parker, Se and Ray, Sl and Birdsall, Ck}, +author = {Friedman, A and Parker, S E and Ray, S L and Birdsall, C K}, doi = {10.1016/0021-9991(91)90265-M}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {1}, pages = {54--70}, title = {{Multiscale Particle-In-Cell Plasma Simulation}}, @@ -1444,11 +1439,11 @@ @article{Friedmanjcp1991 year = {1991} } @article{Liumotl1997, -author = {Liu, Qh}, +author = {Liu, Q H}, doi = {10.1002/(Sici)1098-2760(19970620)15:3<158::Aid-Mop11>3.3.Co;2-T}, issn = {0895-2477}, journal = {Microwave And Optical Technology Letters}, -month = {jun}, +month = {Jun}, number = {3}, pages = {158--165}, title = {{The PSTD Algorithm: A Time-Domain Method Requiring Only Two Cells Per Wavelength}}, @@ -1475,45 +1470,19 @@ @article{Leemansphysicstoday10 author = {Leemans, Wim and Esarey, Eric}, issn = {0031-9228}, journal = {Physics Today}, -month = {mar}, +month = {Mar}, number = {3}, pages = {44--49}, title = {{Laser-Driven Plasma-Wave Electron Accelerators}}, volume = {62}, year = {2009} } -@article{Lehe2015, - -archivePrefix = {arXiv}, -arxivId = {1507.04790}, -author = {Lehe, Remi and Kirchen, Manuel and Andriyash, Igor a. and Godfrey, Brendan B. and Vay, Jean-Luc}, -eprint = {1507.04790}, -isbn = {5104866785}, -journal = {arXiv.org}, -keywords = {cylindrical geometry,hankel transform,particle-in-cell,pseudo-spectral}, -pages = {1507.04790v1}, -title = {{A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm}}, -url = {http://arxiv.org/abs/1507.04790}, -volume = {physics.pl}, -year = {2015} -} -@article{VayJCP13, -author = {Vay, Jean-Luc and Haber, Irving and Godfrey, Brendan B}, -doi = {10.1016/j.jcp.2013.03.010}, -issn = {0021-9991}, -journal = {Journal of Computational Physics}, -month = {jun}, -pages = {260--268}, -title = {{A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas}}, -volume = {243}, -year = {2013} -} @inproceedings{Martinspac09, address = {Vancouver, Canada}, annote = {Th4Gbc05}, -author = {{Martins et al.}, S F}, +author = {Martins, S F and Fonseca, R A and Silva, L O and Mori, W B}, booktitle = {Proc. Particle Accelerator Conference}, -title = {{Boosted Frame Pic Simulations Of Lwfa: Towards The Energy Frontier}}, +title = {{Boosted Frame PIC Simulations of LWFA: Towards the Energy Frontier}}, year = {2009} } @article{Vayscidac09, @@ -1535,7 +1504,7 @@ @article{Sprangleprl90 author = {Sprangle, P and Esarey, E and Ting, A}, issn = {0031-9007}, journal = {Physical Review Letters}, -month = {apr}, +month = {Apr}, number = {17}, pages = {2011--2014}, title = {{Nonlinear theory of intense laser-plasma interactions}}, @@ -1548,19 +1517,19 @@ @article{Molvikpop2007 doi = {10.1063/1.2436850}, institution = {Aps, Div Plasma Phys}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Quantitative Experiments With Electrons In A Positively Charged Beam}}, volume = {14}, year = {2007} } @article{QuiterJAP08, -author = {Quiter, B J and Prussin, S G and Pohl, B and Hall, J and Trebes, J and Stone, G and Descalle, M -A.}, +author = {Quiter, B J and Prussin, S G and Pohl, B and Hall, J and Trebes, J and Stone, G and Descalle, M-A}, doi = {10.1063/1.2876028}, issn = {0021-8979}, journal = {JOURNAL OF APPLIED PHYSICS}, -month = {mar}, +month = {Mar}, number = {6}, title = {{A method for high-resolution x-ray imaging of intermodal cargo containers for fissionable materials}}, volume = {103}, @@ -1568,7 +1537,7 @@ @article{QuiterJAP08 } @book{Taflove2000, address = {Norwood}, -author = {Taflove and Hagness}, +author = {Allen Taflove and Susan C. Hagness}, edition = {2Nd}, publisher = {Ma: Artech House}, title = {{Computational Electrodynamics: The Finite-Difference Time-Domain Method}}, @@ -1578,7 +1547,7 @@ @article{Schroederprl2011 author = {Schroeder, C B and Benedetti, C and Esarey, E and Leemans, W P}, doi = {10.1103/Physrevlett.106.135002}, journal = {Physical Review Letters}, -month = {mar}, +month = {Mar}, number = {13}, pages = {135002}, title = {{Nonlinear Pulse Propagation And Phase Velocity Of Laser-Driven Plasma Waves}}, @@ -1600,10 +1569,8 @@ @article{Logannim2007 year = {2007} } @article{Yu2016, - author = {Yu, Peicheng and Xu, Xinlu and Davidson, Asher and Tableman, Adam and Dalichaouch, Thamine and Li, Fei and Meyers, Michael D. and An, Weiming and Tsung, Frank S. and Decyk, Viktor K. and Fiuza, Frederico and Vieira, Jorge and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.jcp.2016.04.014}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Yu et al. - 2016 - Enabling Lorentz boosted frame particle-in-cell simulations of laser wakefield acceleration in quasi-3D geometry.pdf:pdf}, issn = {00219991}, journal = {Journal of Computational Physics}, title = {{Enabling Lorentz boosted frame particle-in-cell simulations of laser wakefield acceleration in quasi-3D geometry}}, @@ -1611,7 +1578,7 @@ @article{Yu2016 } @article{Borisjcp73, address = {525 B St, Ste 1900, San Diego, Ca 92101-4495}, -author = {Boris, Jp and Lee, R}, +author = {Boris, J P and Lee, R}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {1}, @@ -1624,7 +1591,7 @@ @article{Borisjcp73 } @inproceedings{BorisICNSP70, address = {Naval Res. Lab., Wash., D. C.}, -author = {Boris, Jp}, +author = {Boris, J P}, booktitle = {Proc. Fourth Conf. Num. Sim. Plasmas}, pages = {3--67}, title = {{Relativistic Plasma Simulation-Optimization of a Hybrid Code}}, @@ -1638,7 +1605,7 @@ @article{Vayjcp01 author = {Vay, J.-L.}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {feb}, +month = {Feb}, number = {1}, pages = {72--98}, title = {{An Extended Fdtd Scheme For The Wave Equation: Application To Multiscale Electromagnetic Simulation}}, @@ -1663,10 +1630,10 @@ @article{Leemansnature06 doi = {10.1038/Nphys418}, issn = {1745-2473}, journal = {Nature Physics}, -month = {oct}, +month = {Oct}, number = {10}, pages = {696--699}, -title = {{Gev Electron Beams From A Centimetre-Scale Accelerator}}, +title = {{GeV electron beams from a centimetre-scale accelerator}}, volume = {2}, year = {2006} } @@ -1675,7 +1642,7 @@ @article{ChenPRSTAB13 doi = {10.1103/PhysRevSTAB.16.030701}, issn = {1098-4402}, journal = {PHYSICAL REVIEW SPECIAL TOPICS-ACCELERATORS AND BEAMS}, -month = {mar}, +month = {Mar}, number = {3}, title = {{Modeling classical and quantum radiation from laser-plasma accelerators}}, volume = {16}, @@ -1686,7 +1653,7 @@ @article{Hipace doi = {10.1088/0741-3335/56/8/084012}, issn = {0741-3335}, journal = {Plasma Physics and Controlled Fusion}, -month = {aug}, +month = {Aug}, number = {8}, pages = {084012}, publisher = {IOP Publishing}, @@ -1696,7 +1663,7 @@ @article{Hipace year = {2014} } @article{Morsenielson1971, -author = {Morse, Rl and Nielson, Cw}, +author = {Morse, R L and Nielson, C W}, doi = {10.1063/1.1693518}, issn = {1070-6631}, journal = {Phys. Fluids}, @@ -1706,14 +1673,17 @@ @article{Morsenielson1971 volume = {14}, year = {1971} } -@article{Vaydpf09, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:0909.5603}, -title = {{Speeding Up Simulations Of Relativistic Systems Using An Optimal Boosted Frame}}, +@inproceedings{Vaydpf09, +archivePrefix = {arXiv}, +author = {Vay, J.-L. and Fawley, W. M. and Geddes, C. G. R. and Cormier-Michel, E. and Grote, D. P.}, +booktitle = {{Meeting of the Division of Particles and Fields of the American Physical Society (DPF 2009)}}, +eprint = {0909.5603}, +month = {Sep}, +primaryClass = {physics.acc-ph}, +title = {{Speeding up simulations of relativistic systems using an optimal boosted frame}}, year = {2009} } @misc{Vay2014, - author = {Vay, Jean-Luc and Godfrey, Brendan B.}, booktitle = {Comptes Rendus - Mecanique}, keywords = {Numerical instability,Particle-In-Cell,Plasma simulation,Special relativity}, @@ -1724,14 +1694,7 @@ @misc{Vay2014 volume = {342}, year = {2014} } -@article{Lehearxiv2015, -author = {Lehe, R and Kirchen, M and Andriyash, I.{\~{}}A. and Godfrey, B.{\~{}}B. and Vay, J.-L.}, -journal = {arXiv:1507.04790}, -title = {{A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm}}, -year = {2015} -} @article{Friedman2014, - author = {Friedman, Alex and Cohen, Ronald H. and Grote, David P. and Lund, Steven M. and Sharp, William M. and Vay, Jean-Luc and Haber, Irving and Kishek, Rami A.}, journal = {IEEE Transactions on Plasma Science}, keywords = {Algorithms,Maxwell,Ned Birdsall,computer,laser,numerical simulation,particle beam,particle-in-cell,plasma}, @@ -1743,7 +1706,6 @@ @article{Friedman2014 year = {2014} } @article{Lcode, - author = {Lotov, K. V.}, doi = {10.1063/1.872765}, issn = {1070664X}, @@ -1770,9 +1732,7 @@ @article{GodfreyJCP2014_2 year = {2014} } @article{Turbowave, - author = {Gordon, Daniel F and Mori, W B and Antonsen, Thomas M}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Gordon, Mori, Antonsen - 2000 - A Ponderomotive Guiding Center Particle-in-Cell Code for Efficient Modeling of Laser–Plasma Interactio.pdf:pdf}, journal = {IEEE TRANSACTIONS ON PLASMA SCIENCE}, keywords = {Index Terms—Particle code,plasma simulation}, number = {4}, @@ -1787,7 +1747,7 @@ @article{Habernim2009 institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {64--68}, title = {{Scaled Electron Studies At The University Of Maryland}}, @@ -1805,13 +1765,12 @@ @article{Folegatijpcs2011 year = {2011} } @article{YuCPC2015, - author = {Yu, Peicheng and Xu, Xinlu and Decyk, Viktor K. and Fiuza, Frederico and Vieira, Jorge and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.cpc.2015.02.018}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {ALGORITHM,LASER WAKEFIELD ACCELERATORS,LORENTZ-BOOSTED FRAME,Numerical Cerenkov instability,Numerical dispersion relation,PARTICLE SIMULATION,PLASMA,Particle-in-cell,Plasma simulation,Relativistic drifting plasma,SHOCKS,STABILITY,Spectral solver,WAVES}, -month = {jul}, +month = {Jul}, pages = {32--47}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Elimination of the numerical Cerenkov instability for spectral EM-PIC codes}}, @@ -1824,19 +1783,13 @@ @article{PukhovJPP99 doi = {10.1017/S0022377899007515}, issn = {0022-3778}, journal = {Journal of Plasma Physics}, -month = {apr}, +month = {Apr}, number = {3}, pages = {425--433}, title = {{Three-dimensional electromagnetic relativistic particle-in-cell code VLPL (Virtual Laser Plasma Lab)}}, volume = {61}, year = {1999} } -@article{Vincentiarxiv2015, -author = {Vincenti, H and Vay, J.-L.}, -journal = {arXiv:1507.05572}, -title = {{Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver}}, -year = {2015} -} @inproceedings{Vayipac10, address = {Tokyo, Japan}, annote = {Weobra02}, @@ -1849,7 +1802,7 @@ @article{Cormierprstab2011 author = {Cormier-Michel, E and Esarey, E and Geddes, C G R and Schroeder, C B and Paul, K and Mullowney, P J and Cary, J R and Leemans, W P}, doi = {10.1103/Physrevstab.14.031303}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {mar}, +month = {Mar}, number = {3}, pages = {31303}, title = {{Control Of Focusing Fields In Laser-Plasma Accelerators Using Higher-Order Modes}}, @@ -1857,10 +1810,8 @@ @article{Cormierprstab2011 year = {2011} } @article{Lehe2016, - author = {Lehe, R{\'{e}}mi and Kirchen, Manuel and Andriyash, Igor A. and Godfrey, Brendan B. and Vay, Jean-Luc}, doi = {10.1016/j.cpc.2016.02.007}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Lehe et al. - 2016 - A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm.pdf:pdf}, issn = {00104655}, journal = {Computer Physics Communications}, pages = {66--82}, @@ -1877,7 +1828,6 @@ @book{Birdsalllangdon year = {1991} } @inproceedings{Grote2005, - author = {Grote, David P. and Friedman, Alex and Vay, Jean-Luc and Haber, Irving}, booktitle = {AIP Conference Proceedings}, pages = {55--58}, @@ -1905,7 +1855,7 @@ @article{LeemansPRL2014 author = {Leemans, W P and Gonsalves, A J and Mao, H.-S. and Nakamura, K and Benedetti, C and Schroeder, C B and T{\'{o}}th, Cs. and Daniels, J and Mittelberger, D E and Bulanov, S S and Vay, J.-L. and Geddes, C G R and Esarey, E}, doi = {10.1103/PhysRevLett.113.245002}, journal = {Phys. Rev. Lett.}, -month = {dec}, +month = {Dec}, number = {24}, pages = {245002}, publisher = {American Physical Society}, @@ -1918,7 +1868,7 @@ @article{Abejcp86 author = {Abe, H and Sakairi, N and Itatani, R and Okuda, H}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {apr}, +month = {Apr}, number = {2}, pages = {247--267}, title = {{High-Order Spline Interpolations In The Particle Simulation}}, @@ -1938,11 +1888,11 @@ @article{LeeCPC2015 year = {2015} } @article{Vaylpb2002, -author = {Vay, Jl and Colella, P and Mccorquodale, P and {Van Straalen}, B and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Colella, P and Mccorquodale, P and {Van Straalen}, B and Friedman, A and Grote, D P}, doi = {10.1017/S0263034602204139}, issn = {0263-0346}, journal = {Laser And Particle Beams}, -month = {dec}, +month = {Dec}, number = {4}, pages = {569--575}, title = {{Mesh Refinement For Particle-In-Cell Plasma Simulations: Applications To And Benefits For Heavy Ion Fusion}}, @@ -1962,7 +1912,7 @@ @article{Tzoufrasprl2008 author = {Tzoufras, M and Lu, W and Tsung, F S and Huang, C and Mori, W B and Katsouleas, T and Vieira, J and Fonseca, R A and Silva, L O}, doi = {10.1103/Physrevlett.101.145002}, journal = {Physical Review Letters}, -month = {oct}, +month = {Oct}, number = {14}, pages = {145002}, title = {{Beam Loading In The Nonlinear Regime Of Plasma-Based Acceleration Rid C-6436-2009 Rid B-7680-2009 Rid C-3169-2009}}, @@ -1973,7 +1923,7 @@ @article{Bulanovphysfluid1992 author = {Bulanov, S V and Inovenkov, I N and Kirsanov, V I and Naumova, N M and Sakharov, A S}, doi = {10.1063/1.860046}, journal = {Physics Of Fluids B-Plasma Physics}, -month = {jul}, +month = {Jul}, number = {7}, pages = {1935--1942}, title = {{Nonlinear Depletion Of Ultrashort And Relativistically Strong Laser-Pulses In An Underdense Plasma}}, @@ -1985,7 +1935,7 @@ @article{Martinsnaturephysics10 doi = {10.1038/Nphys1538}, issn = {1745-2473}, journal = {Nature Physics}, -month = {apr}, +month = {Apr}, number = {4}, pages = {311--316}, title = {{Exploring Laser-Wakefield-Accelerator Regimes For Near-Term Lasers Using Particle-In-Cell Simulation In Lorentz-Boosted Frames}}, @@ -1995,8 +1945,8 @@ @article{Martinsnaturephysics10 @article{Friedmanpop10, author = {Friedman, A and Barnard, J J and Cohen, R H and Grote, D P and Lund, S M and Sharp, W M and Faltens, A and Henestroza, E and Jung, J.-Y. and Kwan, J W and Lee, E P and Leitner, M A and Logan, B G and Vay, J.-L. and Waldron, W L and Davidson, R C and Dorf, M and Gilson, E P and Kaganovich, I D}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {056704 (9 Pp.)}, title = {{Beam Dynamics Of The Neutralized Drift Compression Experiment-Ii, A Novel Pulse-Compressing Ion Accelerator}}, @@ -2006,8 +1956,8 @@ @article{Friedmanpop10 @article{Schroederpop2006, author = {Schroeder, C B and Esarey, E and Shadwick, B A and Leemans, W P}, doi = {10.1063/1.2173960}, -journal = {Physics Of Plasmas}, -month = {mar}, +journal = {Physics of Plasmas}, +month = {Mar}, number = {3}, pages = {33103}, title = {{Trapping, Dark Current, And Wave Breaking In Nonlinear Plasma Waves}}, @@ -2016,12 +1966,12 @@ @article{Schroederpop2006 } @article{Friedmanpfb92, annote = {33Rd Annual Meeting Of The Division Of Plasma Physics Of The American Physical Soc, Tampa, Fl, Nov 04-08, 1991}, -author = {Friedman, A and Grote, Dp and Haber, I}, +author = {Friedman, A and Grote, D P and Haber, I}, doi = {10.1063/1.860024}, institution = {Amer Phys Soc, Div Plasma Phys}, issn = {0899-8221}, journal = {Physics Of Fluids B-Plasma Physics}, -month = {jul}, +month = {Jul}, number = {7, Part 2}, pages = {2203--2210}, title = {{3-Dimensional Particle Simulation Of Heavy-Ion Fusion Beams}}, @@ -2039,7 +1989,7 @@ @article{Winklehnerji2010 doi = {10.1088/1748-0221/5/12/P12001}, issn = {1748-0221}, journal = {Journal of Instrumentation}, -month = {dec}, +month = {Dec}, title = {{Comparison Of Extraction And Beam Transport Simulations With Emittance Measurements From The Ecr Ion Source Venus}}, volume = {5}, year = {2010} @@ -2047,19 +1997,18 @@ @article{Winklehnerji2010 @inproceedings{Vaypac09, address = {Vancouver, Canada}, annote = {Tu1Pbi04}, -author = {{Vay et al.}, J.-L.}, +author = {Vay, J-L and Fawley, W M and Geddes, C G R and Cormier-Michel, E and Grote, D P}, booktitle = {Proc. Particle Accelerator Conference}, -title = {{Application Of The Reduction Of Scale Range In A Lorentz Boosted Frame To The Numerical Simulation Of Particle Acceleration Devices}}, +title = {{Application of the reduction of scale range in a Lorentz boosted frame to the numerical simulation of particle acceleration devices}}, year = {2009} } -@article{Vincenti2016a, - +@article{VincentiCPC2017a, author = {Vincenti, H. and Vay, J.-L.}, doi = {10.1016/j.cpc.2015.11.009}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {3D electromagnetic simulations,ABSORPTION,ALGORITHM,APPROXIMATIONS,CLOSED-FORM EXPRESSIONS,Domain decomposition technique,Effects of stencil truncation errors,PERFECTLY MATCHED LAYER,Perfectly Matched Layers,Pseudo-spectral Maxwell solver,SIMULATIONS,TAYLOR-SERIES,Very high-order Maxwell solver,WAVES}, -month = {mar}, +month = {Mar}, pages = {147--167}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver}}, @@ -2072,7 +2021,7 @@ @article{Vayjcp02 doi = {10.1006/Jcph.2002.7175}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {dec}, +month = {Dec}, number = {2}, pages = {367--399}, title = {{Asymmetric Perfectly Matched Layer For The Absorption Of Waves}}, @@ -2093,14 +2042,14 @@ @book{Geddesdissertation05 year = {2005} } @article{Tsungpop06, -author = {Tsung, F S and W Lu, and Tzoufras, M and Mori, W B and Joshi, C and Vieira, Jm and Silva, Lo and Fonseca, Ra}, +author = {Tsung, F. S. and Lu, W. and Tzoufras, M. and Mori, W. B. and Joshi, C. and Vieira, J. M. and Silva, L. O. and Fonseca, R. A.}, doi = {10.1063/1.2198535}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56708}, -title = {{Simulation Of Monoenergetic Electron Generation Via Laser Wakefield Accelerators For 5-25 Tw Lasers}}, +title = {{Simulation Of Monoenergetic Electron Generation Via Laser Wakefield Accelerators For 5-25 TW Lasers}}, volume = {13}, year = {2006} } @@ -2108,7 +2057,6 @@ @inproceedings{INFERNO address = {Rostock-Warnemünde, Germany}, author = {Benedetti, Carlo and Schroeder, Carl B. and Esarey, Eric and Leemans, Wim P.}, booktitle = {ICAP}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Benedetti et al. - 2012 - Efficient Modeling of Laser-plasma Accelerators Using the Ponderomotive-based Code INF{\&}ampRNO.pdf:pdf}, pages = {THAAI2}, publisher = {Jacow}, title = {{Efficient Modeling of Laser-plasma Accelerators Using the Ponderomotive-based Code INF{\&}RNO}}, @@ -2120,7 +2068,7 @@ @article{GonsalvesNP2011 doi = {10.1038/NPHYS2071}, issn = {1745-2473}, journal = {NATURE PHYSICS}, -month = {nov}, +month = {Nov}, number = {11}, pages = {862--866}, title = {{Tunable laser plasma accelerator based on longitudinal density tailoring}}, @@ -2132,14 +2080,14 @@ @article{Ohtsuboprstab2010 doi = {10.1103/Physrevstab.13.044201}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {apr}, +month = {Apr}, number = {4}, title = {{Experimental Study Of Coherent Betatron Resonances With A Paul Trap}}, volume = {13}, year = {2010} } @inproceedings{Warp, -author = {Grote, D P and Friedman, A and Vay, J.-L. and Haber, I}, +author = {Grote, D P and Friedman, A and Vay, J-L and Haber, I}, booktitle = {Aip Conference Proceedings}, issn = {0094-243X}, number = {749}, @@ -2148,11 +2096,11 @@ @inproceedings{Warp year = {2005} } @article{Mccorquodalejcp2004, -author = {Mccorquodale, P and Colella, P and Grote, Dp and Vay, Jl}, +author = {Mccorquodale, P and Colella, P and Grote, D P and Vay, J-L}, doi = {10.1016/J.Jcp.2004.04.022}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {nov}, +month = {Nov}, number = {1}, pages = {34--60}, title = {{A Node-Centered Local Refinement Algorithm For Poisson's Equation In Complex Geometries}}, @@ -2162,7 +2110,6 @@ @article{Mccorquodalejcp2004 @article{Londrillo2010, author = {Londrillo, P. and Benedetti, C. and Sgattoni, A.}, doi = {10.1016/j.nima.2010.01.055}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Londrillo, Benedetti, Sgattoni - 2010 - Charge preserving high order PIC schemes.pdf:pdf}, issn = {01689002}, journal = {Nuclear Instruments and Methods in Physics Research Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, number = {1}, @@ -2181,11 +2128,11 @@ @article{GodfreyJCP2014_FDTD year = {2014} } @article{Shortley-Weller, -author = {Shortley, Gh and Weller, R}, +author = {Shortley, G H and Weller, R}, doi = {10.1063/1.1710426}, issn = {0021-8979}, journal = {Journal of Applied Physics}, -month = {may}, +month = {May}, number = {5}, pages = {334--348}, title = {{The Numerical Solution Of Laplace's Equation}}, @@ -2193,34 +2140,51 @@ @article{Shortley-Weller year = {1938} } @article{VayPOPL2011, -author = {Vay, Jl and Geddes, C G R and Cormier-Michel, E and Grote, D P}, +author = {Vay, J.-L. and Geddes, C. G. R. and Cormier-Michel, E. and Grote, D. P.}, doi = {10.1063/1.3559483}, -journal = {Physics Of Plasmas}, -month = {mar}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.3559483/16019930/030701\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Mar}, number = {3}, -pages = {30701}, -title = {{Effects Of Hyperbolic Rotation In Minkowski Space On The Modeling Of Plasma Accelerators In A Lorentz Boosted Frame}}, +pages = {030701}, +title = {{Effects of hyperbolic rotation in Minkowski space on the modeling of plasma accelerators in a Lorentz boosted frame}}, +url = {https://doi.org/10.1063/1.3559483}, volume = {18}, year = {2011} } - -@article{KirchenARXIV2016, -author = {Kirchen, M. and Lehe, R. and Godfrey, B.~B. and Dornmair, I. and Jalas, S. and Peters, K. and Vay, J.-L. and Maier, A.~R.}, -journal = {arXiv:1608.00215}, +@article{KirchenPOP2016, +author = {Kirchen, M. and Lehe, R. and Godfrey, B. B. and Dornmair, I. and Jalas, S. and Peters, K. and Vay, J.-L. and Maier, A. R.}, +doi = {10.1063/1.4964770}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4964770/14024121/100704\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Oct}, +number = {10}, +pages = {100704}, title = {{Stable discrete representation of relativistically drifting plasmas}}, +url = {https://doi.org/10.1063/1.4964770}, +volume = {23}, year = {2016} } - -@article{LeheARXIV2016, -author = {Lehe, R. and Kirchen, M. and Godfrey, B.~B. and Maier, A.~R. and Vay, J.-L.}, -journal = {arXiv:1608.00227}, -title = {{Elimination of Numerical Cherenkov Instability in flowing-plasma Particle-In-Cell simulations by using Galilean coordinates}}, +@article{LehePRE2016, +author = {Lehe, Remi and Kirchen, Manuel and Godfrey, Brendan B. and Maier, Andreas R. and Vay, Jean-Luc}, +doi = {10.1103/PhysRevE.94.053305}, +issue = {5}, +journal = {Phys. Rev. E}, +month = {Nov}, +numpages = {16}, +pages = {053305}, +publisher = {American Physical Society}, +title = {{Elimination of numerical Cherenkov instability in flowing-plasma particle-in-cell simulations by using Galilean coordinates}}, +url = {https://link.aps.org/doi/10.1103/PhysRevE.94.053305}, +volume = {94}, year = {2016} } @book{godfrey1985iprop, -author={Godfrey, B.B.}, -publisher={Defense Technical Information Center}, -title={{The IPROP Three-Dimensional Beam Propagation Code}}, -year={1985}, +author = {Godfrey, B. B.}, +publisher = {Defense Technical Information Center}, +title = {{The IPROP Three-Dimensional Beam Propagation Code}}, +year = {1985} } diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 0abca79a5d4..5d6d649da2d 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -1,38 +1,38 @@ @article{TajimaDawson1982, - author = {Tajima, T. and Dawson, J. M.}, - title = {{Laser accelerator by plasma waves}}, - journal = {AIP Conference Proceedings}, - volume = {91}, - number = {1}, - pages = {69-93}, - year = {1982}, - month = {09}, - abstract = "{Parallel intense laser beam ω0, k0 and ω1, k1 shone on a plasma with frequency separation equal to the plasma frequency ωp is capable of creating a coherent large electrostatic field and accelerating particles to high energies in large flux. The photon beam excites through the forward Raman scattering large amplitude plasmons whose phase velocity is equal to (ω−ω1)/(k0−k1), close to c in an underdense plasma. The plasmon traps electrons with electrostatic field EL=γ1/2⊥ mcωp/c, of the order of a few GeV/cm for plasma density to 1018 cm−3. Because of the phase velocity of the field close to c this field carries trapped electrons to high energies: W=2mc2(ω0/ωp)2. Preaccelerated particles (ions, for examples) coherent with the plasmon fields can also be accelerated. The (multiple) forward Raman instability saturates only when a sizable electron population is trapped and most of the electromagnetic energy is cascaded down to the frequency close to the cut‐off (ωp).}", - issn = {0094-243X}, - doi = {10.1063/1.33805}, - url = {https://doi.org/10.1063/1.33805} +abstract = "{Parallel intense laser beam ω0, k0 and ω1, k1 shone on a plasma with frequency separation equal to the plasma frequency ωp is capable of creating a coherent large electrostatic field and accelerating particles to high energies in large flux. The photon beam excites through the forward Raman scattering large amplitude plasmons whose phase velocity is equal to (ω−ω1)/(k0−k1), close to c in an underdense plasma. The plasmon traps electrons with electrostatic field EL=γ1/2⊥ mcωp/c, of the order of a few GeV/cm for plasma density to 1018 cm−3. Because of the phase velocity of the field close to c this field carries trapped electrons to high energies: W=2mc2(ω0/ωp)2. Preaccelerated particles (ions, for examples) coherent with the plasmon fields can also be accelerated. The (multiple) forward Raman instability saturates only when a sizable electron population is trapped and most of the electromagnetic energy is cascaded down to the frequency close to the cut‐off (ωp).}", +author = {Tajima, T. and Dawson, J. M.}, +doi = {10.1063/1.33805}, +issn = {0094-243X}, +journal = {AIP Conference Proceedings}, +month = {Sep}, +number = {1}, +pages = {69--93}, +title = {{Laser accelerator by plasma waves}}, +url = {https://doi.org/10.1063/1.33805}, +volume = {91}, +year = {1982} } @article{Esarey1996, - author={Esarey, E. and Sprangle, P. and Krall, J. and Ting, A.}, - journal={IEEE Transactions on Plasma Science}, - title={{Overview of plasma-based accelerator concepts}}, - year={1996}, - volume={24}, - number={2}, - pages={252-288}, - doi={10.1109/27.509991} +author = {Esarey, E. and Sprangle, P. and Krall, J. and Ting, A.}, +doi = {10.1109/27.509991}, +journal = {IEEE Transactions on Plasma Science}, +number = {2}, +pages = {252--288}, +title = {{Overview of plasma-based accelerator concepts}}, +volume = {24}, +year = {1996} } @ARTICLE{Birdsall1991, - author = {Birdsall, C.K.}, - journal = {IEEE Transactions on Plasma Science}, - title = {{Particle-in-cell charged-particle simulations, plus Monte Carlo collisions with neutral atoms, PIC-MCC}}, - year = {1991}, - volume = {19}, - number = {2}, - pages = {65-85}, - doi = {10.1109/27.106800} +author = {Birdsall, C. K.}, +doi = {10.1109/27.106800}, +journal = {IEEE Transactions on Plasma Science}, +number = {2}, +pages = {65--85}, +title = {{Particle-in-cell charged-particle simulations, plus Monte Carlo collisions with neutral atoms, PIC-MCC}}, +volume = {19}, +year = {1991} } @misc{Lim2007, @@ -41,206 +41,206 @@ @misc{Lim2007 language = {eng}, number = {3}, title = {{The interaction of energetic charged particles with gas and boundaries in the particle simulation of plasmas}}, +url = {https://search.library.berkeley.edu/permalink/01UCS_BER/s4lks2/cdi_proquest_miscellaneous_35689087}, volume = {69}, -year = {2007}, -url = {https://search.library.berkeley.edu/permalink/01UCS_BER/s4lks2/cdi_proquest_miscellaneous_35689087} +year = {2007} } @article{Turner2013, author = {Turner, M. M. and Derzsi, A. and Donkó, Z. and Eremin, D. and Kelly, S. J. and Lafleur, T. and Mussenbrock, T.}, -title = {{Simulation benchmarks for low-pressure plasmas: Capacitive discharges}}, +doi = {10.1063/1.4775084}, +issn = {1070-664X}, journal = {Physics of Plasmas}, -volume = {20}, +month = {Jan}, +note = {013507}, number = {1}, -year = {2013}, -month = {01}, -issn = {1070-664X}, -doi = {10.1063/1.4775084}, +title = {{Simulation benchmarks for low-pressure plasmas: Capacitive discharges}}, url = {https://doi.org/10.1063/1.4775084}, -note = {013507} +volume = {20}, +year = {2013} } -@misc{winske2022hybrid, -title={{Hybrid codes (massless electron fluid)}}, -author={D. Winske and Homa Karimabadi and Ari Le and N. Omidi and Vadim Roytershteyn and Adam Stanier}, -year={2022}, -eprint={2204.01676}, -archivePrefix={arXiv}, -primaryClass={physics.plasm-ph} +@article{winske2022hybrid, +archivePrefix = {arXiv}, +author = {D. Winske and Homa Karimabadi and Ari Le and N. Omidi and Vadim Roytershteyn and Adam Stanier}, +eprint = {2204.01676}, +primaryClass = {physics.plasm-ph}, +title = {{Hybrid codes (massless electron fluid)}}, +year = {2022} } @incollection{NIELSON1976, -title = {{Particle-Code Models in the Nonradiative Limit}}, -editor = {JOHN KILLEEN}, -series = {Methods in Computational Physics: Advances in Research and Applications}, -publisher = {Elsevier}, -volume = {16}, -pages = {367-388}, -year = {1976}, +author = {Clair W. Nielson and H. Ralph Lewis}, booktitle = {Controlled Fusion}, -issn = {0076-6860}, doi = {https://doi.org/10.1016/B978-0-12-460816-0.50015-4}, -author = {CLAIR W. NIELSON and H. RALPH LEWIS} +editor = {John Killeen}, +issn = {0076-6860}, +pages = {367--388}, +publisher = {Elsevier}, +series = {Methods in Computational Physics: Advances in Research and Applications}, +title = {{Particle-Code Models in the Nonradiative Limit}}, +volume = {16}, +year = {1976} } @article{MUNOZ2018, -title = {{A new hybrid code (CHIEF) implementing the inertial electron fluid equation without approximation}}, +author = {P. A. Muñoz and N. Jain and P. Kilian and J. Büchner}, +doi = {https://doi.org/10.1016/j.cpc.2017.10.012}, +issn = {0010-4655}, journal = {Computer Physics Communications}, -volume = {224}, +keywords = {Plasma simulation, Hybrid methods, Particle-in-cell method}, pages = {245-264}, -year = {2018}, -issn = {0010-4655}, -doi = {https://doi.org/10.1016/j.cpc.2017.10.012}, +title = {{A new hybrid code (CHIEF) implementing the inertial electron fluid equation without approximation}}, url = {https://www.sciencedirect.com/science/article/pii/S0010465517303521}, -author = {P.A. Muñoz and N. Jain and P. Kilian and J. Büchner}, -keywords = {Plasma simulation, Hybrid methods, Particle-in-cell method} +volume = {224}, +year = {2018} } @article{Le2016, +abstract = "{We present the first hybrid simulations with kinetic ions and recently developed equations of state for the electron fluid appropriate for reconnection with a guide field. The equations of state account for the main anisotropy of the electron pressure tensor. Magnetic reconnection is studied in two systems, an initially force-free current sheet and a Harris sheet. The hybrid model with the equations of state is compared to two other models, hybrid simulations with isothermal electrons and fully kinetic simulations. Including the anisotropic equations of state in the hybrid model provides a better match to the fully kinetic model. In agreement with fully kinetic results, the main feature captured is the formation of an electron current sheet that extends several ion inertial lengths. This electron current sheet modifies the Hall magnetic field structure near the X-line, and it is not observed in the standard hybrid model with isotropic electrons. The saturated reconnection rate in this regime nevertheless remains similar in all three models. Implications for global modeling are discussed.}", author = {Le, A. and Daughton, W. and Karimabadi, H. and Egedal, J.}, -title = {{Hybrid simulations of magnetic reconnection with kinetic ions and fluid electron pressure anisotropy}}, +doi = {10.1063/1.4943893}, +issn = {1070-664X}, journal = {Physics of Plasmas}, -volume = {23}, +month = {Mar}, +note = {032114}, number = {3}, -year = {2016}, -month = {03}, -abstract = "{We present the first hybrid simulations with kinetic ions and recently developed equations of state for the electron fluid appropriate for reconnection with a guide field. The equations of state account for the main anisotropy of the electron pressure tensor. Magnetic reconnection is studied in two systems, an initially force-free current sheet and a Harris sheet. The hybrid model with the equations of state is compared to two other models, hybrid simulations with isothermal electrons and fully kinetic simulations. Including the anisotropic equations of state in the hybrid model provides a better match to the fully kinetic model. In agreement with fully kinetic results, the main feature captured is the formation of an electron current sheet that extends several ion inertial lengths. This electron current sheet modifies the Hall magnetic field structure near the X-line, and it is not observed in the standard hybrid model with isotropic electrons. The saturated reconnection rate in this regime nevertheless remains similar in all three models. Implications for global modeling are discussed.}", -issn = {1070-664X}, -doi = {10.1063/1.4943893}, +title = {{Hybrid simulations of magnetic reconnection with kinetic ions and fluid electron pressure anisotropy}}, url = {https://doi.org/10.1063/1.4943893}, -note = {032114} +volume = {23}, +year = {2016} } @article{Stanier2020, -title = {{A cancellation problem in hybrid particle-in-cell schemes due to finite particle size}}, +author = {A. Stanier and L. Chacón and A. Le}, +doi = {https://doi.org/10.1016/j.jcp.2020.109705}, +issn = {0021-9991}, journal = {Journal of Computational Physics}, -volume = {420}, +keywords = {Hybrid, Particle-in-cell, Plasma, Asymptotic-preserving, Cancellation problem, Space weather}, pages = {109705}, -year = {2020}, -issn = {0021-9991}, -doi = {https://doi.org/10.1016/j.jcp.2020.109705}, +title = {{A cancellation problem in hybrid particle-in-cell schemes due to finite particle size}}, url = {https://www.sciencedirect.com/science/article/pii/S0021999120304794}, -author = {A. Stanier and L. Chacón and A. Le}, -keywords = {Hybrid, Particle-in-cell, Plasma, Asymptotic-preserving, Cancellation problem, Space weather}, +volume = {420}, +year = {2020} } @book{Stix1992, - author = {Stix, T.H.}, - date-added = {2023-06-29 13:51:16 -0700}, - date-modified = {2023-06-29 13:51:16 -0700}, - isbn = {978-0-88318-859-0}, - lccn = {lc91033341}, - publisher = {American Inst. of Physics}, - title = {{Waves in Plasmas}}, - url = {https://books.google.com/books?id=OsOWJ8iHpmMC}, - year = {1992}, - bdsk-url-1 = {https://books.google.com/books?id=OsOWJ8iHpmMC} +author = {Stix, T. H.}, +bdsk-url-1 = {https://books.google.com/books?id=OsOWJ8iHpmMC}, +date-added = {2023-06-29 13:51:16 -0700}, +date-modified = {2023-06-29 13:51:16 -0700}, +isbn = {978-0-88318-859-0}, +lccn = {lc91033341}, +publisher = {American Inst. of Physics}, +title = {{Waves in Plasmas}}, +url = {https://books.google.com/books?id=OsOWJ8iHpmMC}, +year = {1992} } @article{Macchi2013, - title = {{Ion acceleration by superintense laser-plasma interaction}}, - author = {Macchi, Andrea and Borghesi, Marco and Passoni, Matteo}, - journal = {Rev. Mod. Phys.}, - volume = {85}, - issue = {2}, - pages = {751--793}, - numpages = {0}, - year = {2013}, - month = {May}, - publisher = {American Physical Society}, - doi = {10.1103/RevModPhys.85.751}, - url = {https://link.aps.org/doi/10.1103/RevModPhys.85.751} +author = {Macchi, Andrea and Borghesi, Marco and Passoni, Matteo}, +doi = {10.1103/RevModPhys.85.751}, +issue = {2}, +journal = {Rev. Mod. Phys.}, +month = {May}, +numpages = {0}, +pages = {751--793}, +publisher = {American Physical Society}, +title = {{Ion acceleration by superintense laser-plasma interaction}}, +url = {https://link.aps.org/doi/10.1103/RevModPhys.85.751}, +volume = {85}, +year = {2013} } @article{Wilks2001, - author = {Wilks, S. C. and Langdon, A. B. and Cowan, T. E. and Roth, M. and Singh, M. and Hatchett, S. and Key, M. H. and Pennington, D. and MacKinnon, A. and Snavely, R. A.}, - title = {{Energetic proton generation in ultra-intense laser–solid interactions}}, - journal = {Physics of Plasmas}, - volume = {8}, - number = {2}, - pages = {542-549}, - year = {2001}, - month = {02}, - abstract = "{An explanation for the energetic ions observed in the PetaWatt experiments is presented. In solid target experiments with focused intensities exceeding 1020 W/cm2, high-energy electron generation, hard bremsstrahlung, and energetic protons have been observed on the backside of the target. In this report, an attempt is made to explain the physical process present that will explain the presence of these energetic protons, as well as explain the number, energy, and angular spread of the protons observed in experiment. In particular, we hypothesize that hot electrons produced on the front of the target are sent through to the back off the target, where they ionize the hydrogen layer there. These ions are then accelerated by the hot electron cloud, to tens of MeV energies in distances of order tens of μm, whereupon they end up being detected in the radiographic and spectrographic detectors.}", - issn = {1070-664X}, - doi = {10.1063/1.1333697}, - url = {https://doi.org/10.1063/1.1333697}, - eprint = {https://pubs.aip.org/aip/pop/article-pdf/8/2/542/12669088/542\_1\_online.pdf}, +abstract = "{An explanation for the energetic ions observed in the PetaWatt experiments is presented. In solid target experiments with focused intensities exceeding 1020 W/cm2, high-energy electron generation, hard bremsstrahlung, and energetic protons have been observed on the backside of the target. In this report, an attempt is made to explain the physical process present that will explain the presence of these energetic protons, as well as explain the number, energy, and angular spread of the protons observed in experiment. In particular, we hypothesize that hot electrons produced on the front of the target are sent through to the back off the target, where they ionize the hydrogen layer there. These ions are then accelerated by the hot electron cloud, to tens of MeV energies in distances of order tens of μm, whereupon they end up being detected in the radiographic and spectrographic detectors.}", +author = {Wilks, S. C. and Langdon, A. B. and Cowan, T. E. and Roth, M. and Singh, M. and Hatchett, S. and Key, M. H. and Pennington, D. and MacKinnon, A. and Snavely, R. A.}, +doi = {10.1063/1.1333697}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/8/2/542/12669088/542\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Feb}, +number = {2}, +pages = {542-549}, +title = {{Energetic proton generation in ultra-intense laser–solid interactions}}, +url = {https://doi.org/10.1063/1.1333697}, +volume = {8}, +year = {2001} } @article{Bulanov2008, - title = {{Accelerating monoenergetic protons from ultrathin foils by flat-top laser pulses in the directed-Coulomb-explosion regime}}, - author = {Bulanov, S. S. and Brantov, A. and Bychenkov, V. Yu. and Chvykov, V. and Kalinchenko, G. and Matsuoka, T. and Rousseau, P. and Reed, S. and Yanovsky, V. and Litzenberg, D. W. and Krushelnick, K. and Maksimchuk, A.}, - journal = {Phys. Rev. E}, - volume = {78}, - issue = {2}, - pages = {026412}, - numpages = {6}, - year = {2008}, - month = {Aug}, - publisher = {American Physical Society}, - doi = {10.1103/PhysRevE.78.026412}, - url = {https://link.aps.org/doi/10.1103/PhysRevE.78.026412} +author = {Bulanov, S. S. and Brantov, A. and Bychenkov, V. Yu. and Chvykov, V. and Kalinchenko, G. and Matsuoka, T. and Rousseau, P. and Reed, S. and Yanovsky, V. and Litzenberg, D. W. and Krushelnick, K. and Maksimchuk, A.}, +doi = {10.1103/PhysRevE.78.026412}, +issue = {2}, +journal = {Phys. Rev. E}, +month = {Aug}, +numpages = {6}, +pages = {026412}, +publisher = {American Physical Society}, +title = {{Accelerating monoenergetic protons from ultrathin foils by flat-top laser pulses in the directed-Coulomb-explosion regime}}, +url = {https://link.aps.org/doi/10.1103/PhysRevE.78.026412}, +volume = {78}, +year = {2008} } @article{Dromey2004, - author = {Dromey, B. and Kar, S. and Zepf, M. and Foster, P.}, - title = {{The plasma mirror—A subpicosecond optical switch for ultrahigh power lasers}}, - journal = {Review of Scientific Instruments}, - volume = {75}, - number = {3}, - pages = {645-649}, - year = {2004}, - month = {02}, - abstract = "{Plasma mirrors are devices capable of switching very high laser powers on subpicosecond time scales with a dynamic range of 20–30 dB. A detailed study of their performance in the near-field of the laser beam is presented, a setup relevant to improving the pulse contrast of modern ultrahigh power lasers (TW–PW). The conditions under which high reflectivity can be achieved and focusability of the reflected beam retained are identified. At higher intensities a region of high specular reflectivity with rapidly decreasing focusability was observed, suggesting that specular reflectivity alone is not an adequate guide to the ideal range of plasma mirror operation. It was found that to achieve high reflectivity with negligible phasefront distortion of the reflected beam the inequality csΔt\\<λLaser must be met (cs: sound speed, Δt: time from plasma formation to the peak of the pulse). The achievable contrast enhancement is given by the ratio of plasma mirror reflectivity to cold reflectivity.}", - issn = {0034-6748}, - doi = {10.1063/1.1646737}, - url = {https://doi.org/10.1063/1.1646737}, - eprint = {https://pubs.aip.org/aip/rsi/article-pdf/75/3/645/8814694/645\_1\_online.pdf}, +abstract = "{Plasma mirrors are devices capable of switching very high laser powers on subpicosecond time scales with a dynamic range of 20–30 dB. A detailed study of their performance in the near-field of the laser beam is presented, a setup relevant to improving the pulse contrast of modern ultrahigh power lasers (TW–PW). The conditions under which high reflectivity can be achieved and focusability of the reflected beam retained are identified. At higher intensities a region of high specular reflectivity with rapidly decreasing focusability was observed, suggesting that specular reflectivity alone is not an adequate guide to the ideal range of plasma mirror operation. It was found that to achieve high reflectivity with negligible phasefront distortion of the reflected beam the inequality csΔt\\<λLaser must be met (cs: sound speed, Δt: time from plasma formation to the peak of the pulse). The achievable contrast enhancement is given by the ratio of plasma mirror reflectivity to cold reflectivity.}", +author = {Dromey, B. and Kar, S. and Zepf, M. and Foster, P.}, +doi = {10.1063/1.1646737}, +eprint = {https://pubs.aip.org/aip/rsi/article-pdf/75/3/645/8814694/645\_1\_online.pdf}, +issn = {0034-6748}, +journal = {Review of Scientific Instruments}, +month = {Feb}, +number = {3}, +pages = {645-649}, +title = {{The plasma mirror—A subpicosecond optical switch for ultrahigh power lasers}}, +url = {https://doi.org/10.1063/1.1646737}, +volume = {75}, +year = {2004} } @article{Roedel2010, - title = {{High repetition rate plasma mirror for temporal contrast enhancement of terawatt femtosecond laser pulses by three orders of magnitude}}, - volume = {103}, - ISSN = {1432-0649}, - url = {http://dx.doi.org/10.1007/s00340-010-4329-7}, - DOI = {10.1007/s00340-010-4329-7}, - number = {2}, - journal = {Applied Physics B}, - publisher = {Springer Science and Business Media LLC}, - author = {R\"{o}del, C. and Heyer, M. and Behmke, M. and K\"{u}bel, M. and J\"{a}ckel, O. and Ziegler, W. and Ehrt, D. and Kaluza, M. C. and Paulus, G. G.}, - year = {2010}, - month = nov, - pages = {295–302} +author = {R\"{o}del,  C. and Heyer,  M. and Behmke,  M. and K\"{u}bel,  M. and J\"{a}ckel,  O. and Ziegler,  W. and Ehrt,  D. and Kaluza,  M. C. and Paulus,  G. G.}, +DOI = {10.1007/s00340-010-4329-7}, +ISSN = {1432-0649}, +journal = {Applied Physics B}, +month = {Nov}, +number = {2}, +pages = {295–302}, +publisher = {Springer Science and Business Media LLC}, +title = {{High repetition rate plasma mirror for temporal contrast enhancement of terawatt femtosecond laser pulses by three orders of magnitude}}, +url = {http://dx.doi.org/10.1007/s00340-010-4329-7}, +volume = {103}, +year = {2010} } @misc{SandbergPASC24, - author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, - title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, - booktitle = {Proc. of PASC24}, - venue = {Zuerich, Switzerland}, - address = {Zuerich, Switzerland}, - series = {PASC'24 - Platform for Advanced Scientific Computing}, - year = {2024}, - note = {submitted} +address = {Zuerich, Switzerland}, +author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, +booktitle = {Proc. of PASC24}, +note = {submitted}, +series = {PASC'24 - Platform for Advanced Scientific Computing}, +title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, +venue = {Zuerich, Switzerland}, +year = {2024} } @inproceedings{SandbergIPAC23, - author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, - title = {{Hybrid beamline element ML-training for surrogates in the ImpactX beam-dynamics code}}, - booktitle = {Proc. 14th International Particle Accelerator Conference}, - pages = {2885-2888}, - paper = {WEPA101}, - venue = {Venice, Italy}, - address = {Venice, Italy}, - series = {IPAC'23 - 14th International Particle Accelerator Conference}, - number = {14}, - publisher = {JACoW Publishing, Geneva, Switzerland}, - month = {05}, - year = {2023}, - issn = {2673-5490}, - isbn = {978-3-95450-231-8}, - doi = {10.18429/JACoW-IPAC2023-WEPA101}, - url = {https://indico.jacow.org/event/41/contributions/2276}, - language = {English} +address = {Venice, Italy}, +author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, +booktitle = {Proc. 14th International Particle Accelerator Conference}, +doi = {10.18429/JACoW-IPAC2023-WEPA101}, +isbn = {978-3-95450-231-8}, +issn = {2673-5490}, +language = {English}, +month = {May}, +number = {14}, +pages = {2885-2888}, +paper = {WEPA101}, +publisher = {JACoW Publishing, Geneva, Switzerland}, +series = {IPAC'23 - 14th International Particle Accelerator Conference}, +title = {{Hybrid beamline element ML-training for surrogates in the ImpactX beam-dynamics code}}, +url = {https://indico.jacow.org/event/41/contributions/2276}, +venue = {Venice, Italy}, +year = {2023} } diff --git a/Docs/source/theory/boosted_frame.rst b/Docs/source/theory/boosted_frame.rst index 4b80c3d2ccb..7dbcfb9ad68 100644 --- a/Docs/source/theory/boosted_frame.rst +++ b/Docs/source/theory/boosted_frame.rst @@ -420,7 +420,7 @@ Haber, I, R Lee, Hh Klein, and Jp Boris. 1973. “Advances in Electromagnetic Si .. raw:: html -
+
Kirchen, M., R. Lehe, B. B. Godfrey, I. Dornmair, S. Jalas, K. Peters, J.-L. Vay, and A. R. Maier. 2016. “Stable discrete representation of relativistically drifting plasmas.” *arXiv:1608.00215*. @@ -440,7 +440,7 @@ Lehe, Rémi, Manuel Kirchen, Igor A. Andriyash, Brendan B. Godfrey, and Jean-Luc .. raw:: html -
+
Lehe, R., M. Kirchen, B. B. Godfrey, A. R. Maier, and J.-L. Vay. 2016. “Elimination of Numerical Cherenkov Instability in flowing-plasma Particle-In-Cell simulations by using Galilean coordinates.” *arXiv:1608.00227*. diff --git a/Docs/source/theory/picsar_theory.rst b/Docs/source/theory/picsar_theory.rst index 0aadbd558f0..a7a3e33de9c 100644 --- a/Docs/source/theory/picsar_theory.rst +++ b/Docs/source/theory/picsar_theory.rst @@ -764,7 +764,7 @@ Pukhov, A. 1999. “Three-dimensional electromagnetic relativistic particle-in-c .. raw:: html -
+
Vay, Jean-Luc, Irving Haber, and Brendan B Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243 (June): 260–68. https://doi.org/10.1016/j.jcp.2013.03.010. @@ -844,7 +844,7 @@ Villasenor, J, and O Buneman. 1992. “Rigorous Charge Conservation for Local El .. raw:: html -
+
Vincenti, H., and J.-L. Vay. 2016. “Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver.” *Computer Physics Communications* 200 (March). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 147–67. https://doi.org/10.1016/j.cpc.2015.11.009. From a2022bbd674553e71b1e8afa93a76e4ef7199900 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 14 Dec 2023 14:24:42 -0800 Subject: [PATCH 045/176] Doc: Fix Sphinx Warnings Fix formatting issues. --- Docs/source/developers/testing.rst | 2 +- Docs/source/install/cmake.rst | 2 +- Docs/source/install/hpc/leonardo.rst | 4 ++-- Docs/source/refs.bib | 1 + Docs/source/theory/picsar_theory.rst | 4 ++-- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Docs/source/developers/testing.rst b/Docs/source/developers/testing.rst index 2bd74e862f6..2b3fe989f2f 100644 --- a/Docs/source/developers/testing.rst +++ b/Docs/source/developers/testing.rst @@ -18,7 +18,7 @@ We slightly modify this file in ``Regression/prepare_file_ci.py``. For example, if you like to change the compiler to compilation to build on Nvidia GPUs, modify this block to add ``-DWarpX_COMPUTE=CUDA``: -.. code-block:: toml +.. code-block:: ini [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index a76e8c212a2..e299a60c75b 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -130,7 +130,7 @@ CMake Option Default & Values Des ``WarpX_picsar_repo`` ``https://github.com/ECP-WarpX/picsar.git`` Repository URI to pull and build PICSAR from ``WarpX_picsar_branch`` *we set and maintain a compatible commit* Repository branch for ``WarpX_picsar_repo`` ``WarpX_picsar_internal`` **ON**/OFF Needs a pre-installed PICSAR library if set to ``OFF`` -``WarpX_pyamrex_src`` *None* Path to PICSAR source directory (preferred if set) +``WarpX_pyamrex_src`` *None* Path to PICSAR source directory (preferred if set) ``WarpX_pyamrex_repo`` ``https://github.com/AMReX-Codes/pyamrex.git`` Repository URI to pull and build pyAMReX from ``WarpX_pyamrex_branch`` *we set and maintain a compatible commit* Repository branch for ``WarpX_pyamrex_repo`` ``WarpX_pyamrex_internal`` **ON**/OFF Needs a pre-installed pyAMReX library if set to ``OFF`` diff --git a/Docs/source/install/hpc/leonardo.rst b/Docs/source/install/hpc/leonardo.rst index 3de7f4755cb..1f3c1e8fbfa 100644 --- a/Docs/source/install/hpc/leonardo.rst +++ b/Docs/source/install/hpc/leonardo.rst @@ -103,9 +103,9 @@ Additionally, the following commands will install WarpX as a Python module: cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_PYTHON=ON -DWarpX_APP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu_py -j 16 --target pip_install -Now, you can :ref:`submit Leonardo compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Now, you can :ref:`submit Leonardo compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Leonardo jobs (:ref:`example inputs `). -For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$CINECA_SCRATCH``. +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$CINECA_SCRATCH``. .. _building-leonardo-update: diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 5d6d649da2d..271b606377a 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -64,6 +64,7 @@ @article{winske2022hybrid archivePrefix = {arXiv}, author = {D. Winske and Homa Karimabadi and Ari Le and N. Omidi and Vadim Roytershteyn and Adam Stanier}, eprint = {2204.01676}, +journal = {arXiv}, primaryClass = {physics.plasm-ph}, title = {{Hybrid codes (massless electron fluid)}}, year = {2022} diff --git a/Docs/source/theory/picsar_theory.rst b/Docs/source/theory/picsar_theory.rst index a7a3e33de9c..63672c0fd0a 100644 --- a/Docs/source/theory/picsar_theory.rst +++ b/Docs/source/theory/picsar_theory.rst @@ -503,14 +503,14 @@ Three variations are considered: are first interpolated to the staggered positions on an auxiliary grid). -As shown in :raw-latex:`\cite{BirdsallLangdon,HockneyEastwoodBook,LewisJCP1972}`, +As shown in :cite:t:`BirdsallLangdon,HockneyEastwoodBook,LewisJCP1972`, the momentum and energy conserving schemes conserve momentum and energy respectively at the limit of infinitesimal time steps and generally offer better conservation of the respective quantities for a finite time step. The uniform scheme does not conserve momentum nor energy in the sense defined for the others but is given for completeness, as it has been shown to offer some interesting properties in the modeling -of relativistically drifting plasmas :raw-latex:`\cite{GodfreyJCP2013}`. +of relativistically drifting plasmas :cite:p:`GodfreyJCP2013`. .. _theory-pic-filter: From af003486573df8a4196de523a6288b75cf1dce3c Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 14 Dec 2023 14:37:35 -0800 Subject: [PATCH 046/176] Doc: Examples Before APIs Change ToC order as discussed in dev meeting. --- Docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 4c5f791ca6b..75287734ff6 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -75,9 +75,9 @@ Usage :hidden: usage/how_to_run + usage/examples usage/python usage/parameters - usage/examples usage/workflows usage/faq From 262895b36e49f0f183a954301c9113eb051f88f6 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Thu, 14 Dec 2023 21:09:00 -0800 Subject: [PATCH 047/176] Bibliography: make author family names appear last (#4506) * bib-formatting * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Rename bib style class * Delete unused imports * Add comments to conf.py * Tweaks to .bib files * Add pybtex to Doc requirements * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * comment out 'lastfirst' --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/conf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 3735459bf76..93831b9054d 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -70,12 +70,12 @@ # BaseStyle class in pybtex/style/formatting/__init__.py # UnsrtStyle class in pybtex/style/formating/unsrt.py class WarpXBibStyle(UnsrtStyle): - # We want the family name, i.e, "last" name, of an author to appear first. - default_name_style = 'lastfirst' + # This option makes the family name, i.e, "last" name, of an author to appear first. + # default_name_style = 'lastfirst' def __init__(self, *args, **kwargs): - # We want the given names of an author to be abbreviated to just initials. - # Example: "Jean-Luc Vay" becomes "Vay, J.-L." + # This option makes the given names of an author abbreviated to just initials. + # Example: "Jean-Luc" becomes "J.-L." # Set 'abbreviate_names' to True before calling the superclass (BaseStyle class) initializer kwargs['abbreviate_names'] = True super().__init__(*args, **kwargs) From 832c6eded3d5a5ce68d070ff6316918c7f1d4486 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Thu, 14 Dec 2023 21:10:08 -0800 Subject: [PATCH 048/176] Clang-Tidy: performance-noexcept-move-constructor (#4504) --- .clang-tidy | 1 - Source/Particles/Collision/BinaryCollision/BinaryCollision.H | 4 ++-- Source/Particles/NamedComponentParticleContainer.H | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 831f16e1322..9e742bc6014 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -30,7 +30,6 @@ Checks: ' -modernize-use-trailing-return-type, mpi-*, performance-*, - -performance-noexcept-move-constructor, -performance-unnecessary-copy-initialization, -performance-unnecessary-value-param, portability-*, diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 7a9374ae5f8..e50b4d4f2a6 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -109,8 +109,8 @@ public: BinaryCollision ( BinaryCollision const &) = default; BinaryCollision& operator= ( BinaryCollision const & ) = default; - BinaryCollision ( BinaryCollision&& ) = default; - BinaryCollision& operator= ( BinaryCollision&& ) = default; + BinaryCollision ( BinaryCollision&& ) noexcept = default; + BinaryCollision& operator= ( BinaryCollision&& ) noexcept = default; /** Perform the collisions * diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index c9f66e6c2b0..3be0886425d 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -102,9 +102,9 @@ public: NamedComponentParticleContainer& operator= ( const NamedComponentParticleContainer & ) = delete; /** Move constructor for NamedComponentParticleContainer */ - NamedComponentParticleContainer ( NamedComponentParticleContainer && ) = default; + NamedComponentParticleContainer ( NamedComponentParticleContainer && ) noexcept = default; /** Move operator for NamedComponentParticleContainer */ - NamedComponentParticleContainer& operator= ( NamedComponentParticleContainer && ) = default; + NamedComponentParticleContainer& operator= ( NamedComponentParticleContainer && ) noexcept = default; /** Create an empty particle container * From 3c627da8091c5378a9fe9b75721322a935b341d6 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Thu, 14 Dec 2023 22:13:53 -0800 Subject: [PATCH 049/176] Clang-Tidy: readability-qualified-auto (#4507) --- .clang-tidy | 1 - .../AcceleratorLattice/LatticeElementFinder.cpp | 2 +- .../BackTransformParticleFunctor.cpp | 2 +- .../Diagnostics/ComputeDiagFunctors/JFunctor.cpp | 6 +++--- .../FlushFormats/FlushFormatCheckpoint.cpp | 2 +- .../FlushFormats/FlushFormatPlotfile.cpp | 2 +- Source/Diagnostics/ReducedDiags/FieldProbe.cpp | 4 ++-- .../ReducedDiags/LoadBalanceCosts.cpp | 2 +- Source/Diagnostics/WarpXOpenPMD.cpp | 6 +++--- Source/Initialization/WarpXInitData.cpp | 4 ++-- .../LaserProfilesImpl/LaserProfileFromFile.cpp | 6 +++--- .../BackgroundMCC/BackgroundMCCCollision.cpp | 4 ++-- .../BackgroundStopping/BackgroundStopping.cpp | 2 +- .../Collision/BinaryCollision/BinaryCollision.H | 6 +++--- Source/Particles/Deposition/ChargeDeposition.H | 2 +- Source/Particles/Gather/GetExternalFields.cpp | 2 +- Source/Particles/MultiParticleContainer.cpp | 14 +++++++------- Source/Particles/ParticleBoundaryBuffer.cpp | 2 +- .../ParticleCreation/FilterCopyTransform.H | 8 ++++---- .../FilterCreateTransformFromFAB.H | 4 ++-- Source/Particles/PhotonParticleContainer.cpp | 12 ++++++------ Source/Particles/PhysicalParticleContainer.cpp | 16 ++++++++-------- Source/Particles/Pusher/CopyParticleAttribs.H | 2 +- Source/Particles/Resampling/LevelingThinning.cpp | 4 ++-- .../Particles/RigidInjectedParticleContainer.cpp | 8 ++++---- Source/Particles/Sorting/Partition.cpp | 4 ++-- Source/Particles/WarpXParticleContainer.cpp | 16 ++++++++-------- Source/Utils/ParticleUtils.H | 4 ++-- Source/ablastr/utils/Communication.cpp | 2 +- Source/ablastr/utils/SignalHandling.cpp | 4 ++-- 30 files changed, 76 insertions(+), 77 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 9e742bc6014..4e1463affba 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -43,7 +43,6 @@ Checks: ' -readability-magic-numbers, -readability-make-member-function-const, -readability-named-parameter, - -readability-qualified-auto, -readability-uppercase-literal-suffix ' diff --git a/Source/AcceleratorLattice/LatticeElementFinder.cpp b/Source/AcceleratorLattice/LatticeElementFinder.cpp index 62ebdcc1f4f..ef90871a62b 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.cpp +++ b/Source/AcceleratorLattice/LatticeElementFinder.cpp @@ -97,7 +97,7 @@ LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& int const lev = a_pti.GetLevel(); m_get_position = GetParticlePosition(a_pti, a_offset); - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); m_ux = attribs[PIdx::ux].dataPtr() + a_offset; m_uy = attribs[PIdx::uy].dataPtr() + a_offset; m_uz = attribs[PIdx::uz].dataPtr() + a_offset; diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp index 07804a4fa0c..8a6f0765664 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp @@ -40,7 +40,7 @@ LorentzTransformParticles::LorentzTransformParticles ( const WarpXParIter& a_pti if (tmp_particle_data.empty()) { return; } m_get_position = GetParticlePosition(a_pti, a_offset); - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); m_wpnew = attribs[PIdx::w].dataPtr(); m_uxpnew = attribs[PIdx::ux].dataPtr(); m_uypnew = attribs[PIdx::uy].dataPtr(); diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 4663a5e1a70..b21bbded011 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -36,16 +36,16 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ amrex::Vector, 3 > > current_fp_temp; current_fp_temp.resize(1); - auto& current_fp_x = warpx.getcurrent_fp(m_lev,0); + const auto& current_fp_x = warpx.getcurrent_fp(m_lev,0); current_fp_temp[0][0] = std::make_unique( current_fp_x, amrex::make_alias, 0, current_fp_x.nComp() ); - auto& current_fp_y = warpx.getcurrent_fp(m_lev,1); + const auto& current_fp_y = warpx.getcurrent_fp(m_lev,1); current_fp_temp[0][1] = std::make_unique( current_fp_y, amrex::make_alias, 0, current_fp_y.nComp() ); - auto& current_fp_z = warpx.getcurrent_fp(m_lev,2); + const auto& current_fp_z = warpx.getcurrent_fp(m_lev,2); current_fp_temp[0][2] = std::make_unique( current_fp_z, amrex::make_alias, 0, current_fp_z.nComp() ); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 2247a7eaadf..5f59cd723da 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -172,7 +172,7 @@ FlushFormatCheckpoint::CheckpointParticles ( const std::string& dir, const amrex::Vector& particle_diags) const { - for (auto& part_diag: particle_diags) { + for (const auto& part_diag: particle_diags) { WarpXParticleContainer* pc = part_diag.getParticleContainer(); Vector real_names; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 2c5edf72b12..df73ed34c94 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -343,7 +343,7 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, const amrex::Real time, bool isBTD) const { - for (auto& part_diag : particle_diags) { + for (const auto& part_diag : particle_diags) { WarpXParticleContainer* pc = part_diag.getParticleContainer(); PinnedMemoryParticleContainer* pinned_pc = part_diag.getPinnedParticleContainer(); auto tmp = isBTD ? diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index b39b0e89eaf..9f45392bb0a 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -340,8 +340,8 @@ bool FieldProbe::ProbeInDomain () const auto & warpx = WarpX::GetInstance(); int const lev = 0; const amrex::Geometry& gm = warpx.Geom(lev); - const auto prob_lo = gm.ProbLo(); - const auto prob_hi = gm.ProbHi(); + const auto *const prob_lo = gm.ProbLo(); + const auto *const prob_hi = gm.ProbHi(); /* * Determine if probe exists within simulation boundaries. During 2D simulations, diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 15115a15b08..893b00a5f00 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -87,7 +87,7 @@ void LoadBalanceCosts::ComputeDiags (int step) int nBoxes = 0; for (int lev = 0; lev < nLevels; ++lev) { - const auto cost = WarpX::getCosts(lev); + auto *const cost = WarpX::getCosts(lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( cost, "ERROR: costs are not initialized on level " + std::to_string(lev) + " !"); nBoxes += cost->size(); diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 6d3b27160d4..5100291a94a 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -788,7 +788,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, for (auto i = 0; i < numParticleOnTile; i++) { ids.get()[i] = ablastr::particles::localIDtoGlobal(static_cast(aos[i].id()), static_cast(aos[i].cpu())); } - auto const scalar = openPMD::RecordComponent::SCALAR; + const auto *const scalar = openPMD::RecordComponent::SCALAR; currSpecies["id"][scalar].storeChunk(ids, {offset}, {numParticleOnTile64}); } @@ -1024,7 +1024,7 @@ WarpXOpenPMDPlot::SetupPos ( currSpecies["position"][comp].resetDataset( realType ); } - auto const scalar = openPMD::RecordComponent::SCALAR; + const auto *const scalar = openPMD::RecordComponent::SCALAR; currSpecies["id"][scalar].resetDataset( idType ); } @@ -1036,7 +1036,7 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( amrex::ParticleReal const mass) { auto realType = openPMD::Dataset(openPMD::determineDatatype(), {np}); - auto const scalar = openPMD::RecordComponent::SCALAR; + const auto *const scalar = openPMD::RecordComponent::SCALAR; // define record shape to be number of particles auto const positionComponents = detail::getParticlePositionComponentLabels(true); diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index edf083fd839..3b36a6336bf 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1399,12 +1399,12 @@ WarpX::ReadExternalFieldFromFile ( auto FC_chunk_data = FC.loadChunk(chunk_offset,chunk_extent); series.flush(); - auto FC_data_host = FC_chunk_data.get(); + auto *FC_data_host = FC_chunk_data.get(); // Load data to GPU const size_t total_extent = size_t(extent[0]) * extent[1] * extent[2]; amrex::Gpu::DeviceVector FC_data_gpu(total_extent); - auto FC_data = FC_data_gpu.data(); + auto *FC_data = FC_data_gpu.data(); amrex::Gpu::copy(amrex::Gpu::hostToDevice, FC_data_host, FC_data_host + total_extent, FC_data); // Loop over boxes diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp index 46b47e119af..0fdb45c64f8 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp @@ -441,7 +441,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_cartes const auto tmp_y_max = m_params.y_max; const auto tmp_nx = m_params.nx; const auto tmp_ny = m_params.ny; - const auto p_E_lasy_data = m_params.E_lasy_data.dataPtr(); + const auto *const p_E_lasy_data = m_params.E_lasy_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* @@ -524,7 +524,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_cylind const auto tmp_r_max = m_params.r_max; const auto tmp_nr = m_params.nr; const auto tmp_n_rz_azimuthal_components = m_params.n_rz_azimuthal_components; - const auto p_E_lasy_data = m_params.E_lasy_data.dataPtr(); + const auto *const p_E_lasy_data = m_params.E_lasy_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* @@ -626,7 +626,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_binary const auto tmp_ny = m_params.ny; #endif const auto tmp_nx = m_params.nx; - const auto p_E_binary_data = m_params.E_binary_data.dataPtr(); + const auto *const p_E_binary_data = m_params.E_binary_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index 84c50aff674..98ecdf70e5b 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -276,7 +276,7 @@ BackgroundMCCCollision::doCollisions (amrex::Real cur_time, amrex::Real dt, Mult auto const flvl = species1.finestLevel(); for (int lev = 0; lev <= flvl; ++lev) { - auto cost = WarpX::getCosts(lev); + auto *cost = WarpX::getCosts(lev); // firstly loop over particles box by box and do all particle conserving // scattering @@ -324,7 +324,7 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile auto T_a_func = m_background_temperature_func; // get collision parameters - auto scattering_processes = m_scattering_processes_exe.data(); + auto *scattering_processes = m_scattering_processes_exe.data(); auto const process_count = static_cast(m_scattering_processes_exe.size()); auto const total_collision_prob = m_total_collision_prob; diff --git a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp index 83f74840478..8122d7225b4 100644 --- a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp +++ b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp @@ -104,7 +104,7 @@ BackgroundStopping::doCollisions (amrex::Real cur_time, amrex::Real dt, MultiPar auto const flvl = species.finestLevel(); for (int lev = 0; lev <= flvl; ++lev) { - auto cost = WarpX::getCosts(lev); + auto *cost = WarpX::getCosts(lev); // loop over particles box by box #ifdef _OPENMP diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index e50b4d4f2a6..19e98b8a15a 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -157,8 +157,8 @@ public: auto copy_species1_data = device_copy_species1.data(); auto copy_species2_data = device_copy_species2.data(); #else - auto copy_species1_data = copy_species1.data(); - auto copy_species2_data = copy_species2.data(); + auto *copy_species1_data = copy_species1.data(); + auto *copy_species2_data = copy_species2.data(); #endif if (m_have_product_species){ species1.defineAllParticleTiles(); @@ -240,7 +240,7 @@ public: products_np.push_back(ptile_product.numParticles()); products_mass.push_back(product_species_vector[i]->getMass()); } - auto tile_products_data = tile_products.data(); + auto *tile_products_data = tile_products.data(); if ( m_isSameSpecies ) // species_1 == species_2 { diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index 50ed74439e5..d0db678dfda 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -261,7 +261,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio { using namespace amrex; - auto permutation = a_bins.permutationPtr(); + const auto *permutation = a_bins.permutationPtr(); #if !defined(AMREX_USE_GPU) amrex::ignore_unused(ix_type, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size, bin_size); diff --git a/Source/Particles/Gather/GetExternalFields.cpp b/Source/Particles/Gather/GetExternalFields.cpp index a4ded3c9f69..8e3c679aa3a 100644 --- a/Source/Particles/Gather/GetExternalFields.cpp +++ b/Source/Particles/Gather/GetExternalFields.cpp @@ -71,7 +71,7 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset if (mypc.m_E_ext_particle_s == "repeated_plasma_lens") { m_Etype = RepeatedPlasmaLens; } if (mypc.m_B_ext_particle_s == "repeated_plasma_lens") { m_Btype = RepeatedPlasmaLens; } m_dt = warpx.getdt(a_pti.GetLevel()); - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); m_ux = attribs[PIdx::ux].dataPtr() + a_offset; m_uy = attribs[PIdx::uy].dataPtr() + a_offset; m_uz = attribs[PIdx::uz].dataPtr() + a_offset; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 369e653ee8b..213a5344ee8 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -711,7 +711,7 @@ MultiParticleContainer::SetParticleDistributionMap (int lev, DistributionMapping void MultiParticleContainer::ContinuousInjection (const RealBox& injection_box) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ pc->ContinuousInjection(injection_box); } @@ -721,7 +721,7 @@ MultiParticleContainer::ContinuousInjection (const RealBox& injection_box) const void MultiParticleContainer::UpdateAntennaPosition (const amrex::Real dt) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ pc->UpdateAntennaPosition(dt); } @@ -732,7 +732,7 @@ int MultiParticleContainer::doContinuousInjection () const { int warpx_do_continuous_injection = 0; - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ warpx_do_continuous_injection = 1; } @@ -747,7 +747,7 @@ MultiParticleContainer::doContinuousInjection () const void MultiParticleContainer::ContinuousFluxInjection (amrex::Real t, amrex::Real dt) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ pc->ContinuousFluxInjection(t, dt); } } @@ -871,7 +871,7 @@ MultiParticleContainer::doFieldIonization (int lev, auto& pc_product = allcontainers[pc_source->ionization_product]; const SmartCopyFactory copy_factory(*pc_source, *pc_product); - auto phys_pc_ptr = static_cast(pc_source.get()); + auto *phys_pc_ptr = static_cast(pc_source.get()); auto Copy = copy_factory.getSmartCopy(); auto Transform = IonizationTransformFunc(); @@ -1502,7 +1502,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, const SmartCopyFactory copy_factory_ele(*pc_source, *pc_product_ele); const SmartCopyFactory copy_factory_pos(*pc_source, *pc_product_pos); - auto phys_pc_ptr = static_cast(pc_source.get()); + auto *phys_pc_ptr = static_cast(pc_source.get()); const auto Filter = phys_pc_ptr->getPairGenerationFilterFunc(); const auto CopyEle = copy_factory_ele.getSmartCopy(); @@ -1582,7 +1582,7 @@ void MultiParticleContainer::doQedQuantumSync (int lev, allcontainers[pc_source->m_qed_quantum_sync_phot_product]; const SmartCopyFactory copy_factory_phot(*pc_source, *pc_product_phot); - auto phys_pc_ptr = + auto *phys_pc_ptr = static_cast(pc_source.get()); const auto Filter = phys_pc_ptr->getPhotonEmissionFilterFunc(); diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index c4024417539..54c4396379d 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -149,7 +149,7 @@ void ParticleBoundaryBuffer::printNumParticles () const { { for (int iside = 0; iside < 2; ++iside) { - auto& buffer = m_particle_containers[2*idim+iside]; + const auto& buffer = m_particle_containers[2*idim+iside]; for (int i = 0; i < numSpecies(); ++i) { const auto np = buffer[i].isDefined() ? buffer[i].TotalNumberOfParticles(false) : 0; diff --git a/Source/Particles/ParticleCreation/FilterCopyTransform.H b/Source/Particles/ParticleCreation/FilterCopyTransform.H index 264b8b86e29..26d70fc6718 100644 --- a/Source/Particles/ParticleCreation/FilterCopyTransform.H +++ b/Source/Particles/ParticleCreation/FilterCopyTransform.H @@ -63,7 +63,7 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind const Index num_added = N * total; dst.resize(std::max(dst_index + num_added, dst.numParticles())); - const auto p_offsets = offsets.dataPtr(); + auto *const p_offsets = offsets.dataPtr(); const auto src_data = src.getParticleTileData(); const auto dst_data = dst.getParticleTileData(); @@ -133,7 +133,7 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, Gpu::DeviceVector mask(np); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); const auto src_data = src.getParticleTileData(); amrex::ParallelForRNG(np, @@ -207,7 +207,7 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, dst1.resize(std::max(dst1_index + num_added, dst1.numParticles())); dst2.resize(std::max(dst2_index + num_added, dst2.numParticles())); - auto p_offsets = offsets.dataPtr(); + auto *p_offsets = offsets.dataPtr(); const auto src_data = src.getParticleTileData(); const auto dst1_data = dst1.getParticleTileData(); @@ -290,7 +290,7 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, Gpu::DeviceVector mask(np); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); const auto src_data = src.getParticleTileData(); amrex::ParallelForRNG(np, diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 04973d2d7b8..8f16530f020 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -86,7 +86,7 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B dst1.resize(std::max(dst1_index + num_added, dst1.numParticles())); dst2.resize(std::max(dst2_index + num_added, dst2.numParticles())); - auto p_offsets = offsets.dataPtr(); + auto *p_offsets = offsets.dataPtr(); const auto dst1_data = dst1.getParticleTileData(); const auto dst2_data = dst2.getParticleTileData(); @@ -188,7 +188,7 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B Gpu::DeviceVector mask(ncells); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); // for loop over all cells in the box. We apply the filter function to each cell // and store the result in arrNumPartCreation. If the result is strictly greater than diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index 4dc442bdf95..732d986a897 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -205,17 +205,17 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, nox, galerkin_interpolation); } - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; // workaround for nvcc + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; // workaround for nvcc if constexpr (exteb_control == has_exteb) { getExternalEB(i, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } #ifdef WARPX_QED - [[maybe_unused]] auto& evolve_opt_tmp = evolve_opt; - [[maybe_unused]] auto p_optical_depth_BW_tmp = p_optical_depth_BW; - [[maybe_unused]] auto ux_tmp = ux; // for nvhpc - [[maybe_unused]] auto uy_tmp = uy; - [[maybe_unused]] auto uz_tmp = uz; + [[maybe_unused]] const auto& evolve_opt_tmp = evolve_opt; + [[maybe_unused]] auto *p_optical_depth_BW_tmp = p_optical_depth_BW; + [[maybe_unused]] auto *ux_tmp = ux; // for nvhpc + [[maybe_unused]] auto *uy_tmp = uy; + [[maybe_unused]] auto *uz_tmp = uz; [[maybe_unused]] auto dt_tmp = dt; if constexpr (qed_control == has_qed) { evolve_opt(ux[i], uy[i], uz[i], Exp, Eyp, Ezp, Bxp, Byp, Bzp, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index f21e3d32c08..729abd6f092 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1109,7 +1109,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // count the number of particles that each cell in overlap_box could add Gpu::DeviceVector counts(overlap_box.numPts(), 0); Gpu::DeviceVector offset(overlap_box.numPts()); - auto pcounts = counts.data(); + auto *pcounts = counts.data(); const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { @@ -1282,7 +1282,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // particles, in particular does not consider xmin, xmax etc.). // The invalid ones are given negative ID and are deleted during the // next redistribute. - const auto poffset = offset.data(); + auto *const poffset = offset.data(); #ifdef WARPX_DIM_RZ const bool rz_random_theta = m_rz_random_theta; #endif @@ -1686,7 +1686,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // count the number of particles that each cell in overlap_box could add Gpu::DeviceVector counts(overlap_box.numPts(), 0); Gpu::DeviceVector offset(overlap_box.numPts()); - auto pcounts = counts.data(); + auto *pcounts = counts.data(); const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { @@ -1837,7 +1837,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // particles, in particular does not consider xmin, xmax etc.). // The invalid ones are given negative ID and are deleted during the // next redistribute. - const auto poffset = offset.data(); + auto *const poffset = offset.data(); amrex::ParallelForRNG(overlap_box, [=] AMREX_GPU_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept { @@ -2663,7 +2663,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, } // Externally applied E and B-field in Cartesian co-ordinates - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } @@ -2913,7 +2913,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, nox, galerkin_interpolation); } - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } @@ -2950,8 +2950,8 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, #ifdef WARPX_QED [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; - [[maybe_unused]] auto foo_podq = p_optical_depth_QSR; - [[maybe_unused]] auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc + [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; + [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc if constexpr (qed_control == has_qed) { if (local_has_quantum_sync) { evolve_opt(ux[ip], uy[ip], uz[ip], diff --git a/Source/Particles/Pusher/CopyParticleAttribs.H b/Source/Particles/Pusher/CopyParticleAttribs.H index d20626a53a7..4f961fe94aa 100644 --- a/Source/Particles/Pusher/CopyParticleAttribs.H +++ b/Source/Particles/Pusher/CopyParticleAttribs.H @@ -51,7 +51,7 @@ struct CopyParticleAttribs { if (tmp_particle_data.empty()) { return; } - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); uxp = attribs[PIdx::ux].dataPtr() + a_offset; uyp = attribs[PIdx::uy].dataPtr() + a_offset; diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index e48c478a6f7..680e33ebe6a 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -71,8 +71,8 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, auto bins = ParticleUtils::findParticlesInEachCell(lev, pti, ptile); const auto n_cells = static_cast(bins.numBins()); - const auto indices = bins.permutationPtr(); - const auto cell_offsets = bins.offsetsPtr(); + auto *const indices = bins.permutationPtr(); + auto *const cell_offsets = bins.offsetsPtr(); const amrex::Real target_ratio = m_target_ratio; const int min_ppc = m_min_ppc; diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index ff7dbf48166..162645a9e57 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -117,9 +117,9 @@ RigidInjectedParticleContainer::RemapParticles() for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { const auto& attribs = pti.GetAttribs(); - const auto uxp = attribs[PIdx::ux].dataPtr(); - const auto uyp = attribs[PIdx::uy].dataPtr(); - const auto uzp = attribs[PIdx::uz].dataPtr(); + const auto *const uxp = attribs[PIdx::ux].dataPtr(); + const auto *const uyp = attribs[PIdx::uy].dataPtr(); + const auto *const uzp = attribs[PIdx::uz].dataPtr(); const auto GetPosition = GetParticlePosition(pti); auto SetPosition = SetParticlePosition(pti); @@ -447,7 +447,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp index 18bebed185f..58511cfd5e7 100644 --- a/Source/Particles/Sorting/Partition.cpp +++ b/Source/Particles/Sorting/Partition.cpp @@ -76,7 +76,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // - Find the indices that reorder particles so that the last particles // are in the larger buffer fillWithConsecutiveIntegers( pid ); - auto const sep = stablePartition( pid.begin(), pid.end(), inexflag ); + auto *const sep = stablePartition( pid.begin(), pid.end(), inexflag ); // At the end of this step, `pid` contains the indices that should be used to // reorder the particles, and `sep` is the position in the array that // separates the particles that deposit/gather on the fine patch (first part) @@ -110,7 +110,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // the smaller buffer, by looking up the mask. Store the answer in `inexflag`. amrex::ParallelFor( np - n_fine, fillBufferFlagRemainingParticles(pti, bmasks, inexflag, Geom(lev), pid, n_fine) ); - auto const sep2 = stablePartition( sep, pid.end(), inexflag ); + auto *const sep2 = stablePartition( sep, pid.end(), inexflag ); if (bmasks == gather_masks) { nfine_gather = iteratorDistance(pid.begin(), sep2); diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index a1dc8d14f4a..447721a3e68 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -470,7 +470,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, { auto& ptile = ParticlesAt(lev, pti); auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto *pstruct_ptr = aos().dataPtr(); const int ntiles = numTilesInBox(box, true, bin_size); @@ -795,7 +795,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, auto& ptile = ParticlesAt(lev, pti); auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto *pstruct_ptr = aos().dataPtr(); Box box = pti.validbox(); box.grow(ng_rho); @@ -824,15 +824,15 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, auto& ptile = ParticlesAt(lev, pti); auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto *pstruct_ptr = aos().dataPtr(); Box box = pti.validbox(); box.grow(ng_rho); const amrex::IntVect bin_size = WarpX::shared_tilesize; - const auto offsets_ptr = bins.offsetsPtr(); - auto tbox_ptr = tboxes.dataPtr(); - auto permutation = bins.permutationPtr(); + auto *const offsets_ptr = bins.offsetsPtr(); + auto *tbox_ptr = tboxes.dataPtr(); + auto *permutation = bins.permutationPtr(); amrex::ParallelFor(bins.numBins(), [=] AMREX_GPU_DEVICE (int ibin) { const auto bin_start = offsets_ptr[ibin]; @@ -857,7 +857,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; - const auto boxes_ptr = tboxes.dataPtr(); + auto *const boxes_ptr = tboxes.dataPtr(); reduce_op.eval(tboxes.size(), reduce_data, [=] AMREX_GPU_DEVICE (int i) -> ReduceTuple { @@ -1108,7 +1108,7 @@ amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { for (int lev = 0; lev <= nLevels; ++lev) { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - const auto wp = pti.GetAttribs(PIdx::w).data(); + auto *const wp = pti.GetAttribs(PIdx::w).data(); reduce_op.eval(pti.numParticles(), reduce_data, [=] AMREX_GPU_DEVICE (int ip) diff --git a/Source/Utils/ParticleUtils.H b/Source/Utils/ParticleUtils.H index ecd6b7c2739..b04176d4d83 100644 --- a/Source/Utils/ParticleUtils.H +++ b/Source/Utils/ParticleUtils.H @@ -181,8 +181,8 @@ namespace ParticleUtils { */ AMREX_GPU_HOST_DEVICE AMREX_INLINE bool containsInclusive (amrex::RealBox const& tilebox, amrex::XDim3 const point) { - const auto xlo = tilebox.lo(); - const auto xhi = tilebox.hi(); + const auto *const xlo = tilebox.lo(); + const auto *const xhi = tilebox.hi(); return AMREX_D_TERM((xlo[0] <= point.x) && (point.x <= xhi[0]), && (xlo[1] <= point.y) && (point.y <= xhi[1]), && (xlo[2] <= point.z) && (point.z <= xhi[2])); diff --git a/Source/ablastr/utils/Communication.cpp b/Source/ablastr/utils/Communication.cpp index a3cd9dec98a..bfe34d7d875 100644 --- a/Source/ablastr/utils/Communication.cpp +++ b/Source/ablastr/utils/Communication.cpp @@ -124,7 +124,7 @@ void FillBoundary (amrex::Vector const &mf, bool do_single_precision_comms, const amrex::Periodicity &period, std::optional nodal_sync) { - for (auto x : mf) { + for (auto *x : mf) { ablastr::utils::communication::FillBoundary(*x, do_single_precision_comms, period, nodal_sync); } } diff --git a/Source/ablastr/utils/SignalHandling.cpp b/Source/ablastr/utils/SignalHandling.cpp index 3681eb0e171..9521e5f9c3d 100644 --- a/Source/ablastr/utils/SignalHandling.cpp +++ b/Source/ablastr/utils/SignalHandling.cpp @@ -172,12 +172,12 @@ SignalHandling::CheckSignals () } #if defined(AMREX_USE_MPI) - auto comm = amrex::ParallelDescriptor::Communicator(); // Due to a bug in Cray's MPICH 8.1.13 implementation (CUDA builds on Perlmutter@NERSC in 2022), // we cannot use the MPI_CXX_BOOL C++ datatype here. See WarpX PR #3029 and NERSC INC0183281 static_assert(sizeof(bool) == 1, "We communicate bools as 1 byte-sized type in MPI"); BL_MPI_REQUIRE(MPI_Ibcast(signal_actions_requested, SIGNAL_REQUESTS_SIZE, - MPI_BYTE, 0, comm,&signal_mpi_ibcast_request)); + MPI_BYTE, 0, amrex::ParallelDescriptor::Communicator(), + &signal_mpi_ibcast_request)); #endif } From 7233e4b4872e846622825b45956c29442933d948 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 15 Dec 2023 08:45:05 -0800 Subject: [PATCH 050/176] Add ability to run checksum on openPMD files (#4519) * Add ability to evaluate checksum on openPMD files * Switch to openpmd output for 3D laser wakefield * Fix issues with CI on plotfile * Fix Python syntax * Prevent error where the reg_test code does not find plotfiles * Make the openPMD checksum identical to checkpoint checksum --- .../laser_acceleration/inputs_3d | 1 + .../analysis_default_openpmd_regression.py | 20 ++ Regression/Checksum/checksum.py | 208 ++++++++++++------ Regression/Checksum/checksumAPI.py | 103 +++++---- Regression/WarpX-tests.ini | 6 +- 5 files changed, 219 insertions(+), 119 deletions(-) create mode 100755 Examples/analysis_default_openpmd_regression.py diff --git a/Examples/Physics_applications/laser_acceleration/inputs_3d b/Examples/Physics_applications/laser_acceleration/inputs_3d index 27ecc00117b..bdcfd7676a4 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs_3d +++ b/Examples/Physics_applications/laser_acceleration/inputs_3d @@ -77,6 +77,7 @@ diagnostics.diags_names = diag1 diag1.intervals = 100 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho +diag1.format = openpmd # Reduced Diagnostics warpx.reduced_diags_names = FP diff --git a/Examples/analysis_default_openpmd_regression.py b/Examples/analysis_default_openpmd_regression.py new file mode 100755 index 00000000000..3aadc49ac51 --- /dev/null +++ b/Examples/analysis_default_openpmd_regression.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import os +import re +import sys + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Get name of the test +test_name = os.path.split(os.getcwd())[1] + +# Run checksum regression test +if re.search( 'single_precision', fn ): + checksumAPI.evaluate_checksum(test_name, fn, output_format='openpmd', rtol=2.e-6) +else: + checksumAPI.evaluate_checksum(test_name, fn, output_format='openpmd') diff --git a/Regression/Checksum/checksum.py b/Regression/Checksum/checksum.py index cc3c53a24c2..454edfc2606 100644 --- a/Regression/Checksum/checksum.py +++ b/Regression/Checksum/checksum.py @@ -11,6 +11,8 @@ from benchmark import Benchmark import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c import yt yt.funcs.mylog.setLevel(50) @@ -20,19 +22,22 @@ class Checksum: """Class for checksum comparison of one test. """ - def __init__(self, test_name, plotfile, do_fields=True, do_particles=True): + def __init__(self, test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): """ Checksum constructor. - Store test_name and plotfile name, and compute checksum - from plotfile and store it in self.data. + Store test_name, output file name and format, compute checksum + from output file and store it in self.data. Parameters ---------- test_name: string Name of test, as found between [] in .ini file. - plotfile: string - Plotfile from which the checksum is computed. + output_file: string + Output file from which the checksum is computed. + + output_format: string + Format of the output file (plotfile, openpmd). do_fields: bool, default=True Whether to compare fields in the checksum. @@ -42,83 +47,142 @@ def __init__(self, test_name, plotfile, do_fields=True, do_particles=True): """ self.test_name = test_name - self.plotfile = plotfile - self.data = self.read_plotfile(do_fields=do_fields, - do_particles=do_particles) + self.output_file = output_file + self.output_format = output_format + self.data = self.read_output_file(do_fields=do_fields, + do_particles=do_particles) - def read_plotfile(self, do_fields=True, do_particles=True): + def read_output_file(self, do_fields=True, do_particles=True): """ - Get checksum from plotfile. - Read an AMReX plotfile with yt, compute 1 checksum per field and return - all checksums in a dictionary. + Get checksum from output file. + Read an AMReX plotfile with yt or an openPMD file with openPMD viewer, + compute 1 checksum per field and return all checksums in a dictionary. The checksum of quantity Q is max(abs(Q)). Parameters ---------- do_fields: bool, default=True - Whether to read fields from the plotfile. + Whether to read fields from the output file. do_particles: bool, default=True - Whether to read particles from the plotfile. + Whether to read particles from the output file. """ - ds = yt.load(self.plotfile) - # yt 4.0+ has rounding issues with our domain data: - # RuntimeError: yt attempted to read outside the boundaries - # of a non-periodic domain along dimension 0. - if 'force_periodicity' in dir(ds): ds.force_periodicity() - grid_fields = [item for item in ds.field_list if item[0] == 'boxlib'] - - # "fields"/"species" we remove: - # - nbody: added by yt by default, unused by us - species_list = set([item[0] for item in ds.field_list if - item[1][:9] == 'particle_' and - item[0] != 'all' and - item[0] != 'nbody']) - - data = {} - - # Compute checksum for field quantities - if do_fields: - for lev in range(ds.max_level+1): - data_lev = {} - lev_grids = [grid for grid in ds.index.grids - if grid.Level == lev] - # Warning: For now, we assume all levels are rectangular - LeftEdge = np.min( - np.array([grid.LeftEdge.v for grid in lev_grids]), axis=0) - all_data_level = ds.covering_grid( - level=lev, left_edge=LeftEdge, dims=ds.domain_dimensions) - for field in grid_fields: - Q = all_data_level[field].v.squeeze() - data_lev[field[1]] = np.sum(np.abs(Q)) - data['lev=' + str(lev)] = data_lev - - # Compute checksum for particle quantities - if do_particles: - ad = ds.all_data() - for species in species_list: - # properties we remove: - # - particle_cpu/id: they depend on the parallelism: MPI-ranks and - # on-node acceleration scheme, thus not portable - # and irrelevant for physics benchmarking - part_fields = [item[1] for item in ds.field_list - if item[0] == species and - item[1] != 'particle_cpu' and - item[1] != 'particle_id' - ] - data_species = {} - for field in part_fields: - Q = ad[(species, field)].v - data_species[field] = np.sum(np.abs(Q)) - data[species] = data_species + if self.output_format == 'plotfile': + ds = yt.load(self.output_file) + # yt 4.0+ has rounding issues with our domain data: + # RuntimeError: yt attempted to read outside the boundaries + # of a non-periodic domain along dimension 0. + if 'force_periodicity' in dir(ds): ds.force_periodicity() + grid_fields = [item for item in ds.field_list if item[0] == 'boxlib'] + + # "fields"/"species" we remove: + # - nbody: added by yt by default, unused by us + species_list = set([item[0] for item in ds.field_list if + item[1][:9] == 'particle_' and + item[0] != 'all' and + item[0] != 'nbody']) + + data = {} + + # Compute checksum for field quantities + if do_fields: + for lev in range(ds.max_level+1): + data_lev = {} + lev_grids = [grid for grid in ds.index.grids + if grid.Level == lev] + # Warning: For now, we assume all levels are rectangular + LeftEdge = np.min( + np.array([grid.LeftEdge.v for grid in lev_grids]), axis=0) + all_data_level = ds.covering_grid( + level=lev, left_edge=LeftEdge, dims=ds.domain_dimensions) + for field in grid_fields: + Q = all_data_level[field].v.squeeze() + data_lev[field[1]] = np.sum(np.abs(Q)) + data['lev=' + str(lev)] = data_lev + + # Compute checksum for particle quantities + if do_particles: + ad = ds.all_data() + for species in species_list: + # properties we remove: + # - particle_cpu/id: they depend on the parallelism: MPI-ranks and + # on-node acceleration scheme, thus not portable + # and irrelevant for physics benchmarking + part_fields = [item[1] for item in ds.field_list + if item[0] == species and + item[1] != 'particle_cpu' and + item[1] != 'particle_id' + ] + data_species = {} + for field in part_fields: + Q = ad[(species, field)].v + data_species[field] = np.sum(np.abs(Q)) + data[species] = data_species + + elif self.output_format == 'openpmd': + # Load time series + ts = OpenPMDTimeSeries(self.output_file) + data = {} + # Compute number of MR levels + # TODO This calculation of nlevels assumes that the last element + # of level_fields is by default on the highest MR level. + level_fields = [field for field in ts.avail_fields if 'lvl' in field] + nlevels = 0 if level_fields == [] else int(level_fields[-1][-1]) + # Compute checksum for field quantities + if do_fields: + for lev in range(nlevels+1): + # Create list of fields specific to level lev + grid_fields = [] + if lev == 0: + grid_fields = [field for field in ts.avail_fields if 'lvl' not in field] + else: + grid_fields = [field for field in ts.avail_fields if f'lvl{lev}' in field] + data_lev = {} + for field in grid_fields: + vector_components = ts.fields_metadata[field]['avail_components'] + if vector_components != []: + for coord in vector_components: + Q, info = ts.get_field(field=field, iteration=ts.iterations[-1], coord=coord) + # key stores strings composed of field name and vector components + # (e.g., field='B' or field='B_lvl1' + coord='y' results in key='By') + key = field.replace(f'_lvl{lev}', '') + coord + data_lev[key] = np.sum(np.abs(Q)) + else: # scalar field + Q, info = ts.get_field(field=field, iteration=ts.iterations[-1]) + data_lev[field] = np.sum(np.abs(Q)) + data[f'lev={lev}'] = data_lev + # Compute checksum for particle quantities + if do_particles: + species_list = [] + if ts.avail_record_components is not None: + species_list = ts.avail_record_components.keys() + for species in species_list: + data_species = {} + part_fields = [item for item in ts.avail_record_components[species] + if item != 'id' and item != 'charge' and item != 'mass'] + # Convert the field name to the name used in plotfiles + for field in part_fields: + Q = ts.get_particle(var_list=[field], species=species, iteration=ts.iterations[-1]) + if field in ['x', 'y', 'z']: + field_name = 'particle_position_' + field + elif field in ['ux', 'uy', 'uz']: + field_name = 'particle_momentum_' + field[-1] + m, = ts.get_particle(['mass'], species=species, iteration=ts.iterations[-1]) + Q *= m*c + elif field in ['w']: + field_name = 'particle_weight' + else: + field_name = 'particle_' + field + data_species[field_name] = np.sum(np.abs(Q)) + data[species] = data_species return data def evaluate(self, rtol=1.e-9, atol=1.e-40): """ - Compare plotfile checksum with benchmark. - Read checksum from input plotfile, read benchmark + Compare output file checksum with benchmark. + Read checksum from output file, read benchmark corresponding to test_name, and assert that they are equal. Almost all the body of this functions is for user-readable print statements. @@ -136,10 +200,10 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): # Dictionaries have same outer keys (levels, species)? if (self.data.keys() != ref_benchmark.data.keys()): - print("ERROR: Benchmark and plotfile checksum " + print("ERROR: Benchmark and output file checksum " "have different outer keys:") print("Benchmark: %s" % ref_benchmark.data.keys()) - print("Plotfile : %s" % self.data.keys()) + print("Test file: %s" % self.data.keys()) print("\n----------------\nNew file for " + self.test_name + ":") print(json.dumps(self.data, indent=2)) print("----------------") @@ -148,12 +212,12 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): # Dictionaries have same inner keys (field and particle quantities)? for key1 in ref_benchmark.data.keys(): if (self.data[key1].keys() != ref_benchmark.data[key1].keys()): - print("ERROR: Benchmark and plotfile checksum have " + print("ERROR: Benchmark and output file checksum have " "different inner keys:") print("Common outer keys: %s" % ref_benchmark.data.keys()) print("Benchmark inner keys in %s: %s" % (key1, ref_benchmark.data[key1].keys())) - print("Plotfile inner keys in %s: %s" + print("Test file inner keys in %s: %s" % (key1, self.data[key1].keys())) print("\n----------------\nNew file for " + self.test_name + ":") print(json.dumps(self.data, indent=2)) @@ -168,11 +232,11 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): ref_benchmark.data[key1][key2], rtol=rtol, atol=atol) if not passed: - print("ERROR: Benchmark and plotfile checksum have " + print("ERROR: Benchmark and output file checksum have " "different value for key [%s,%s]" % (key1, key2)) print("Benchmark: [%s,%s] %.15e" % (key1, key2, ref_benchmark.data[key1][key2])) - print("Plotfile : [%s,%s] %.15e" + print("Test file: [%s,%s] %.15e" % (key1, key2, self.data[key1][key2])) checksums_differ = True # Print absolute and relative error for each failing key diff --git a/Regression/Checksum/checksumAPI.py b/Regression/Checksum/checksumAPI.py index c45b6e233f4..cc13ceefa28 100755 --- a/Regression/Checksum/checksumAPI.py +++ b/Regression/Checksum/checksumAPI.py @@ -23,23 +23,23 @@ Example: add these lines to a WarpX CI Python analysis script to run the checksum test. > import checksumAPI - > checksumAPI.evaluate_checksum(test_name, plotfile) + > checksumAPI.evaluate_checksum(test_name, output_file, output_format) - As a script, to evaluate or to reset a benchmark: * Evaluate a benchmark. From a bash terminal: - $ ./checksumAPI.py --evaluate --plotfile \ - --test-name + $ ./checksumAPI.py --evaluate --output-file \ + --output-format 'plotfile' --test-name * Reset a benchmark. From a bash terminal: - $ ./checksumAPI.py --reset-benchmark --plotfile \ - --test-name + $ ./checksumAPI.py --reset-benchmark --output-file \ + --output-format 'openpmd' --test-name """ -def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, +def evaluate_checksum(test_name, output_file, output_format='plotfile', rtol=1.e-9, atol=1.e-40, do_fields=True, do_particles=True): """ - Compare plotfile checksum with benchmark. - Read checksum from input plotfile, read benchmark + Compare output file checksum with benchmark. + Read checksum from output file, read benchmark corresponding to test_name, and assert their equality. Parameters @@ -47,8 +47,11 @@ def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, test_name: string Name of test, as found between [] in .ini file. - plotfile : string - Plotfile from which the checksum is computed. + output_file : string + Output file from which the checksum is computed. + + output_format : string + Format of the output file (plotfile, openpmd). rtol: float, default=1.e-9 Relative tolerance for the comparison. @@ -62,24 +65,27 @@ def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, do_particles: bool, default=True Whether to compare particles in the checksum. """ - test_checksum = Checksum(test_name, plotfile, do_fields=do_fields, - do_particles=do_particles) + test_checksum = Checksum(test_name, output_file, output_format, + do_fields=do_fields, do_particles=do_particles) test_checksum.evaluate(rtol=rtol, atol=atol) -def reset_benchmark(test_name, plotfile, do_fields=True, do_particles=True): +def reset_benchmark(test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): """ Update the benchmark (overwrites reference json file). Overwrite value of benchmark corresponding to - test_name with checksum read from input plotfile. + test_name with checksum read from output file. Parameters ---------- test_name: string Name of test, as found between [] in .ini file. - plotfile: string - Plotfile from which the checksum is computed. + output_file: string + Output file from which the checksum is computed. + + output_format: string + Format of the output file (plotfile, openpmd). do_fields: bool, default=True Whether to write field checksums in the benchmark. @@ -87,34 +93,37 @@ def reset_benchmark(test_name, plotfile, do_fields=True, do_particles=True): do_particles: bool, default=True Whether to write particles checksums in the benchmark. """ - ref_checksum = Checksum(test_name, plotfile, do_fields=do_fields, - do_particles=do_particles) + ref_checksum = Checksum(test_name, output_file, output_format, + do_fields=do_fields, do_particles=do_particles) ref_benchmark = Benchmark(test_name, ref_checksum.data) ref_benchmark.reset() -def reset_all_benchmarks(path_to_all_plotfiles): +def reset_all_benchmarks(path_to_all_output_files, output_format='plotfile'): """ Update all benchmarks (overwrites reference json files) - found in path_to_all_plotfiles + found in path_to_all_output_files Parameters ---------- - path_to_all_plotfiles: string - Path to all plotfiles for which the benchmarks - are to be reset. The plotfiles should be named _plt, which is + path_to_all_output_files: string + Path to all output files for which the benchmarks + are to be reset. The output files should be named _plt, which is what regression_testing.regtests.py does, provided we're careful enough. + + output_format: string + Format of the output files (plotfile, openpmd). """ - # Get list of plotfiles in path_to_all_plotfiles - plotfile_list = glob.glob(path_to_all_plotfiles + '*_plt*[0-9]', - recursive=True) - plotfile_list.sort() + # Get list of output files in path_to_all_output_files + output_file_list = glob.glob(path_to_all_output_files + '*_plt*[0-9]', + recursive=True) + output_file_list.sort() - # Loop over plotfiles and reset the corresponding benchmark - for plotfile in plotfile_list: - test_name = os.path.split(plotfile)[1][:-9] - reset_benchmark(test_name, plotfile) + # Loop over output files and reset the corresponding benchmark + for output_file in output_file_list: + test_name = os.path.split(output_file)[1][:-9] + reset_benchmark(test_name, output_file, output_format) if __name__ == '__main__': @@ -131,11 +140,14 @@ def reset_all_benchmarks(path_to_all_plotfiles): required='--evaluate' in sys.argv or '--reset-benchmark' in sys.argv, help='Name of the test (as in WarpX-tests.ini)') - parser.add_argument('--plotfile', dest='plotfile', type=str, default='', + parser.add_argument('--output-file', dest='output_file', type=str, default='', required='--evaluate' in sys.argv or '--reset-benchmark' in sys.argv, - help='Name of WarpX plotfile') - + help='Name of WarpX output file') + parser.add_argument('--output-format', dest='output_format', type=str, default='plotfile', + required='--evaluate' in sys.argv or + '--reset-benchmark' in sys.argv, + help='Format of the output file (plotfile, openpmd)') parser.add_argument('--skip-fields', dest='do_fields', default=True, action='store_false', help='If used, do not read/write field checksums') @@ -143,7 +155,7 @@ def reset_all_benchmarks(path_to_all_plotfiles): default=True, action='store_false', help='If used, do not read/write particle checksums') - # Fields and/or particles are read from plotfile/written to benchmark? + # Fields and/or particles are read from output file/written to benchmark? parser.add_argument('--rtol', dest='rtol', type=float, default=1.e-9, help='relative tolerance for comparison') @@ -155,26 +167,27 @@ def reset_all_benchmarks(path_to_all_plotfiles): parser.add_argument('--reset-all-benchmarks', dest='reset_all_benchmarks', action='store_true', default=False, help='Reset all benchmarks.') - parser.add_argument('--path-to-all-plotfiles', - dest='path_to_all_plotfiles', type=str, default='', + parser.add_argument('--path-to-all-output-files', + dest='path_to_all_output_files', type=str, default='', required='--reset-all-benchmarks' in sys.argv, - help='Directory containing all benchmark plotfiles, \ + help='Directory containing all benchmark output files, \ typically WarpX-benchmarks generated by \ regression_testing/regtest.py') args = parser.parse_args() if args.reset_benchmark: - reset_benchmark(args.test_name, args.plotfile, + reset_benchmark(args.test_name, args.output_file, args.output_format, do_fields=args.do_fields, do_particles=args.do_particles) if args.evaluate: - evaluate_checksum(args.test_name, args.plotfile, rtol=args.rtol, - atol=args.atol, do_fields=args.do_fields, - do_particles=args.do_particles) + evaluate_checksum(args.test_name, args.output_file, args.output_format, + rtol=args.rtol, atol=args.atol, + do_fields=args.do_fields, do_particles=args.do_particles) if args.reset_all_benchmarks: - # WARNING: this mode does not support skip-fields/particles - # and tolerances - reset_all_benchmarks(args.path_to_all_plotfiles) + if args.output_format == 'openpmd': + sys.exit('Option --reset-all-benchmarks does not work with openPMD format') + # WARNING: this mode does not support skip-fields/particles and tolerances + reset_all_benchmarks(args.path_to_all_output_files, args.output_format) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index d4f2e68d0d3..2934c686c4d 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -1812,7 +1812,8 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = electrons -analysisRoutine = Examples/analysis_default_regression.py +outputFile = LaserAcceleration_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserAcceleration_1d] buildDir = . @@ -1973,7 +1974,8 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = electrons -analysisRoutine = Examples/analysis_default_regression.py +outputFile = LaserAcceleration_single_precision_comms_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserInjection] buildDir = . From 50e04bff649d68b593ac46a0252295100bb07b3e Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Fri, 15 Dec 2023 22:29:53 +0100 Subject: [PATCH 051/176] move function definition to cpp file (#4522) --- .../AcceleratorLattice/LatticeElementFinder.H | 47 +----------------- .../LatticeElementFinder.cpp | 48 ++++++++++++++++++- 2 files changed, 48 insertions(+), 47 deletions(-) diff --git a/Source/AcceleratorLattice/LatticeElementFinder.H b/Source/AcceleratorLattice/LatticeElementFinder.H index 24cdb691bb0..b7ca556d9ee 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.H +++ b/Source/AcceleratorLattice/LatticeElementFinder.H @@ -89,52 +89,7 @@ struct LatticeElementFinder */ void setup_lattice_indices (amrex::Gpu::DeviceVector const & zs, amrex::Gpu::DeviceVector const & ze, - amrex::Gpu::DeviceVector & indices) - { - - using namespace amrex::literals; - - const auto nelements = static_cast(zs.size()); - amrex::ParticleReal const * zs_arr = zs.data(); - amrex::ParticleReal const * ze_arr = ze.data(); - int * indices_arr = indices.data(); - - amrex::Real const zmin = m_zmin; - amrex::Real const dz = m_dz; - - amrex::ParticleReal const gamma_boost = m_gamma_boost; - amrex::ParticleReal const uz_boost = m_uz_boost; - amrex::Real const time = m_time; - - amrex::ParallelFor( m_nz, - [=] AMREX_GPU_DEVICE (int iz) { - - // Get the location of the grid node - amrex::Real z_node = zmin + iz*dz; - - if (gamma_boost > 1._prt) { - // Transform to lab frame - z_node = gamma_boost*z_node + uz_boost*time; - } - - // Find the index to the element that is closest to the grid cell. - // For now, this assumes that there is no overlap among elements of the same type. - for (int ie = 0 ; ie < nelements ; ie++) { - // Find the mid points between element ie and the ones before and after it. - // The first and last element need special handling. - const amrex::ParticleReal zcenter_left = (ie == 0)? - (std::numeric_limits::lowest()) : (0.5_prt*(ze_arr[ie-1] + zs_arr[ie])); - const amrex::ParticleReal zcenter_right = (ie < nelements - 1)? - (0.5_prt*(ze_arr[ie] + zs_arr[ie+1])) : (std::numeric_limits::max()); - if (zcenter_left <= z_node && z_node < zcenter_right) { - indices_arr[iz] = ie; - } - - } - } - ); - } - + amrex::Gpu::DeviceVector & indices); }; /** diff --git a/Source/AcceleratorLattice/LatticeElementFinder.cpp b/Source/AcceleratorLattice/LatticeElementFinder.cpp index ef90871a62b..bb2fe9c34a5 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.cpp +++ b/Source/AcceleratorLattice/LatticeElementFinder.cpp @@ -85,7 +85,6 @@ LatticeElementFinder::GetFinderDeviceInstance (WarpXParIter const& a_pti, int co return result; } - void LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& a_pti, int const a_offset, AcceleratorLattice const& accelerator_lattice, @@ -121,3 +120,50 @@ LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& } } + +void +LatticeElementFinder::setup_lattice_indices (amrex::Gpu::DeviceVector const & zs, + amrex::Gpu::DeviceVector const & ze, + amrex::Gpu::DeviceVector & indices) +{ + + using namespace amrex::literals; + + const auto nelements = static_cast(zs.size()); + amrex::ParticleReal const * zs_arr = zs.data(); + amrex::ParticleReal const * ze_arr = ze.data(); + int * indices_arr = indices.data(); + + amrex::Real const zmin = m_zmin; + amrex::Real const dz = m_dz; + + amrex::ParticleReal const gamma_boost = m_gamma_boost; + amrex::ParticleReal const uz_boost = m_uz_boost; + amrex::Real const time = m_time; + + amrex::ParallelFor( m_nz, + [=] AMREX_GPU_DEVICE (int iz) { + + // Get the location of the grid node + amrex::Real z_node = zmin + iz*dz; + + if (gamma_boost > 1._prt) { + // Transform to lab frame + z_node = gamma_boost*z_node + uz_boost*time; + } + + // Find the index to the element that is closest to the grid cell. + // For now, this assumes that there is no overlap among elements of the same type. + for (int ie = 0 ; ie < nelements ; ie++) { + // Find the mid points between element ie and the ones before and after it. + // The first and last element need special handling. + const amrex::ParticleReal zcenter_left = (ie == 0)? + (std::numeric_limits::lowest()) : (0.5_prt*(ze_arr[ie-1] + zs_arr[ie])); + const amrex::ParticleReal zcenter_right = (ie < nelements - 1)? + (0.5_prt*(ze_arr[ie] + zs_arr[ie+1])) : (std::numeric_limits::max()); + if (zcenter_left <= z_node && z_node < zcenter_right) { + indices_arr[iz] = ie; + } + } + }); +} From 56a5e63a0e207528c8d696efc5937d3f95d5cf6c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 15 Dec 2023 13:53:30 -0800 Subject: [PATCH 052/176] Improve documentation for laser chirp (#4525) * Improve documentation for laser chirp * Fix Latex * Apply suggestions from code review Co-authored-by: Axel Huebl --------- Co-authored-by: Axel Huebl --- Docs/source/usage/parameters.rst | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 18c1e53ada2..61f1d8fc155 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1243,7 +1243,7 @@ Laser initialization be parallel to ``warpx.boost_direction``, for now. * ``.e_max`` (`float` ; in V/m) - Peak amplitude of the laser field. + Peak amplitude of the laser field, in the focal plane. For a laser with a wavelength :math:`\lambda = 0.8\,\mu m`, the peak amplitude is related to :math:`a_0` by: @@ -1257,9 +1257,9 @@ Laser initialization perform the conversion to the boosted frame. * ``.a0`` (`float` ; dimensionless) - Peak normalized amplitude of the laser field (given in the lab frame, just as ``e_max`` above). + Peak normalized amplitude of the laser field, in the focal plane (given in the lab frame, just as ``e_max`` above). See the description of ``.e_max`` for the conversion between ``a0`` and ``e_max``. - Exactly one of ``a0`` and ``e_max`` must be specified. + Either ``a0`` or ``e_max`` must be specified. * ``.wavelength`` (`float`; in meters) The wavelength of the laser in vacuum. @@ -1337,12 +1337,15 @@ Laser initialization Note that :math:`\tau` relates to the full width at half maximum (FWHM) of *intensity*, which is closer to pulse length measurements in experiments, as :math:`\tau = \mathrm{FWHM}_I / \sqrt{2\ln(2)}` :math:`\approx \mathrm{FWHM}_I / 1.1774`. + For a chirped laser pulse (i.e. with a non-zero ``.phi2``), ``profile_duration`` is the Fourier-limited duration of the pulse, not the actual duration of the pulse. See the documentation for ``.phi2`` for more detail. + When running a **boosted-frame simulation**, provide the value of ``.profile_duration`` in the laboratory frame, and use ``warpx.gamma_boost`` to automatically perform the conversion to the boosted frame. * ``.profile_waist`` (`float` ; in meters) - The waist of the transverse Gaussian laser profile, defined as :math:`w_0` : + The waist of the transverse Gaussian :math:`w_0`, i.e. defined such that the electric field of the + laser pulse in the focal plane is of the form: .. math:: @@ -1375,8 +1378,23 @@ Laser initialization See definition in Akturk et al., Opt Express, vol 12, no 19 (2004). * ``.phi2`` (`float`; in seconds**2) optional (default `0.`) - Temporal chirp at focus. - See definition in Akturk et al., Opt Express, vol 12, no 19 (2004). + The amount of temporal chirp :math:`\phi^{(2)}`, at focus (in the lab frame). Namely, a wave packet + centered on the frequency :math:`(\omega_0 + \delta \omega)` will reach its peak intensity + at :math:`z(\delta \omega) = z_0 - c \phi^{(2)} \, \delta \omega`. Thus, a positive + :math:`\phi^{(2)}` corresponds to positive chirp, i.e. red part of the spectrum in the + front of the pulse and blue part of the spectrum in the back. More specifically, the electric + field in the focal plane is of the form: + + .. math:: + + E(\boldsymbol{x},t) \propto Re\left[ \exp\left( -\frac{(t-t_{peak})^2}{\tau^2 + 2i\phi^{(2)}} + i\omega_0 (t-t_{peak}) + i\phi_0 \right) \right] + + where :math:`\tau` is given by ``.laser_duration`` and represents the + Fourier-limited duration of the laser pulse. Thus, the actual duration of the chirped laser pulse is: + + .. math:: + + \tau' = \sqrt{ \tau^2 + 4 \phi^{(2)}/\tau^2 } * ``.do_continuous_injection`` (`0` or `1`) optional (default `0`). Whether or not to use continuous injection. From ade2df1e042c14323c92293f21c8bde7a31ad06d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 15 Dec 2023 13:56:54 -0800 Subject: [PATCH 053/176] Doc: Fix ChecksumAPI `autofunction` (#4524) Ensure the ChecksumAPI can be rendered in Sphinx via `autofunction`. --- Docs/requirements.txt | 3 +++ Docs/source/conf.py | 2 +- Docs/source/developers/checksum.rst | 4 ++-- Docs/spack.yaml | 4 +++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Docs/requirements.txt b/Docs/requirements.txt index b43cc1329fb..b16bf161144 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -9,6 +9,8 @@ breathe docutils>=0.17.1 +openpmd-viewer # for checksumAPI + # PICMI API docs # note: keep in sync with version in ../requirements.txt picmistandard==0.28.0 @@ -26,3 +28,4 @@ sphinx-design sphinx_rtd_theme>=1.1.1 sphinxcontrib-bibtex sphinxcontrib-napoleon +yt # for checksumAPI diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 93831b9054d..af9a63ecef7 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -34,7 +34,7 @@ from pybtex.style.formatting.unsrt import Style as UnsrtStyle import sphinx_rtd_theme -sys.path.insert(0, os.path.join( os.path.abspath(__file__), '../Python') ) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../Regression/Checksum')) # -- General configuration ------------------------------------------------ diff --git a/Docs/source/developers/checksum.rst b/Docs/source/developers/checksum.rst index 4023bc06d50..1099d68dccf 100644 --- a/Docs/source/developers/checksum.rst +++ b/Docs/source/developers/checksum.rst @@ -14,9 +14,9 @@ From a user point of view, you should only need to use ``checksumAPI.py``. It co Include a checksum regression test in an analysis Python script --------------------------------------------------------------- -This relies on function ``evaluate_checksum``: +This relies on the function ``evaluate_checksum``: -.. doxygenfunction:: evaluate_checksum +.. autofunction:: checksumAPI.evaluate_checksum For an example, see diff --git a/Docs/spack.yaml b/Docs/spack.yaml index db1add0018a..f17c36e5d97 100644 --- a/Docs/spack.yaml +++ b/Docs/spack.yaml @@ -19,11 +19,13 @@ spack: - doxygen - graphviz - python + - py-openpmd-viewer - py-breathe - - py-recommonmark - py-pybtex - py-pygments + - py-recommonmark - py-sphinx - py-sphinx-copybutton - py-sphinx-design - py-sphinx-rtd-theme + - py-yt From 5650b0dcbc37220204c5e19bc471c0177f5ec47f Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Sat, 16 Dec 2023 01:29:07 +0100 Subject: [PATCH 054/176] WarpXSumGuardCells: move function definitions in cpp file (#4521) * move function definition in cpp file * fix bugs * fix issue found with clang-tidy * Clean Commented Line Co-authored-by: Luca Fedeli --------- Co-authored-by: Axel Huebl --- Source/Parallelization/CMakeLists.txt | 1 + Source/Parallelization/Make.package | 1 + Source/Parallelization/WarpXSumGuardCells.H | 37 ++------------ Source/Parallelization/WarpXSumGuardCells.cpp | 51 +++++++++++++++++++ 4 files changed, 57 insertions(+), 33 deletions(-) create mode 100644 Source/Parallelization/WarpXSumGuardCells.cpp diff --git a/Source/Parallelization/CMakeLists.txt b/Source/Parallelization/CMakeLists.txt index 48f0d3a49bd..d38f8d6bbf8 100644 --- a/Source/Parallelization/CMakeLists.txt +++ b/Source/Parallelization/CMakeLists.txt @@ -5,5 +5,6 @@ foreach(D IN LISTS WarpX_DIMS) GuardCellManager.cpp WarpXComm.cpp WarpXRegrid.cpp + WarpXSumGuardCells.cpp ) endforeach() diff --git a/Source/Parallelization/Make.package b/Source/Parallelization/Make.package index 629cfafea62..7930118a1d2 100644 --- a/Source/Parallelization/Make.package +++ b/Source/Parallelization/Make.package @@ -1,5 +1,6 @@ CEXE_sources += WarpXComm.cpp CEXE_sources += WarpXRegrid.cpp CEXE_sources += GuardCellManager.cpp +CEXE_sources += WarpXSumGuardCells.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Parallelization diff --git a/Source/Parallelization/WarpXSumGuardCells.H b/Source/Parallelization/WarpXSumGuardCells.H index 13d6dce4af2..260ebb3871a 100644 --- a/Source/Parallelization/WarpXSumGuardCells.H +++ b/Source/Parallelization/WarpXSumGuardCells.H @@ -8,10 +8,6 @@ #ifndef WARPX_SUM_GUARD_CELLS_H_ #define WARPX_SUM_GUARD_CELLS_H_ -#include "Utils/WarpXAlgorithmSelection.H" - -#include - #include /** \brief Sum the values of `mf`, where the different boxes overlap @@ -26,21 +22,10 @@ * updates both the *valid* cells and *guard* cells. (This is because a * spectral solver requires the value of the sources over a large stencil.) */ -inline void +void WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, const amrex::IntVect& src_ngrow, - const int icomp=0, const int ncomp=1) -{ - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { - n_updated_guards = mf.nGrowVect(); - } else { // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - } - ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); -} + int icomp=0, int ncomp=1); /** \brief Sum the values of `src` where the different boxes overlap * (i.e. in the guard cells) and copy them into `dst` @@ -57,24 +42,10 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, * Note: `i_comp` is the component where the results will be stored in `dst`; * The component from which we copy in `src` is always 0. */ -inline void +void WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, const amrex::Periodicity& period, const amrex::IntVect& src_ngrow, - const int icomp=0, const int ncomp=1) -{ - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { - n_updated_guards = dst.nGrowVect(); - } else { // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - } - - dst.setVal(0., icomp, ncomp, n_updated_guards); -// ablastr::utils::communication::ParallelAdd(dst, src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); - dst.ParallelAdd(src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); -} + int icomp=0, int ncomp=1); #endif // WARPX_SUM_GUARD_CELLS_H_ diff --git a/Source/Parallelization/WarpXSumGuardCells.cpp b/Source/Parallelization/WarpXSumGuardCells.cpp new file mode 100644 index 00000000000..20ac73cab50 --- /dev/null +++ b/Source/Parallelization/WarpXSumGuardCells.cpp @@ -0,0 +1,51 @@ +/* Copyright 2019 Maxence Thevenet, Remi Lehe, Weiqun Zhang + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "WarpXSumGuardCells.H" + +#include "Utils/WarpXAlgorithmSelection.H" + +#include "WarpX.H" + +#include + +void +WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, + const amrex::IntVect& src_ngrow, + const int icomp, const int ncomp) +{ + amrex::IntVect n_updated_guards; + + // Update both valid cells and guard cells + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { + n_updated_guards = mf.nGrowVect(); + } else { // Update only the valid cells + n_updated_guards = amrex::IntVect::TheZeroVector(); + } + ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); +} + + +void +WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, + const amrex::Periodicity& period, + const amrex::IntVect& src_ngrow, + const int icomp, const int ncomp) +{ + amrex::IntVect n_updated_guards; + + // Update both valid cells and guard cells + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { + n_updated_guards = dst.nGrowVect(); + } else { // Update only the valid cells + n_updated_guards = amrex::IntVect::TheZeroVector(); + } + + dst.setVal(0., icomp, ncomp, n_updated_guards); + dst.ParallelAdd(src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); +} From 7ad3df9f463207a59202b4d2b3ac23a9a5025624 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Fri, 15 Dec 2023 16:35:57 -0800 Subject: [PATCH 055/176] Spruce up amr.rst (#4529) * Documentation: spruce up AMR section * Change prefix to "amr-" * Undo unnecessary changes * Adjust size to 95% for better aesthetics * Grammar * Normalize Quotes --------- Co-authored-by: Axel Huebl --- Docs/source/latex_theory/AMR/AMR.tex | 2 +- Docs/source/latex_theory/allbibs.bib | 1 - Docs/source/theory/amr.rst | 136 +++++---------------------- 3 files changed, 22 insertions(+), 117 deletions(-) diff --git a/Docs/source/latex_theory/AMR/AMR.tex b/Docs/source/latex_theory/AMR/AMR.tex index 0e9eace8390..3ca94d95436 100644 --- a/Docs/source/latex_theory/AMR/AMR.tex +++ b/Docs/source/latex_theory/AMR/AMR.tex @@ -22,7 +22,7 @@ \section{Mesh refinement} In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss' Law around the refined patch, leading to long range errors \cite{Vaylpb2002,Colellajcp2010}. As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. \subsection{Electrostatic} -A cornerstone of the Particle-In-Cell method is that assuming a particle lying in a hypothetical infinite grid, then if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called ``momentum conserving'' gathering scheme; b) on average within one cell if using the ``energy conserving'' gathering scheme \cite{Birdsalllangdon}. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) \cite{Vaylpb2002,Colellajcp2010}. +A cornerstone of the Particle-In-Cell method is that, given a particle lying in a hypothetical infinite grid, if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called ``momentum conserving'' gathering scheme; b) on average within one cell if using the ``energy conserving'' gathering scheme \cite{Birdsalllangdon}. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) \cite{Vaylpb2002,Colellajcp2010}. A sketch of the implementation of mesh refinement in WarpX is given in Figure~\ref{fig:ESAMR} (left). Given the solution of the electric potential at a refinement level $L_n$, it is interpolated onto the boundaries of the grid patch(es) at the next refined level $L_{n+1}$. The electric potential is then computed at level $L_{n+1}$ by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. diff --git a/Docs/source/latex_theory/allbibs.bib b/Docs/source/latex_theory/allbibs.bib index a3c355aae25..b3475c5a81b 100644 --- a/Docs/source/latex_theory/allbibs.bib +++ b/Docs/source/latex_theory/allbibs.bib @@ -2181,7 +2181,6 @@ @article{LehePRE2016 volume = {94}, year = {2016} } - @book{godfrey1985iprop, author = {Godfrey, B. B.}, publisher = {Defense Technical Information Center}, diff --git a/Docs/source/theory/amr.rst b/Docs/source/theory/amr.rst index 730593beea4..d83b1d7db9d 100644 --- a/Docs/source/theory/amr.rst +++ b/Docs/source/theory/amr.rst @@ -3,158 +3,64 @@ Mesh refinement =============== -.. raw:: latex - - \centering +.. _fig_ESAMR: .. figure:: ICNSP_2011_Vay_fig1.png :alt: Sketches of the implementation of mesh refinement in WarpX with the electrostatic (left) and electromagnetic (right) solvers. In both cases, the charge/current from particles are deposited at the finest levels first, then interpolated recursively to coarser levels. In the electrostatic case, the potential is calculated first at the coarsest level :math:`L_0`, the solution interpolated to the boundaries of the refined patch :math:`r` at the next level :math:`L_{1}` and the potential calculated at :math:`L_1`. The procedure is repeated iteratively up to the highest level. In the electromagnetic case, the fields are computed independently on each grid and patch without interpolation at boundaries. Patches are terminated by absorbing layers (PML) to prevent the reflection of electromagnetic waves. Additional coarse patch :math:`c` and fine grid :math:`a` are needed so that the full solution is obtained by substitution on :math:`a` as :math:`F_{n+1}(a)=F_{n+1}(r)+I[F_n( s )-F_{n+1}( c )]` where :math:`F` is the field, and :math:`I` is a coarse-to-fine interpolation operator. In both cases, the field solution at a given level :math:`L_n` is unaffected by the solution at higher levels :math:`L_{n+1}` and up, allowing for mitigation of some spurious effects (see text) by providing a transition zone via extension of the patches by a few cells beyond the desired refined area (red & orange rectangles) in which the field is interpolated onto particles from the coarser parent level only. - :name: fig:ESAMR - :width: 15cm + :width: 95% Sketches of the implementation of mesh refinement in WarpX with the electrostatic (left) and electromagnetic (right) solvers. In both cases, the charge/current from particles are deposited at the finest levels first, then interpolated recursively to coarser levels. In the electrostatic case, the potential is calculated first at the coarsest level :math:`L_0`, the solution interpolated to the boundaries of the refined patch :math:`r` at the next level :math:`L_{1}` and the potential calculated at :math:`L_1`. The procedure is repeated iteratively up to the highest level. In the electromagnetic case, the fields are computed independently on each grid and patch without interpolation at boundaries. Patches are terminated by absorbing layers (PML) to prevent the reflection of electromagnetic waves. Additional coarse patch :math:`c` and fine grid :math:`a` are needed so that the full solution is obtained by substitution on :math:`a` as :math:`F_{n+1}(a)=F_{n+1}(r)+I[F_n( s )-F_{n+1}( c )]` where :math:`F` is the field, and :math:`I` is a coarse-to-fine interpolation operator. In both cases, the field solution at a given level :math:`L_n` is unaffected by the solution at higher levels :math:`L_{n+1}` and up, allowing for mitigation of some spurious effects (see text) by providing a transition zone via extension of the patches by a few cells beyond the desired refined area (red & orange rectangles) in which the field is interpolated onto particles from the coarser parent level only. -The mesh refinement methods that have been implemented in WarpX were developed following the following principles: i) avoidance of spurious effects from mesh refinement, or minimization of such effects; ii) user controllability of the spurious effects’ relative magnitude; iii) simplicity of implementation. The two main generic issues that were identified are: a) spurious self-force on macroparticles close to the mesh refinement interface (J. Vay et al. 2002; Colella and Norgaard 2010); b) reflection (and possible amplification) of short wavelength electromagnetic waves at the mesh refinement interface (Vay 2001). The two effects are due to the loss of translation invariance introduced by the asymmetry of the grid on each side of the mesh refinement interface. +The mesh refinement methods that have been implemented in WarpX were developed following the following principles: i) avoidance of spurious effects from mesh refinement, or minimization of such effects; ii) user controllability of the spurious effects’ relative magnitude; iii) simplicity of implementation. The two main generic issues that were identified are: a) spurious self-force on macroparticles close to the mesh refinement interface :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`; b) reflection (and possible amplification) of short wavelength electromagnetic waves at the mesh refinement interface :cite:p:`amr-Vayjcp01`. The two effects are due to the loss of translation invariance introduced by the asymmetry of the grid on each side of the mesh refinement interface. -In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss’ Law around the refined patch, leading to long range errors (J. Vay et al. 2002; Colella and Norgaard 2010). As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. +In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss’ Law around the refined patch, leading to long range errors :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`. As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. Electrostatic ------------- -A cornerstone of the Particle-In-Cell method is that assuming a particle lying in a hypothetical infinite grid, then if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called “momentum conserving” gathering scheme; b) on average within one cell if using the “energy conserving” gathering scheme (Birdsall and Langdon 1991). A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) (J. Vay et al. 2002; Colella and Norgaard 2010). - -A sketch of the implementation of mesh refinement in WarpX is given in Figure \ `[fig:ESAMR] <#fig:ESAMR>`__ (left). Given the solution of the electric potential at a refinement level :math:`L_n`, it is interpolated onto the boundaries of the grid patch(es) at the next refined level :math:`L_{n+1}`. The electric potential is then computed at level :math:`L_{n+1}` by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. +A cornerstone of the Particle-In-Cell method is that given a particle lying in a hypothetical infinite grid, if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called “momentum conserving” gathering scheme; b) on average within one cell if using the “energy conserving” gathering scheme :cite:p:`amr-Birdsalllangdon`. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`. -.. raw:: latex +A sketch of the implementation of mesh refinement in WarpX is given in :numref:`fig_ESAMR`. Given the solution of the electric potential at a refinement level :math:`L_n`, it is interpolated onto the boundaries of the grid patch(es) at the next refined level :math:`L_{n+1}`. The electric potential is then computed at level :math:`L_{n+1}` by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. - \centering +.. _fig_ESselfforce: .. figure:: ICNSP_2011_Vay_fig2.png :alt: Position history of one charged particle attracted by its image induced by a nearby metallic (dirichlet) boundary. The particle is initialized at rest. Without refinement patch (reference case), the particle is accelerated by its image, is reflected specularly at the wall, then decelerates until it reaches its initial position at rest. If the particle is initialized inside a refinement patch, the particle is initially accelerated toward the wall but is spuriously reflected before it reaches the boundary of the patch whether using the method implemented in WarpX or the MC method. Providing a surrounding transition region 2 or 4 cells wide in which the potential is interpolated from the parent coarse solution reduces significantly the effect of the spurious self-force. - :name: fig:ESselfforce - :width: 15cm + :width: 95% Position history of one charged particle attracted by its image induced by a nearby metallic (dirichlet) boundary. The particle is initialized at rest. Without refinement patch (reference case), the particle is accelerated by its image, is reflected specularly at the wall, then decelerates until it reaches its initial position at rest. If the particle is initialized inside a refinement patch, the particle is initially accelerated toward the wall but is spuriously reflected before it reaches the boundary of the patch whether using the method implemented in WarpX or the MC method. Providing a surrounding transition region 2 or 4 cells wide in which the potential is interpolated from the parent coarse solution reduces significantly the effect of the spurious self-force. -The presence of the self-force is illustrated on a simple test case that was introduced in (J. Vay et al. 2002) and also used in (Colella and Norgaard 2010): a single macroparticle is initialized at rest within a single refinement patch four cells away from the patch refinement boundary. The patch at level :math:`L_1` has :math:`32\times32` cells and is centered relative to the lowest :math:`64\times64` grid at level :math:`L_0` (“main grid”), while the macroparticle is centered in one direction but not in the other. The boundaries of the main grid are perfectly conducting, so that the macroparticle is attracted to the closest wall by its image. Specular reflection is applied when the particle reaches the boundary so that the motion is cyclic. The test was performed with WarpX using either linear or quadratic interpolation when gathering the main grid solution onto the refined patch boundary. It was also performed using another method from P. McCorquodale et al (labeled “MC” in this paper) based on the algorithm given in (Mccorquodale et al. 2004), which employs a more elaborate procedure involving two-ways interpolations between the main grid and the refined patch. A reference case was also run using a single :math:`128\times128` grid with no refined patch, in which it is observed that the particle propagates toward the closest boundary at an accelerated pace, is reflected specularly at the boundary, then slows down until it reaches its initial position at zero velocity. The particle position histories are shown for the various cases in Fig. `[fig:ESselfforce] <#fig:ESselfforce>`__. In all the cases using the refinement patch, the particle was spuriously reflected near the patch boundary and was effectively trapped in the patch. We notice that linear interpolation performs better than quadratic, and that the simple method implemented in WarpX performs better than the other proposed method for this test (see discussion below). - -.. raw:: latex +The presence of the self-force is illustrated on a simple test case that was introduced in :cite:t:`amr-Vaylpb2002` and also used in :cite:t:`amr-Colellajcp2010`: a single macroparticle is initialized at rest within a single refinement patch four cells away from the patch refinement boundary. The patch at level :math:`L_1` has :math:`32\times32` cells and is centered relative to the lowest :math:`64\times64` grid at level :math:`L_0` ("main grid"), while the macroparticle is centered in one direction but not in the other. The boundaries of the main grid are perfectly conducting, so that the macroparticle is attracted to the closest wall by its image. Specular reflection is applied when the particle reaches the boundary so that the motion is cyclic. The test was performed with WarpX using either linear or quadratic interpolation when gathering the main grid solution onto the refined patch boundary. It was also performed using another method from P. McCorquodale et al (labeled "MC" in this paper) based on the algorithm given in :cite:t:`amr-Mccorquodalejcp2004`, which employs a more elaborate procedure involving two-ways interpolations between the main grid and the refined patch. A reference case was also run using a single :math:`128\times128` grid with no refined patch, in which it is observed that the particle propagates toward the closest boundary at an accelerated pace, is reflected specularly at the boundary, then slows down until it reaches its initial position at zero velocity. The particle position histories are shown for the various cases in :numref:`fig_ESselfforce`. In all the cases using the refinement patch, the particle was spuriously reflected near the patch boundary and was effectively trapped in the patch. We notice that linear interpolation performs better than quadratic, and that the simple method implemented in WarpX performs better than the other proposed method for this test (see discussion below). - \centering +.. _fig_ESselfforcemap: .. figure:: ICNSP_2011_Vay_fig3.png :alt: (left) Maps of the magnitude of the spurious self-force :math:`\epsilon` in arbitrary units within one quarter of the refined patch, defined as :math:`\epsilon=\sqrt{(E_x-E_x^{ref})^2+(E_y-E_y^{ref})^2}`, where :math:`E_x` and :math:`E_y` are the electric field components within the patch experienced by one particle at a given location and :math:`E_x^{ref}` and :math:`E_y^{ref}` are the electric field from a reference solution. The map is given for the WarpX and the MC mesh refinement algorithms and for linear and quadratic interpolation at the patch refinement boundary. (right) Lineouts of the maximum (taken over neighboring cells) of the spurious self-force. Close to the interface boundary (x=0), the spurious self-force decreases at a rate close to one order of magnitude per cell (red line), then at about one order of magnitude per six cells (green line). - :name: fig:ESselfforcemap - :width: 15cm + :width: 95% (left) Maps of the magnitude of the spurious self-force :math:`\epsilon` in arbitrary units within one quarter of the refined patch, defined as :math:`\epsilon=\sqrt{(E_x-E_x^{ref})^2+(E_y-E_y^{ref})^2}`, where :math:`E_x` and :math:`E_y` are the electric field components within the patch experienced by one particle at a given location and :math:`E_x^{ref}` and :math:`E_y^{ref}` are the electric field from a reference solution. The map is given for the WarpX and the MC mesh refinement algorithms and for linear and quadratic interpolation at the patch refinement boundary. (right) Lineouts of the maximum (taken over neighboring cells) of the spurious self-force. Close to the interface boundary (x=0), the spurious self-force decreases at a rate close to one order of magnitude per cell (red line), then at about one order of magnitude per six cells (green line). -The magnitude of the spurious self-force as a function of the macroparticle position was mapped and is shown in Fig. `[fig:ESselfforcemap] <#fig:ESselfforcemap>`__ for the WarpX and MC algorithms using linear or quadratic interpolations between grid levels. It is observed that the magnitude of the spurious self-force decreases rapidly with the distance between the particle and the refined patch boundary, at a rate approaching one order of magnitude per cell for the four cells closest to the boundary and about one order of magnitude per six cells beyond. The method implemented in WarpX offers a weaker spurious force on average and especially at the cells that are the closest to the coarse-fine interface where it is the largest and thus matters most. +The magnitude of the spurious self-force as a function of the macroparticle position was mapped and is shown in :numref:`fig_ESselfforcemap` for the WarpX and MC algorithms using linear or quadratic interpolations between grid levels. It is observed that the magnitude of the spurious self-force decreases rapidly with the distance between the particle and the refined patch boundary, at a rate approaching one order of magnitude per cell for the four cells closest to the boundary and about one order of magnitude per six cells beyond. The method implemented in WarpX offers a weaker spurious force on average and especially at the cells that are the closest to the coarse-fine interface where it is the largest and thus matters most. We notice that the magnitude of the spurious self-force depends strongly on the distance to the edge of the patch and to the nodes of the underlying coarse grid, but weakly on the order of deposition and size of the patch. -A method was devised and implemented in WarpX for reducing the magnitude of spurious self-forces near the coarse-fine boundaries as follows. Noting that the coarse grid solution is unaffected by the presence of the patch and is thus free of self-force, extra “transition” cells are added around the “effective” refined area. -Within the effective area, the particles gather the potential in the fine grid. In the extra transition cells surrounding the refinement patch, the force is gathered directly from the coarse grid (an option, which has not yet been implemented, would be to interpolate between the coarse and fine grid field solutions within the transition zone so as to provide continuity of the force experienced by the particles at the interface). The number of cells allocated in the transition zones is controllable by the user in WarpX, giving the opportunity to check whether the spurious self-force is affecting the calculation by repeating it using different thicknesses of the transition zones. The control of the spurious force using the transition zone is illustrated in Fig. \ `[fig:ESselfforce] <#fig:ESselfforce>`__, where the calculation with WarpX using linear interpolation at the patch interface was repeated using either two or four cells transition regions (measured in refined patch cell units). Using two extra cells allowed for the particle to be free of spurious trapping within the refined area and follow a trajectory that is close to the reference one, and using four extra cells improved further to the point where the resulting trajectory becomes indistinguishable from the reference one. -We note that an alternative method was devised for reducing the magnitude of self-force near the coarse-fine boundaries for the MC method, by using a special deposition procedure near the interface (Colella and Norgaard 2010). +A method was devised and implemented in WarpX for reducing the magnitude of spurious self-forces near the coarse-fine boundaries as follows. Noting that the coarse grid solution is unaffected by the presence of the patch and is thus free of self-force, extra "transition" cells are added around the "effective" refined area. +Within the effective area, the particles gather the potential in the fine grid. In the extra transition cells surrounding the refinement patch, the force is gathered directly from the coarse grid (an option, which has not yet been implemented, would be to interpolate between the coarse and fine grid field solutions within the transition zone so as to provide continuity of the force experienced by the particles at the interface). The number of cells allocated in the transition zones is controllable by the user in WarpX, giving the opportunity to check whether the spurious self-force is affecting the calculation by repeating it using different thicknesses of the transition zones. The control of the spurious force using the transition zone is illustrated in :numref:`fig_ESselfforce`, where the calculation with WarpX using linear interpolation at the patch interface was repeated using either two or four cells transition regions (measured in refined patch cell units). Using two extra cells allowed for the particle to be free of spurious trapping within the refined area and follow a trajectory that is close to the reference one, and using four extra cells improved further to the point where the resulting trajectory becomes indistinguishable from the reference one. +We note that an alternative method was devised for reducing the magnitude of self-force near the coarse-fine boundaries for the MC method, by using a special deposition procedure near the interface :cite:p:`amr-Colellajcp2010`. Electromagnetic --------------- -The method that is used for electrostatic mesh refinement is not directly applicable to electromagnetic calculations. As was shown in section 3.4 of (Vay 2001), refinement schemes relying solely on interpolation between coarse and fine patches lead to the reflection with amplification of the short wavelength modes that fall below the cutoff of the Nyquist frequency of the coarse grid. Unless these modes are damped heavily or prevented from occurring at their source, they may affect particle motion and their effect can escalate if trapped within a patch, via multiple successive reflections with amplification. +The method that is used for electrostatic mesh refinement is not directly applicable to electromagnetic calculations. As was shown in section 3.4 of :cite:t:`amr-Vayjcp01`, refinement schemes relying solely on interpolation between coarse and fine patches lead to the reflection with amplification of the short wavelength modes that fall below the cutoff of the Nyquist frequency of the coarse grid. Unless these modes are damped heavily or prevented from occurring at their source, they may affect particle motion and their effect can escalate if trapped within a patch, via multiple successive reflections with amplification. -To circumvent this issue, an additional coarse patch (with the same resolution as the parent grid) is added, as shown in Fig. \ `[fig:ESAMR] <#fig:ESAMR>`__-right and described in (Vay, Adam, and Heron 2004). Both the fine and the coarse grid patches are terminated by Perfectly Matched Layers, reducing wave reflection by orders of magnitude, controllable by the user (Berenger 1996; J.-L. Vay 2002). The source current resulting from the motion of charged macroparticles within the refined region is accumulated on the fine patch and is then interpolated onto the coarse patch and added onto the parent grid. The process is repeated recursively from the finest level down to the coarsest. The Maxwell equations are then solved for one time interval on the entire set of grids, by default for one time step using the time step of the finest grid. The field on the coarse and fine patches only contain the contributions from the particles that have evolved within the refined area but not from the current sources outside the area. The total contribution of the field from sources within and outside the refined area is obtained by adding the field from the refined grid :math:`F(r)`, and adding an interpolation :math:`I` of the difference between the relevant subset :math:`s` of the field in the parent grid :math:`F(s)` and the field of the coarse grid :math:`F( c )`, on an auxiliary grid :math:`a`, i.e. :math:`F(a)=F(r)+I[F(s)-F( c )]`. The field on the parent grid subset :math:`F(s)` contains contributions from sources from both within and outside of the refined area. Thus, in effect, there is substitution of the coarse field resulting from sources within the patch area by its fine resolution counterpart. The operation is carried out recursively starting at the coarsest level up to the finest. +To circumvent this issue, an additional coarse patch (with the same resolution as the parent grid) is added, as shown in :numref:`fig_ESAMR` and described in :cite:t:`amr-Vaycpc04`. Both the fine and the coarse grid patches are terminated by Perfectly Matched Layers, reducing wave reflection by orders of magnitude, controllable by the user :cite:p:`amr-Berengerjcp96,amr-Vayjcp02`. The source current resulting from the motion of charged macroparticles within the refined region is accumulated on the fine patch and is then interpolated onto the coarse patch and added onto the parent grid. The process is repeated recursively from the finest level down to the coarsest. The Maxwell equations are then solved for one time interval on the entire set of grids, by default for one time step using the time step of the finest grid. The field on the coarse and fine patches only contain the contributions from the particles that have evolved within the refined area but not from the current sources outside the area. The total contribution of the field from sources within and outside the refined area is obtained by adding the field from the refined grid :math:`F(r)`, and adding an interpolation :math:`I` of the difference between the relevant subset :math:`s` of the field in the parent grid :math:`F(s)` and the field of the coarse grid :math:`F( c )`, on an auxiliary grid :math:`a`, i.e. :math:`F(a)=F(r)+I[F(s)-F( c )]`. The field on the parent grid subset :math:`F(s)` contains contributions from sources from both within and outside of the refined area. Thus, in effect, there is substitution of the coarse field resulting from sources within the patch area by its fine resolution counterpart. The operation is carried out recursively starting at the coarsest level up to the finest. An option has been implemented in which various grid levels are pushed with different time steps, given as a fixed fraction of the individual grid Courant conditions (assuming same cell aspect ratio for all grids and refinement by integer factors). In this case, the fields from the coarse levels, which are advanced less often, are interpolated in time. The substitution method has two potential drawbacks due to the inexact cancellation between the coarse and fine patches of : (i) the remnants of ghost fixed charges created by the particles entering and leaving the patches (this effect is due to the use of the electromagnetic solver and is different from the spurious self-force that was described for the electrostatic case); (ii) if using a Maxwell solver with a low-order stencil, the electromagnetic waves traveling on each patch at slightly different velocity due to numerical dispersion. The first issue results in an effective spurious multipole field whose magnitude decreases very rapidly with the distance to the patch boundary, similarly to the spurious self-force in the electrostatic case. Hence, adding a few extra transition cells surrounding the patches mitigates this effect very effectively. -The tunability of WarpX’s electromagnetic finite-difference and pseudo-spectral solvers provides the means to optimize the numerical dispersion so as to minimize the second effect for a given application, which has been demonstrated on the laser-plasma interaction test case presented in (Vay, Adam, and Heron 2004). -Both effects and their mitigation are described in more detail in (Vay, Adam, and Heron 2004). +The tunability of WarpX’s electromagnetic finite-difference and pseudo-spectral solvers provides the means to optimize the numerical dispersion so as to minimize the second effect for a given application, which has been demonstrated on the laser-plasma interaction test case presented in :cite:t:`amr-Vaycpc04`. +Both effects and their mitigation are described in more detail in :cite:t:`amr-Vaycpc04`. Caustics are supported anywhere on the grid with an accuracy that is set by the local resolution, and will be adequately resolved if the grid resolution supports the necessary modes from their sources to the points of wavefront crossing. The mesh refinement method that is implemented in WarpX has the potential to provide higher efficiency than the standard use of fixed gridding, by offering a path toward adaptive gridding following wavefronts. -.. raw:: html - -
- -.. raw:: html - -
- -Berenger, Jp. 1996. “Three-Dimensional Perfectly Matched Layer for the Absorption of Electromagnetic Waves.” *Journal of Computational Physics* 127 (2): 363–79. - -.. raw:: html - -
- -.. raw:: html - -
- -Birdsall, C K, and A B Langdon. 1991. *Plasma Physics via Computer Simulation*. Adam-Hilger. - -.. raw:: html - -
- -.. raw:: html - -
- -Colella, Phillip, and Peter C Norgaard. 2010. “Controlling Self-Force Errors at Refinement Boundaries for Amr-Pic.” *Journal of Computational Physics* 229 (4): 947–57. https://doi.org/10.1016/J.Jcp.2009.07.004. - -.. raw:: html - -
- -.. raw:: html - -
- -Mccorquodale, P, P Colella, Dp Grote, and Jl Vay. 2004. “A Node-Centered Local Refinement Algorithm For Poisson’s Equation In Complex Geometries.” *Journal of Computational Physics* 201 (1): 34–60. https://doi.org/10.1016/J.Jcp.2004.04.022. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2001. “An Extended Fdtd Scheme for the Wave Equation: Application to Multiscale Electromagnetic Simulation.” *Journal of Computational Physics* 167 (1): 72–98. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2002. “Asymmetric Perfectly Matched Layer for the Absorption of Waves.” *Journal of Computational Physics* 183 (2): 367–99. https://doi.org/10.1006/Jcph.2002.7175. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., J.-C. Adam, and A Heron. 2004. “Asymmetric Pml for the Absorption of Waves. Application to Mesh Refinement in Electromagnetic Particle-in-Cell Plasma Simulations.” *Computer Physics Communications* 164 (1-3): 171–77. https://doi.org/10.1016/J.Cpc.2004.06.026. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jl, P Colella, P Mccorquodale, B Van Straalen, A Friedman, and Dp Grote. 2002. “Mesh Refinement for Particle-in-Cell Plasma Simulations: Applications to and Benefits for Heavy Ion Fusion.” *Laser and Particle Beams* 20 (4): 569–75. https://doi.org/10.1017/S0263034602204139. - -.. raw:: html - -
- -.. raw:: html - -
+.. bibliography:: + :keyprefix: amr- From fe1d018a6463df34ea694ac163c97bf4f841ea4d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 15 Dec 2023 17:06:01 -0800 Subject: [PATCH 056/176] Doc: Extend a Simulation Workflow (#4520) * Doc: Extend a Simulation Workflow Rework and move the workflow on how to extend simulations from Python. * Fix Typos, Rename File & Section Co-authored-by: David Grote * Section: Modify Solvers * Example: Calc E_x*E_x * Fix Typo Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * Docs: `MFIter` loop and WarpX MR level Props * Explicit Callback Includes * Fix: MultiFab `level=...` syntax * Also tested `pti` loop * Write: Do not Change Positions Let's avoid to break the sim with this example :D * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: David Grote Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/usage/python.rst | 120 +------- Docs/source/usage/workflows.rst | 3 +- Docs/source/usage/workflows/python_extend.rst | 279 ++++++++++++++++++ Python/pywarpx/callbacks.py | 33 ++- Source/FieldSolver/ElectrostaticSolver.H | 2 +- Source/Python/WarpX.cpp | 32 +- 6 files changed, 332 insertions(+), 137 deletions(-) create mode 100644 Docs/source/usage/workflows/python_extend.rst diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index c9d846b41e7..2cf7982c69c 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -13,7 +13,7 @@ In the input file, instances of classes are created defining the various aspects A variable of type :py:class:`pywarpx.picmi.Simulation` is the central object to which all other options are passed, defining the simulation time, field solver, registered species, etc. Once the simulation is fully configured, it can be used in one of two modes. -**Interactive** use is the most common and can be :ref:`extended with custom runtime functionality `: +**Interactive** use is the most common and can be :ref:`extended with custom runtime functionality `: .. tab-set:: @@ -25,6 +25,9 @@ Once the simulation is fully configured, it can be used in one of two modes. :py:meth:`~pywarpx.picmi.Simulation.write_input_file`: create an :ref:`inputs file for a WarpX executable ` +When run directly from Python, one can also extend WarpX with further custom user logic. +See the :ref:`detailed workflow page ` on how to extend WarpX from Python. + .. _usage-picmi-parameters: @@ -137,118 +140,3 @@ Laser profiles can be used to initialize laser pulses in the simulation. Laser injectors control where to initialize laser pulses on the simulation grid. .. autoclass:: pywarpx.picmi.LaserAntenna - - -.. _usage-picmi-extend: - -Extending a Simulation from Python ----------------------------------- - -When running WarpX directly from Python it is possible to interact with the simulation. - -For instance, with the :py:meth:`~pywarpx.picmi.Simulation.step` method of the simulation class, one could run ``sim.step(nsteps=1)`` in a loop: - -.. code-block:: python3 - - # Preparation: set up the simulation - # sim = picmi.Simulation(...) - # ... - - steps = 1000 - for _ in range(steps): - sim.step(nsteps=1) - - # do something custom with the sim object - -As a more flexible alternative, one can install `callback functions `__, which will execute a given Python function at a -specific location in the WarpX simulation loop. - -.. automodule:: pywarpx.callbacks - :members: installcallback, uninstallcallback, isinstalled - -Data Access -^^^^^^^^^^^ - -While the simulation is running, callbacks can access the WarpX simulation data *in situ*. - -An important object for data access is ``Simulation.extension.warpx``, which is available only during the simulation run. -This object is the Python equivalent to the C++ ``WarpX`` simulation class and provides access to field ``MultiFab`` and ``ParticleContainer`` data. - -.. py:function:: pywarpx.picmi.Simulation.extension.warpx.getistep() - -.. py:function:: pywarpx.picmi.Simulation.extension.warpx.gett_new() - -.. py:function:: pywarpx.picmi.Simulation.extension.warpx.evolve() - -.. autofunction:: pywarpx.picmi.Simulation.extension.finalize() - -These and other classes are provided through `pyAMReX `__. -After the simulation is initialized, pyAMReX can be accessed via - -.. code-block:: python - - from pywarpx import picmi, libwarpx - - # ... simulation definition ... - - # equivalent to - # import amrex.space3d as amr - # for a 3D simulation - amr = libwarpx.amr # picks the right 1d, 2d or 3d variant - -.. py:function:: amr.ParallelDescriptor.NProcs() - -.. py:function:: amr.ParallelDescriptor.MyProc() - -.. py:function:: amr.ParallelDescriptor.IOProcessor() - -.. py:function:: amr.ParallelDescriptor.IOProcessorNumber() - -Particles can be added to the simulation at specific positions and with specific attribute values: - -.. code-block:: python - - from pywarpx import particle_containers, picmi - - # ... - - electron_wrapper = particle_containers.ParticleContainerWrapper("electrons") - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.add_particles - -Properties of the particles already in the simulation can be obtained with various functions. - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_particle_count - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_particle_structs - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_particle_arrays - -The ``get_particle_structs()`` and ``get_particle_arrays()`` functions are called -by several utility functions of the form ``get_particle_{comp_name}`` where -``comp_name`` is one of ``x``, ``y``, ``z``, ``r``, ``theta``, ``id``, ``cpu``, -``weight``, ``ux``, ``uy`` or ``uz``. - -New components can be added via Python. - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.add_real_comp - -Various diagnostics are also accessible from Python. -This includes getting the deposited or total charge density from a given species as well as accessing the scraped particle buffer. -See the example in ``Examples/Tests/ParticleBoundaryScrape`` for a reference on how to interact with scraped particle data. - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_species_charge_sum - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.deposit_charge_density - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.get_particle_boundary_buffer_size - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.get_particle_boundary_buffer_structs - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.get_particle_boundary_buffer - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.clear_buffer - -The embedded boundary conditions can be modified when using the electrostatic solver. - -.. py:function:: pywarpx.picmi.Simulation.extension.warpx.set_potential_on_eb() diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index 7a93d081c6b..e68ec9391be 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -8,11 +8,12 @@ This section collects typical user workflows and best practices for WarpX. .. toctree:: :maxdepth: 2 + workflows/python_extend workflows/domain_decomposition + workflows/plot_distribution_mapping workflows/debugging workflows/libensemble workflows/plot_timestep_duration - workflows/plot_distribution_mapping workflows/psatd_stencil workflows/archiving workflows/ml_dataset_training diff --git a/Docs/source/usage/workflows/python_extend.rst b/Docs/source/usage/workflows/python_extend.rst new file mode 100644 index 00000000000..6c9286c02ce --- /dev/null +++ b/Docs/source/usage/workflows/python_extend.rst @@ -0,0 +1,279 @@ +.. _usage-python-extend: + +Extend a Simulation with Python +=============================== + +When running WarpX directly :ref:`from Python ` it is possible to interact with the simulation. + +For instance, with the :py:meth:`~pywarpx.picmi.Simulation.step` method of the simulation class, one could run ``sim.step(nsteps=1)`` in a loop: + +.. code-block:: python3 + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + steps = 1000 + for _ in range(steps): + sim.step(nsteps=1) + + # do something custom with the sim object + +As a more flexible alternative, one can install `callback functions `__, which will execute a given Python function at a +specific location in the WarpX simulation loop. + +.. automodule:: pywarpx.callbacks + :members: installcallback, uninstallcallback, isinstalled + + +pyAMReX +------- + +Many of the following classes are provided through `pyAMReX `__. +After the simulation is initialized, the pyAMReX module can be accessed via + +.. code-block:: python + + from pywarpx import picmi, libwarpx + + # ... simulation definition ... + + # equivalent to + # import amrex.space3d as amr + # for a 3D simulation + amr = libwarpx.amr # picks the right 1d, 2d or 3d variant + + +Full details for pyAMReX APIs are `documented here `__. +Important APIs include: + +* `amr.ParallelDescriptor `__: MPI-parallel rank information +* `amr.MultiFab `__: MPI-parallel field data +* `amr.ParticleContainer_* `__: MPI-parallel particle data for a particle species + + +Data Access +----------- + +While the simulation is running, callbacks can have read and write access the WarpX simulation data *in situ*. + +An important object in the ``pywarpx.picmi`` module for data access is ``Simulation.extension.warpx``, which is available only during the simulation run. +This object is the Python equivalent to the C++ ``WarpX`` simulation class. + +.. py:class:: WarpX + + .. py:method:: getistep(lev: int) + + Get the current step on mesh-refinement level ``lev``. + + .. py:method:: gett_new(lev: int) + + Get the current physical time on mesh-refinement level ``lev``. + + .. py:method:: getdt(lev: int) + + Get the current physical time step size on mesh-refinement level ``lev``. + + .. py:method:: multifab(multifab_name: str) + + Return MultiFabs by name, e.g., ``"Efield_aux[x][level=0]"``, ``"Efield_cp[x][level=0]"``, ... + + The physical fields in WarpX have the following naming: + + - ``_fp`` are the "fine" patches, the regular resolution of a current mesh-refinement level + - ``_aux`` are temporary (auxiliar) patches at the same resolution as ``_fp``. + They usually include contributions from other levels and can be interpolated for gather routines of particles. + - ``_cp`` are "coarse" patches, at the same resolution (but not necessary values) as the ``_fp`` of ``level - 1`` + (only for level 1 and higher). + + .. py:method:: multi_particle_container + + .. py:method:: get_particle_boundary_buffer + + .. py:method:: set_potential_on_eb(potential: str) + + The embedded boundary (EB) conditions can be modified when using the electrostatic solver. + This set the EB potential string and updates the function parser. + + .. py:method:: evolve(numsteps=-1) + + Evolve the simulation the specified number of steps. + + .. autofunction:: pywarpx.picmi.Simulation.extension.finalize + +.. py::def:: pywarpx.picmi.Simulation.extension.get_instance + + Return a reference to the WarpX object. + + +The :py:class:`WarpX` also provides read and write access to field ``MultiFab`` and ``ParticleContainer`` data, shown in the following examples. + +Fields +^^^^^^ + +This example accesses the :math:`E_x(x,y,z)` field at level 0 after every time step and calculate the largest value in it. + +.. code-block:: python3 + + from pywarpx import picmi + from pywarpx.callbacks import callfromafterstep + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + + @callfromafterstep + def set_E_x(): + warpx = sim.extension.warpx + + # data access + E_x_mf = warpx.multifab(f"Efield_fp[x][level=0]") + + # compute + # iterate over mesh-refinement levels + for lev in range(warpx.finest_level + 1): + # grow (aka guard/ghost/halo) regions + ngv = E_x_mf.n_grow_vect + + # get every local block of the field + for mfi in E_x_mf: + # global index space box, including guards + bx = mfi.tilebox().grow(ngv) + print(bx) # note: global index space of this block + + # numpy representation: non-copying view, including the + # guard/ghost region; .to_cupy() for GPU! + E_x_np = E_x_mf.array(mfi).to_numpy() + + # notes on indexing in E_x_np: + # - numpy uses locally zero-based indexing + # - layout is F_CONTIGUOUS by default, just like AMReX + + # notes: + # Only the next lines are the "HOT LOOP" of the computation. + # For efficiency, use numpy array operation for speed on CPUs. + # For GPUs use .to_cupy() above and compute with cupy or numba. + E_x_np[()] = 42.0 + + + sim.step(nsteps=100) + +For further details on how to `access GPU data `__ or compute on ``E_x``, please see the `pyAMReX documentation `__. + + +High-Level Field Wrapper +"""""""""""""""""""""""" + +.. note:: + + TODO + +.. note:: + + TODO: What are the benefits of using the high-level wrapper? + TODO: What are the limitations (e.g., in memory usage or compute scalability) of using the high-level wrapper? + + +Particles +^^^^^^^^^ + +.. code-block:: python3 + + from pywarpx import picmi + from pywarpx.callbacks import callfromafterstep + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + @callfromafterstep + def my_after_step_callback(): + warpx = sim.extension.warpx + + # data access + multi_pc = warpx.multi_particle_container() + pc = multi_pc.get_particle_container_from_name("electrons") + + # compute + # iterate over mesh-refinement levels + for lvl in range(pc.finest_level + 1): + # get every local chunk of particles + for pti in pc.iterator(pc, level=lvl): + # default layout: AoS with positions and cpuid + aos = pti.aos().to_numpy() + + # additional compile-time and runtime attributes in SoA format + soa = pti.soa().to_numpy() + + # notes: + # Only the next lines are the "HOT LOOP" of the computation. + # For efficiency, use numpy array operation for speed on CPUs. + # For GPUs use .to_cupy() above and compute with cupy or numba. + + # write to all particles in the chunk + # note: careful, if you change particle positions, you need to + # redistribute particles before continuing the simulation step + # aos[()]["x"] = 0.30 + # aos[()]["y"] = 0.35 + # aos[()]["z"] = 0.40 + + for soa_real in soa.real: + soa_real[()] = 42.0 + + for soa_int in soa.int: + soa_int[()] = 12 + + + sim.step(nsteps=100) + +For further details on how to `access GPU data `__ or compute on ``electrons``, please see the `pyAMReX documentation `__. + + +High-Level Particle Wrapper +""""""""""""""""""""""""""" + +.. note:: + + TODO: What are the benefits of using the high-level wrapper? + TODO: What are the limitations (e.g., in memory usage or compute scalability) of using the high-level wrapper? + +Particles can be added to the simulation at specific positions and with specific attribute values: + +.. code-block:: python + + from pywarpx import particle_containers, picmi + + # ... + + electron_wrapper = particle_containers.ParticleContainerWrapper("electrons") + + +.. autoclass:: pywarpx.particle_containers.ParticleContainerWrapper + :members: + +The ``get_particle_structs()`` and ``get_particle_arrays()`` functions are called +by several utility functions of the form ``get_particle_{comp_name}`` where +``comp_name`` is one of ``x``, ``y``, ``z``, ``r``, ``theta``, ``id``, ``cpu``, +``weight``, ``ux``, ``uy`` or ``uz``. + + +Diagnostics +----------- + +Various diagnostics are also accessible from Python. +This includes getting the deposited or total charge density from a given species as well as accessing the scraped particle buffer. +See the example in ``Examples/Tests/ParticleBoundaryScrape`` for a reference on how to interact with scraped particle data. + + +.. autoclass:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper + :members: + + +Modify Solvers +-------------- + +From Python, one can also replace numerical solvers in the PIC loop or add new physical processes into the time step loop. +Examples: + +* :ref:`Capacitive Discharge `: replaces the Poisson solver of an electrostatic simulation (default: MLMG) with a python function that uses `superLU `__ to directly solve the Poisson equation. diff --git a/Python/pywarpx/callbacks.py b/Python/pywarpx/callbacks.py index 1c191e607f4..8f3c1dc5e2c 100644 --- a/Python/pywarpx/callbacks.py +++ b/Python/pywarpx/callbacks.py @@ -7,7 +7,7 @@ # License: BSD-3-Clause-LBNL """Callback Locations -================== +------------------ These are the functions which allow installing user created functions so that they are called at various places along the time step. @@ -23,9 +23,6 @@ instance method as an argument. Note that if an instance method is used, an extra reference to the method's object is saved. -The install can be done using a decorator, which has the prefix ``callfrom``. See -example below. - Functions can be called at the following times: * ``beforeInitEsolve``: before the initial solve for the E fields (i.e. before the PIC loop starts) @@ -46,27 +43,37 @@ * ``particleinjection``: called when particle injection happens, after the position advance and before deposition is called, allowing a user defined particle distribution to be injected each time step -* ``appliedfields``: allows directly specifying any fields to be applied to the particles - during the advance -To use a decorator, the syntax is as follows. This will install the function -``myplots`` to be called after each step. +Example that calls the Python function ``myplots`` after each step: .. code-block:: python3 - @callfromafterstep + from pywarpx.callbacks import installcallback + def myplots(): - ppzx() + # do something here + + installcallback('afterstep', myplots) -This is equivalent to the following: + # run simulation + sim.step(nsteps=100) + +The install can also be done using a `Python decorator `__, which has the prefix ``callfrom``. +To use a decorator, the syntax is as follows. This will install the function ``myplots`` to be called after each step. +The above example is quivalent to the following: .. code-block:: python3 + from pywarpx.callbacks import callfromafterstep + + @callfromafterstep def myplots(): - ppzx() + # do something here - installcallback('afterstep', myplots) + # run simulation + sim.step(nsteps=100) """ + import copy import sys import time diff --git a/Source/FieldSolver/ElectrostaticSolver.H b/Source/FieldSolver/ElectrostaticSolver.H index 7bd982bb4e0..7fc6c9bf6da 100644 --- a/Source/FieldSolver/ElectrostaticSolver.H +++ b/Source/FieldSolver/ElectrostaticSolver.H @@ -52,7 +52,7 @@ namespace ElectrostaticSolver { void buildParsers (); void buildParsersEB (); - /* \brief Sets the EB potential string and updates the parsers + /* \brief Sets the EB potential string and updates the function parser * * \param [in] potential The string value of the potential */ diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 2ceb17be116..e42b8268fe1 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -86,7 +86,15 @@ void init_WarpX (py::module& m) "Evolve the simulation the specified number of steps" ) - // from AmrCore->AmrMesh + // from amrex::AmrCore / amrex::AmrMesh + .def_property_readonly("max_level", + [](WarpX const & wx){ return wx.maxLevel(); }, + "The maximum mesh-refinement level for the simulation." + ) + .def_property_readonly("finest_level", + [](WarpX const & wx){ return wx.finestLevel(); }, + "The currently finest level of mesh-refinement used. This is always less or equal to max_level." + ) .def("Geom", //[](WarpX const & wx, int const lev) { return wx.Geom(lev); }, py::overload_cast< int >(&WarpX::Geom, py::const_), @@ -112,7 +120,15 @@ void init_WarpX (py::module& m) }, py::arg("multifab_name"), py::return_value_policy::reference_internal, - "Return MultiFabs by name, e.g., 'Efield_aux[x][l=0]', 'Efield_cp[x][l=0]', ..." + R"doc(Return MultiFabs by name, e.g., ``\"Efield_aux[x][level=0]\"``, ``\"Efield_cp[x][level=0]\"``, ... + +The physical fields in WarpX have the following naming: + +- ``_fp`` are the "fine" patches, the regular resolution of a current mesh-refinement level +- ``_aux`` are temporary (auxiliar) patches at the same resolution as ``_fp``. + They usually include contributions from other levels and can be interpolated for gather routines of particles. +- ``_cp`` are "coarse" patches, at the same resolution (but not necessary values) as the ``_fp`` of ``level - 1`` + (only for level 1 and higher).)doc" ) .def("multi_particle_container", [](WarpX& wx){ return &wx.GetPartContainer(); }, @@ -140,22 +156,26 @@ void init_WarpX (py::module& m) // Expose functions to get the current simulation step and time .def("getistep", [](WarpX const & wx, int lev){ return wx.getistep(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current step on mesh-refinement level ``lev``." ) .def("gett_new", [](WarpX const & wx, int lev){ return wx.gett_new(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current physical time on mesh-refinement level ``lev``." ) .def("getdt", [](WarpX const & wx, int lev){ return wx.getdt(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current physical time step size on mesh-refinement level ``lev``." ) .def("set_potential_on_eb", [](WarpX& wx, std::string potential) { wx.m_poisson_boundary_handler.setPotentialEB(potential); }, - py::arg("potential") + py::arg("potential"), + "Sets the EB potential string and updates the function parser." ) ; From ac226bc3f4f65f226efac03f5914c47b779e1211 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 15 Dec 2023 17:06:49 -0800 Subject: [PATCH 057/176] Fix chirp documentation (#4533) --- Docs/source/usage/parameters.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 61f1d8fc155..ec360135f6f 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1389,12 +1389,12 @@ Laser initialization E(\boldsymbol{x},t) \propto Re\left[ \exp\left( -\frac{(t-t_{peak})^2}{\tau^2 + 2i\phi^{(2)}} + i\omega_0 (t-t_{peak}) + i\phi_0 \right) \right] - where :math:`\tau` is given by ``.laser_duration`` and represents the + where :math:`\tau` is given by ``.profile_duration`` and represents the Fourier-limited duration of the laser pulse. Thus, the actual duration of the chirped laser pulse is: .. math:: - \tau' = \sqrt{ \tau^2 + 4 \phi^{(2)}/\tau^2 } + \tau' = \sqrt{ \tau^2 + 4 (\phi^{(2)})^2/\tau^2 } * ``.do_continuous_injection`` (`0` or `1`) optional (default `0`). Whether or not to use continuous injection. From fc3b1c7ad431ca64a9a5f4872612de43a3260947 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Fri, 15 Dec 2023 17:08:27 -0800 Subject: [PATCH 058/176] Spruce up input_output.rst (#4532) --- .../input_output/input_output.tex | 2 +- Docs/source/theory/input_output.rst | 124 +++++------------- 2 files changed, 31 insertions(+), 95 deletions(-) diff --git a/Docs/source/latex_theory/input_output/input_output.tex b/Docs/source/latex_theory/input_output/input_output.tex index e013e236445..446f3f2b5d6 100644 --- a/Docs/source/latex_theory/input_output/input_output.tex +++ b/Docs/source/latex_theory/input_output/input_output.tex @@ -30,7 +30,7 @@ \subsection{Inputs and outputs in a boosted frame simulation} \subsubsection{Input in a boosted frame simulation} \paragraph{Particles - } -Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse ``electron cloud'' instability \cite{Vayprl07}. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity $\{x,y,z,v_x,v_y,v_z\}$ for each beam macroparticle at time $t=0$ for a beam moving at the average velocity $v_b=\beta_b c$ (where $c$ is the speed of light) in the laboratory, and using the standard synchronization ($z=z'=0$ at $t=t'=0$) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity $\beta c$ in the laboratory is as follows (the superscript $'$ relates to quantities known in the boosted frame while the superscript $^*$ relates to quantities that are know at a given longitudinal position $z^*$ but different times of arrival): +Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse ``electron cloud'' instability \cite{Vayprl07}. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity $\{x,y,z,v_x,v_y,v_z\}$ for each beam macroparticle at time $t=0$ for a beam moving at the average velocity $v_b=\beta_b c$ (where $c$ is the speed of light) in the laboratory, and using the standard synchronization ($z=z'=0$ at $t=t'=0$) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity $\beta c$ in the laboratory is as follows (the superscript $'$ relates to quantities known in the boosted frame while the superscript $^*$ relates to quantities that are know at a given longitudinal position $z^*$ but different times of arrival): \begin{enumerate} \item project positions at $z^*=0$ assuming ballistic propagation diff --git a/Docs/source/theory/input_output.rst b/Docs/source/theory/input_output.rst index 75427ad9c6d..21a5f5c8d2c 100644 --- a/Docs/source/theory/input_output.rst +++ b/Docs/source/theory/input_output.rst @@ -3,7 +3,7 @@ Inputs and Outputs ================== -Initialization of the plasma columns and drivers (laser or particle beam) is performed via the specification of multidimensional functions that describe the initial state with, if needed, a time dependence, or from reconstruction of distributions based on experimental data. Care is needed when initializing quantities in parallel to avoid double counting and ensure smoothness of the distributions at the interface of computational domains. When the sum of the initial distributions of charged particles is not charge neutral, initial fields are computed using generally a static approximation with Poisson solves accompanied by proper relativistic scalings (Vay 2008; Cowan et al. 2013). +Initialization of the plasma columns and drivers (laser or particle beam) is performed via the specification of multidimensional functions that describe the initial state with, if needed, a time dependence, or from reconstruction of distributions based on experimental data. Care is needed when initializing quantities in parallel to avoid double counting and ensure smoothness of the distributions at the interface of computational domains. When the sum of the initial distributions of charged particles is not charge neutral, initial fields are computed using generally a static approximation with Poisson solves accompanied by proper relativistic scalings :cite:p:`io-Vaypop2008, io-CowanPRSTAB13`. Outputs include dumps of particle and field quantities at regular intervals, histories of particle distributions moments, spectra, etc, and plots of the various quantities. In parallel simulations, the diagnostic subroutines need to handle additional complexity from the domain decomposition, as well as large amount of data that may necessitate data reduction in some form before saving to disk. @@ -12,16 +12,17 @@ Simulations in a Lorentz boosted frame require additional considerations, as des Inputs and outputs in a boosted frame simulation ------------------------------------------------ -.. _Fig_inputoutput: +.. _fig_inputoutput: + .. figure:: Input_output.png :alt: (top) Snapshot of a particle beam showing “frozen" (grey spheres) and “active" (colored spheres) macroparticles traversing the injection plane (red rectangle). (bottom) Snapshot of the beam macroparticles (colored spheres) passing through the background of electrons (dark brown streamlines) and the diagnostic stations (red rectangles). The electrons, the injection plane and the diagnostic stations are fixed in the laboratory plane, and are thus counter-propagating to the beam in a boosted frame. - :width: 120mm + :width: 100% (top) Snapshot of a particle beam showing “frozen" (grey spheres) and “active" (colored spheres) macroparticles traversing the injection plane (red rectangle). (bottom) Snapshot of the beam macroparticles (colored spheres) passing through the background of electrons (dark brown streamlines) and the diagnostic stations (red rectangles). The electrons, the injection plane and the diagnostic stations are fixed in the laboratory plane, and are thus counter-propagating to the beam in a boosted frame. The input and output data are often known from, or compared to, experimental data. Thus, calculating in a frame other than the laboratory entails transformations of the data between the calculation frame and the laboratory -frame. This section describes the procedures that have been implemented in the Particle-In-Cell framework Warp (Grote et al. 2005) to handle the input and output of data between the frame of calculation and the laboratory frame (Vay et al. 2011). Simultaneity of events between two frames is valid only for a plane that is perpendicular to the relative motion of the frame. As a result, the input/output processes involve the input of data (particles or fields) through a plane, as well as output through a series of planes, all of which are perpendicular to the direction of the relative velocity between the frame of calculation and the other frame of choice. +frame. This section describes the procedures that have been implemented in the Particle-In-Cell framework Warp :cite:p:`io-Warp` to handle the input and output of data between the frame of calculation and the laboratory frame :cite:p:`io-Vaypop2011`. Simultaneity of events between two frames is valid only for a plane that is perpendicular to the relative motion of the frame. As a result, the input/output processes involve the input of data (particles or fields) through a plane, as well as output through a series of planes, all of which are perpendicular to the direction of the relative velocity between the frame of calculation and the other frame of choice. Input in a boosted frame simulation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -29,17 +30,17 @@ Input in a boosted frame simulation Particles - ^^^^^^^^^^^^ -Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse “electron cloud” instability (Vay 2007). In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity :math:`\{x,y,z,v_x,v_y,v_z\}` for each beam macroparticle at time :math:`t=0` for a beam moving at the average velocity :math:`v_b=\beta_b c` (where :math:`c` is the speed of light) in the laboratory, and using the standard synchronization (:math:`z=z'=0` at :math:`t=t'=0`) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity :math:`\beta c` in the laboratory is as follows (the superscript :math:`'` relates to quantities known in the boosted frame while the superscript :math:`^*` relates to quantities that are know at a given longitudinal position :math:`z^*` but different times of arrival): +Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse “electron cloud” instability :cite:p:`io-Vayprl07`. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity :math:`\{x,y,z,v_x,v_y,v_z\}` for each beam macroparticle at time :math:`t=0` for a beam moving at the average velocity :math:`v_b=\beta_b c` (where :math:`c` is the speed of light) in the laboratory, and using the standard synchronization (:math:`z=z'=0` at :math:`t=t'=0`) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity :math:`\beta c` in the laboratory is as follows (the superscript :math:`'` relates to quantities known in the boosted frame while the superscript :math:`^*` relates to quantities that are know at a given longitudinal position :math:`z^*` but different times of arrival): #. project positions at :math:`z^*=0` assuming ballistic propagation .. math:: \begin{aligned} - t^* &=& \left(z-\bar{z}\right)/v_z \label{Eq:t*}\\ - x^* &=& x-v_x t^* \label{Eq:x*}\\ - y^* &=& y-v_y t^* \label{Eq:y*}\\ - z^* &=& 0 \label{Eq:z*}\end{aligned} + t^* &= \left(z-\bar{z}\right)/v_z \label{Eq:t*}\\ + x^* &= x-v_x t^* \label{Eq:x*}\\ + y^* &= y-v_y t^* \label{Eq:y*}\\ + z^* &= 0 \label{Eq:z*}\end{aligned} the velocity components being left unchanged, @@ -48,13 +49,13 @@ Particles are launched through a plane using a technique that is generic and app .. math:: \begin{aligned} - t'^* &=& -\gamma t^* \label{Eq:tp*}\\ - x'^* &=& x^* \label{Eq:xp*}\\ - y'^* &=& y^* \label{Eq:yp*}\\ - z'^* &=& \gamma\beta c t^* \label{Eq:zp*}\\ - v'^*_x&=&\frac{v_x^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vxp*}\\ - v'^*_y&=&\frac{v_y^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vyp*}\\ - v'^*_z&=&\frac{v_z^*-\beta c}{1-\beta \beta_b} \label{Eq:vzp*}\end{aligned} + t'^* &= -\gamma t^* \label{Eq:tp*}\\ + x'^* &= x^* \label{Eq:xp*}\\ + y'^* &= y^* \label{Eq:yp*}\\ + z'^* &= \gamma\beta c t^* \label{Eq:zp*}\\ + v'^*_x&=\frac{v_x^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vxp*}\\ + v'^*_y&=\frac{v_y^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vyp*}\\ + v'^*_z&=\frac{v_z^*-\beta c}{1-\beta \beta_b} \label{Eq:vzp*}\end{aligned} where :math:`\gamma=1/\sqrt{1-\beta^2}`. With the knowledge of the time at which each beam macroparticle crosses the plane into consideration, one can inject each beam macroparticle in the simulation at the appropriate location and time. @@ -63,11 +64,11 @@ Particles are launched through a plane using a technique that is generic and app .. math:: \begin{aligned} - z' &=& z'^*-\bar{v}'^*_z t'^* \label{Eq:zp}\end{aligned} + z' &= z'^*-\bar{v}'^*_z t'^* \label{Eq:zp}\end{aligned} - This additional step is needed for setting the electrostatic or electromagnetic fields at the plane of injection. In a Particle-In-Cell code, the three-dimensional fields are calculated by solving the Maxwell equations (or static approximation like Poisson, Darwin or other (Vay 2008)) on a grid on which the source term is obtained from the macroparticles distribution. This requires generation of a three-dimensional representation of the beam distribution of macroparticles at a given time before they cross the injection plane at :math:`z'^*`. This is accomplished by expanding the beam distribution longitudinally such that all macroparticles (so far known at different times of arrival at the injection plane) are synchronized to the same time in the boosted frame. To keep the beam shape constant, the particles are “frozen” until they cross that plane: the three velocity components and the two position components perpendicular to the boosted frame velocity are kept constant, while the remaining position component is advanced at the average beam velocity. As particles cross the plane of injection, they become regular “active” particles with full 6-D dynamics. + This additional step is needed for setting the electrostatic or electromagnetic fields at the plane of injection. In a Particle-In-Cell code, the three-dimensional fields are calculated by solving the Maxwell equations (or static approximation like Poisson, Darwin or other :cite:p:`io-Vaypop2008`) on a grid on which the source term is obtained from the macroparticles distribution. This requires generation of a three-dimensional representation of the beam distribution of macroparticles at a given time before they cross the injection plane at :math:`z'^*`. This is accomplished by expanding the beam distribution longitudinally such that all macroparticles (so far known at different times of arrival at the injection plane) are synchronized to the same time in the boosted frame. To keep the beam shape constant, the particles are “frozen” until they cross that plane: the three velocity components and the two position components perpendicular to the boosted frame velocity are kept constant, while the remaining position component is advanced at the average beam velocity. As particles cross the plane of injection, they become regular “active” particles with full 6-D dynamics. -Figure :numref:`Fig_inputoutput` (top) shows a snapshot of a beam that has passed partly through the injection plane. As the frozen beam macroparticles pass through the injection plane (which moves opposite to the beam in the boosted frame), they are converted to “active" macroparticles. The charge or current density is accumulated from the active and the frozen particles, thus ensuring that the fields at the plane of injection are consistent. +A snapshot of a beam that has passed partly through the injection plane in shown in :numref:`fig_inputoutput` (top). As the frozen beam macroparticles pass through the injection plane (which moves opposite to the beam in the boosted frame), they are converted to “active" macroparticles. The charge or current density is accumulated from the active and the frozen particles, thus ensuring that the fields at the plane of injection are consistent. Laser - ^^^^^^^^ @@ -89,93 +90,28 @@ If, for convenience, the injection plane is moving at constant velocity :math:`\ .. math:: \begin{aligned} - E_\perp\left(x,y,t\right)&=&\left(1-\beta_s\right)E_0 f\left(x,y,t\right)\nonumber \\ - &\times& \sin\left[\left(1-\beta_s\right)\omega t+\phi\left(x,y,\omega\right)\right].\end{aligned} + E_\perp\left(x,y,t\right)&=\left(1-\beta_s\right)E_0 f\left(x,y,t\right)\sin\left[\left(1-\beta_s\right)\omega t+\phi\left(x,y,\omega\right)\right] + \end{aligned} The injection of a laser of frequency :math:`\omega` is considered for a simulation using a boosted frame moving at :math:`\beta c` with respect to the laboratory. Assuming that the laser is injected at a plane that is fixed in the laboratory, and thus moving at :math:`\beta_s=-\beta` in the boosted frame, the injection in the boosted frame is given by .. math:: \begin{aligned} - E_\perp\left(x',y',t'\right)&=&\left(1-\beta_s\right)E'_0 f\left(x',y',t'\right)\nonumber \\ - &\times&\sin\left[\left(1-\beta_s\right)\omega' t'+\phi\left(x',y',\omega'\right)\right]\\ - &=&\left(E_0/\gamma\right) f\left(x',y',t'\right) \nonumber\\ - &\times&\sin\left[\omega t'/\gamma+\phi\left(x',y',\omega'\right)\right]\end{aligned} + E_\perp\left(x',y',t'\right)&=\left(1-\beta_s\right)E'_0 f\left(x',y',t'\right)\sin\left[\left(1-\beta_s\right)\omega' t'+\phi\left(x',y',\omega'\right)\right] + \\ + &=\left(E_0/\gamma\right) f\left(x',y',t'\right)\sin\left[\omega t'/\gamma+\phi\left(x',y',\omega'\right)\right] + \end{aligned} since :math:`E'_0/E_0=\omega'/\omega=1/\left(1+\beta\right)\gamma`. -The electric field is then converted into currents that get injected via a 2D array of macro-particles, with one positive and one dual negative macro-particle for each array cell in the plane of injection, whose weights and motion are governed by :math:`E_\perp\left(x',y',t'\right)`. Injecting using this dual array of macroparticles offers the advantage of automatically including the longitudinal component that arises from emitting into a boosted frame, and to automatically verify the discrete Gauss’ law thanks to using charge conserving (e.g. Esirkepov) current deposition scheme (Esirkepov 2001). +The electric field is then converted into currents that get injected via a 2D array of macro-particles, with one positive and one dual negative macro-particle for each array cell in the plane of injection, whose weights and motion are governed by :math:`E_\perp\left(x',y',t'\right)`. Injecting using this dual array of macroparticles offers the advantage of automatically including the longitudinal component that arises from emitting into a boosted frame, and to automatically verify the discrete Gauss’ law thanks to using charge conserving (e.g. Esirkepov) current deposition scheme :cite:p:`io-Esirkepovcpc01`. Output in a boosted frame simulation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Some quantities, e.g. charge or dimensions perpendicular to the boost velocity, are Lorentz invariant. -Those quantities are thus readily available from standard diagnostics in the boosted frame calculations. Quantities that do not fall in this category are recorded at a number of regularly spaced “stations", immobile in the laboratory frame, at a succession of time intervals to record data history, or averaged over time. A visual example is given on Fig. :numref:`Fig_inputoutput` (bottom). Since the space-time locations of the diagnostic grids in the laboratory frame generally do not coincide with the space-time positions of the macroparticles and grid nodes used for the calculation in a boosted frame, some interpolation is performed at runtime during the data collection process. As a complement or an alternative, selected particle or field quantities can be dumped at regular intervals and quantities are reconstructed in the laboratory frame during a post-processing phase. The choice of the methods depends on the requirements of the diagnostics and particular implementations. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, Benjamin M, David L Bruhwiler, John R Cary, Estelle Cormier-Michel, and Cameron G R Geddes. 2013. “Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations.” *Physical Review Special Topics-Accelerators and Beams* 16 (4). https://doi.org/10.1103/PhysRevSTAB.16.041303. - -.. raw:: html - -
- -.. raw:: html - -
- -Esirkepov, Tz. 2001. “Exact Charge Conservation Scheme for Particle-in-Cell Simulation with an Arbitrary Form-Factor.” *Computer Physics Communications* 135 (2): 144–53. - -.. raw:: html - -
- -.. raw:: html - -
- -Grote, D P, A Friedman, J.-L. Vay, and I Haber. 2005. “The Warp Code: Modeling High Intensity Ion Beams.” In *Aip Conference Proceedings*, 55–58. 749. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L. 2008. “Simulation of Beams or Plasmas Crossing at Relativistic Velocity.” *Physics of Plasmas* 15 (5): 56701. https://doi.org/10.1063/1.2837054. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2007. “Noninvariance of Space- and Time-Scale Ranges Under A Lorentz Transformation and the Implications for the Study of Relativistic Interactions.” *Physical Review Letters* 98 (13): 130405/1–4. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -L., C G R Geddes, E Esarey, C B Schroeder, W P Leemans, E Cormier-Michel, and D P Grote. 2011. “Modeling of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations.” *Physics of Plasmas* 18 (12). https://doi.org/10.1063/1.3663841. - -.. raw:: html - -
- -.. raw:: html +Those quantities are thus readily available from standard diagnostics in the boosted frame calculations. Quantities that do not fall in this category are recorded at a number of regularly spaced “stations", immobile in the laboratory frame, at a succession of time intervals to record data history, or averaged over time. A visual example is given on :numref:`fig_inputoutput` (bottom). Since the space-time locations of the diagnostic grids in the laboratory frame generally do not coincide with the space-time positions of the macroparticles and grid nodes used for the calculation in a boosted frame, some interpolation is performed at runtime during the data collection process. As a complement or an alternative, selected particle or field quantities can be dumped at regular intervals and quantities are reconstructed in the laboratory frame during a post-processing phase. The choice of the methods depends on the requirements of the diagnostics and particular implementations. -
+.. bibliography:: + :keyprefix: io- From 4badb9b51fb3f65295bcd78e56da3fab49d12349 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Fri, 15 Dec 2023 17:18:00 -0800 Subject: [PATCH 059/176] Spruce up cold_fluid_model.rst (#4531) --- Docs/source/refs.bib | 41 +++++++++++++++++++ Docs/source/theory/cold_fluid_model.rst | 52 +++++++------------------ 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 271b606377a..65545dfca87 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -245,3 +245,44 @@ @inproceedings{SandbergIPAC23 venue = {Venice, Italy}, year = {2023} } + +@article{HigueraPOP2017, +author = {Higuera, A. V. and Cary, J. R.}, +doi = {10.1063/1.4979989}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4979989/15988441/052104\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {04}, +number = {5}, +pages = {052104}, +title = {{Structure-preserving second-order integration of relativistic charged particle trajectories in electromagnetic fields}}, +url = {https://doi.org/10.1063/1.4979989}, +volume = {24}, +year = {2017} +} + +@article{ShuJCP1988, +author = {Chi-Wang Shu and Stanley Osher}, +doi = {https://doi.org/10.1016/0021-9991(88)90177-5}, +issn = {0021-9991}, +journal = {Journal of Computational Physics}, +number = {2}, +pages = {439-471}, +title = {Efficient implementation of essentially non-oscillatory shock-capturing schemes}, +url = {https://www.sciencedirect.com/science/article/pii/0021999188901775}, +volume = {77}, +year = {1988} +} + +@Inbook{VanLeerBookChapter1997, +author = {Van Leer, Bram}, +bookTitle = {Upwind and High-Resolution Schemes}, +doi = {10.1007/978-3-642-60543-7_3}, +editor = {Hussaini, M. Yousuff and {van Leer}, Bram and {Van Rosendale}, John}, +isbn = {978-3-642-60543-7}, +pages = {33--52}, +publisher = {Springer Berlin Heidelberg}, +title = {On The Relation Between The Upwind-Differencing Schemes Of Godunov, Engquist---Osher and Roe}, +url = {https://doi.org/10.1007/978-3-642-60543-7_3}, +year = {1997} +} diff --git a/Docs/source/theory/cold_fluid_model.rst b/Docs/source/theory/cold_fluid_model.rst index 7e69516270b..813a568c540 100644 --- a/Docs/source/theory/cold_fluid_model.rst +++ b/Docs/source/theory/cold_fluid_model.rst @@ -42,15 +42,19 @@ where the particle quantities are calculated by the PIC algorithm. Implementation details ---------------------- +.. _fig_fluid_loop: + +.. figure:: https://github.com/ECP-WarpX/WarpX/assets/69021085/dcbcc0e4-7899-43e4-b580-f57eb359b457 + :alt: Figure showing fluid Loop embedded within the overall PIC loop. + + Fluid Loop embedded within the overall PIC loop. + The fluid timeloop is embedded inside the standard PIC timeloop and consists of the following steps: 1. Higuera and Cary push of the momentum 2. Non-inertial (momentum source) terms (only in cylindrical geometry) 3. boundary conditions and MPI Communications 4. MUSCL -scheme for advection terms 5. Current and Charge Deposition. The figure here gives +scheme for advection terms 5. Current and Charge Deposition. :numref:`fig_fluid_loop` gives a visual representation of these steps, and we describe each of these in more detail. -.. figure:: https://github.com/ECP-WarpX/WarpX/assets/69021085/dcbcc0e4-7899-43e4-b580-f57eb359b457 - :alt: Fluid Loop embedded within the overall PIC loop. - Step 0: **Preparation** Before the fluid loop begins, it is assumed that the program is in the state where fields :math:`\mathbf{E}` and :math:`\mathbf{B}` are available integer timestep. The @@ -59,14 +63,14 @@ Step 0: **Preparation** on a nodal grid and at half-integer timestep. Step 1: **Higuera and Cary Push** - The time staggering of the fields is used by the momentum source term, which is solved with a the - Higeura and Cary push (Higuera et al, 2017). We do not adopt spatial + The time staggering of the fields is used by the momentum source term, which is solved with a + Higuera and Cary push :cite:p:`cfm-HigueraPOP2017`. We do not adopt spatial grid staggering, all discretized fluid quantities exist on the nodal grid. External fields can be included at this step. Step 2: **Non-inertial Terms** In RZ, the divergence of the flux terms has additional non-zero elements outside of the - derivatives. These terms are Strang split and are time integrated via equation 2.18 from (Osher et al, 1988), + derivatives. These terms are Strang split and are time integrated via equation 2.18 from :cite:t:`cfm-ShuJCP1988`, which is the SSP-RK3 integrator. Step 3: **Boundary Conditions and Communications** @@ -79,7 +83,7 @@ Step 4: **Advective Push** limiting is used. We further simplify the conservative equations in terms of primitive variables, :math:`\{ N, U_x, U_y, U_z \}`. Which we found to be more stable than conservative variables for the MUSCL reconstruction. Details of - the scheme can be found here (Van Leer et al, 1984). + the scheme can be found in :cite:t:`cfm-VanLeerBookChapter1997`. Step 5: **Current and Charge Deposition** Once this series of steps is complete and the fluids have been evolved by an entire @@ -104,33 +108,5 @@ Step 5: **Current and Charge Deposition** If using the fluid model with the Kinetic-Fluid Hybrid model or the electrostatic solver, there is a known issue that the fluids deposit at a half-timestep offset in the charge-density. -.. raw:: html - -
- -.. raw:: html - -
- -Higuera, Adam V., and John R. Cary. "Structure-preserving second-order integration of relativistic charged particle trajectories in electromagnetic fields." Physics of Plasmas 24.5 (2017). - -.. raw:: html - -
- -.. raw:: html - -
- -Osher, Stanley, and Chi-Wang Shu. "Efficient implementation of essentially non-oscillatory shock-capturing schemes." J. Comput. Phys 77.2 (1988): 439-471. - - -.. raw:: html - -
- -.. raw:: html - -
- -Van Leer, Bram. "On the relation between the upwind-differencing schemes of Godunov, Engquist–Osher and Roe." SIAM Journal on Scientific and statistical Computing 5.1 (1984): 1-20. +.. bibliography:: + :keyprefix: cfm- From c98cd6b5e8baad7e7b50c8a342b431c1b9811bcf Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Fri, 15 Dec 2023 17:20:27 -0800 Subject: [PATCH 060/176] Spruce up boosted_frame.rst (#4530) --- Docs/source/theory/boosted_frame.rst | 515 ++++++--------------------- 1 file changed, 110 insertions(+), 405 deletions(-) diff --git a/Docs/source/theory/boosted_frame.rst b/Docs/source/theory/boosted_frame.rst index 7dbcfb9ad68..ea1f662bd30 100644 --- a/Docs/source/theory/boosted_frame.rst +++ b/Docs/source/theory/boosted_frame.rst @@ -5,16 +5,18 @@ Moving window and optimal Lorentz boosted frame The simulations of plasma accelerators from first principles are extremely computationally intensive, due to the need to resolve the evolution of a driver (laser or particle beam) and an accelerated particle beam into a plasma structure that is orders of magnitude longer and wider than the accelerated beam. As is customary in the modeling of particle beam dynamics in standard particle accelerators, a moving window is commonly used to follow the driver, the wake and the accelerated beam. This results in huge savings, by avoiding the meshing of the entire plasma that is orders of magnitude longer than the other length scales of interest. +.. _fig_Boosted_frame: + .. figure:: Boosted_frame.png - :alt: [fig:PIC] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. + :alt: [fig:Boosted-frame] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. - [fig:PIC] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. + A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. -Even using a moving window, however, a full PIC simulation of a plasma accelerator can be extraordinarily demanding computationally, as many time steps are needed to resolve the crossing of the short driver beam with the plasma column. As it turns out, choosing an optimal frame of reference that travels close to the speed of light in the direction of the laser or particle beam (as opposed to the usual choice of the laboratory frame) enables speedups by orders of magnitude (Vay 2007; J -L. Vay et al. 2011). This is a result of the properties of Lorentz contraction and dilation of space and time. In the frame of the laboratory, a very short driver (laser or particle) beam propagates through a much longer plasma column, necessitating millions to tens of millions of time steps for parameters in the range of the BELLA or FACET-II experiments. As sketched in Fig. `[fig:PIC] <#fig:PIC>`__, in a frame moving with the driver beam in the plasma at velocity :math:`v=\beta c` (where :math:`c` is the speed of light in vacuum), the beam length is now elongated by :math:`\approx(1+\beta)\gamma` while the plasma contracts by :math:`\gamma` (where :math:`\gamma=1/\sqrt{1-\beta^2}` is the relativistic factor associated with the frame velocity). The number of time steps that is needed to simulate a “longer” beam through a “shorter” plasma is now reduced by up to :math:`\approx(1+\beta) \gamma^2` (a detailed derivation of the speedup is given below). +Even using a moving window, however, a full PIC simulation of a plasma accelerator can be extraordinarily demanding computationally, as many time steps are needed to resolve the crossing of the short driver beam with the plasma column. As it turns out, choosing an optimal frame of reference that travels close to the speed of light in the direction of the laser or particle beam (as opposed to the usual choice of the laboratory frame) enables speedups by orders of magnitude :cite:p:`bf-Vayprl07,bf-Vaypop2011`. This is a result of the properties of Lorentz contraction and dilation of space and time. In the frame of the laboratory, a very short driver (laser or particle) beam propagates through a much longer plasma column, necessitating millions to tens of millions of time steps for parameters in the range of the BELLA or FACET-II experiments. As sketched in :numref:`fig_Boosted_frame`, in a frame moving with the driver beam in the plasma at velocity :math:`v=\beta c` (where :math:`c` is the speed of light in vacuum), the beam length is now elongated by :math:`\approx(1+\beta)\gamma` while the plasma contracts by :math:`\gamma` (where :math:`\gamma=1/\sqrt{1-\beta^2}` is the relativistic factor associated with the frame velocity). The number of time steps that is needed to simulate a “longer” beam through a “shorter” plasma is now reduced by up to :math:`\approx(1+\beta) \gamma^2` (a detailed derivation of the speedup is given below). The modeling of a plasma acceleration stage in a boosted frame involves the fully electromagnetic modeling of a plasma propagating at near the speed of light, for which Numerical Cerenkov -(Boris and Lee 1973; Haber et al. 1973) is a potential issue, as explained in more details below. +:cite:p:`bf-Borisjcp73,bf-Habericnsp73` is a potential issue, as explained in more details below. In addition, for a frame of reference moving in the direction of the accelerated beam (or equivalently the wake of the laser), waves emitted by the plasma in the forward direction expand while the ones emitted in the backward direction contract, following the properties of the Lorentz transformation. @@ -25,16 +27,15 @@ Backscatter is weak in the short-pulse regime, and does not interact as strongly with the beam as do the forward propagating waves which stay in phase for a long period. It is thus often assumed that the backward propagating waves can be neglected in the modeling of plasma accelerator stages. The accuracy of this assumption has been demonstrated by -comparison between explicit codes which include both forward and backward waves and envelope or quasistatic codes which neglect backward waves -(Geddes et al. 2008; Geddes et al. 2009; Cowan et al. 2009). +comparison between explicit codes which include both forward and backward waves and envelope or quasistatic codes which neglect backward waves :cite:p:`bf-Geddesjp08,bf-Geddespac09,bf-Cowanaac08`. Theoretical speedup dependency with the frame boost --------------------------------------------------- -The derivation that is given here reproduces the one given in (J -L. Vay et al. 2011), where the obtainable speedup is derived as an extension of the formula that was derived earlier(Vay 2007), taking in addition into account the group velocity of the laser as it traverses the plasma. +The derivation that is given here reproduces the one given in :cite:t:`bf-Vaypop2011`, where the obtainable speedup is derived as an extension of the formula that was derived earlier :cite:p:`bf-Vayprl07`, taking in addition into account the group velocity of the laser as it traverses the plasma. Assuming that the simulation box is a fixed number of plasma periods long, which implies the use (which is standard) of a moving window following -the wake and accelerated beam, the speedup is given by the ratio of the time taken by the laser pulse and the plasma to cross each other, divided by the shortest time scale of interest, that is the laser period. To first order, the wake velocity :math:`v_w` is set by the 1D group velocity of the laser driver, which in the linear (low intensity) limit, is given by (Esarey, Schroeder, and Leemans 2009): +the wake and accelerated beam, the speedup is given by the ratio of the time taken by the laser pulse and the plasma to cross each other, divided by the shortest time scale of interest, that is the laser period. To first order, the wake velocity :math:`v_w` is set by the 1D group velocity of the laser driver, which in the linear (low intensity) limit, is given by :cite:p:`bf-Esareyrmp09`: .. math:: v_w/c=\beta_w=\left(1-\frac{\omega_p^2}{\omega^2}\right)^{1/2} @@ -58,76 +59,78 @@ In a frame moving at :math:`\beta c`, the quantities become .. math:: \begin{aligned} - \lambda_p^*&=&\lambda_p/\left[\gamma \left(1-\beta_w \beta\right)\right] \\ - L^*&=&L/\gamma \\ - \lambda^*&=& \gamma\left(1+\beta\right) \lambda\\ - \beta_w^*&=&\left(\beta_w-\beta\right)/\left(1-\beta_w\beta\right) \\ - v_p^*&=&-\beta c \\ - T^*&=&\frac{L^*+\eta \lambda_p^*}{v_w^*-v_p^*} \\ - R_t^*&=&\frac{T^* c}{\lambda^*} = \frac{\left(L^*+\eta \lambda_p^*\right)}{\left(\beta_w^*+\beta\right) \lambda^*}\end{aligned} + \lambda_p^* & = \lambda_p/\left[\gamma \left(1-\beta_w \beta\right)\right] + \\ + L^* & = L/\gamma + \\ + \lambda^* & = \gamma\left(1+\beta\right) \lambda + \\ + \beta_w^* & = \left(\beta_w-\beta\right)/\left(1-\beta_w\beta\right) + \\ + v_p^* & = -\beta c + \\ + T^* & = \frac{L^*+\eta \lambda_p^*}{v_w^*-v_p^*} + \\ + R_t^* & = \frac{T^* c}{\lambda^*} = \frac{\left(L^*+\eta \lambda_p^*\right)}{\left(\beta_w^*+\beta\right) \lambda^*} + \end{aligned} where :math:`\gamma=1/\sqrt{1-\beta^2}`. The expected speedup from performing the simulation in a boosted frame is given by the ratio of :math:`R_{lab}` and :math:`R_t^*` .. math:: - S=\frac{R_{lab}}{R_t^*}=\frac{\left(1+\beta\right)\left(L+\eta \lambda_p\right)}{\left(1-\beta\beta_w\right)L+\eta \lambda_p} - \label{Eq_scaling1d0} + :label: Eq_scaling1d0 -We note that assuming that :math:`\beta_w\approx1` (which is a valid approximation for most practical cases of interest) and that :math:`\gamma<<\gamma_w`, this expression is consistent with the expression derived earlier (Vay 2007) for the laser-plasma acceleration case, which states that :math:`R_t^*=\alpha R_t/\left(1+\beta\right)` with :math:`\alpha=\left(1-\beta+l/L\right)/\left(1+l/L\right)`, where :math:`l` is the laser length which is generally proportional to :math:`\eta \lambda_p`, and :math:`S=R_t/R_T^*`. However, higher values of :math:`\gamma` are of interest for maximum speedup, as shown below. +We note that assuming that :math:`\beta_w\approx1` (which is a valid approximation for most practical cases of interest) and that :math:`\gamma<<\gamma_w`, this expression is consistent with the expression derived earlier :cite:p:`bf-Vayprl07` for the laser-plasma acceleration case, which states that :math:`R_t^*=\alpha R_t/\left(1+\beta\right)` with :math:`\alpha=\left(1-\beta+l/L\right)/\left(1+l/L\right)`, where :math:`l` is the laser length which is generally proportional to :math:`\eta \lambda_p`, and :math:`S=R_t/R_T^*`. However, higher values of :math:`\gamma` are of interest for maximum speedup, as shown below. -For intense lasers (:math:`a\sim 1`) typically used for acceleration, the energy gain is limited by dephasing (Schroeder et al. 2011), which occurs over a scale length :math:`L_d \sim \lambda_p^3/2\lambda^2`. -Acceleration is compromised beyond :math:`L_d` and in practice, the plasma length is proportional to the dephasing length, i.e. :math:`L= \xi L_d`. In most cases, :math:`\gamma_w^2>>1`, which allows the approximations :math:`\beta_w\approx1-\lambda^2/2\lambda_p^2`, and :math:`L=\xi \lambda_p^3/2\lambda^2\approx \xi \gamma_w^2 \lambda_p/2>>\eta \lambda_p`, so that Eq.(\ `[Eq_scaling1d0] <#Eq_scaling1d0>`__) becomes +For intense lasers (:math:`a\sim 1`) typically used for acceleration, the energy gain is limited by dephasing :cite:p:`bf-Schroederprl2011`, which occurs over a scale length :math:`L_d \sim \lambda_p^3/2\lambda^2`. +Acceleration is compromised beyond :math:`L_d` and in practice, the plasma length is proportional to the dephasing length, i.e. :math:`L= \xi L_d`. In most cases, :math:`\gamma_w^2>>1`, which allows the approximations :math:`\beta_w\approx1-\lambda^2/2\lambda_p^2`, and :math:`L=\xi \lambda_p^3/2\lambda^2\approx \xi \gamma_w^2 \lambda_p/2>>\eta \lambda_p`, so that Eq.(:eq:`Eq_scaling1d0`) becomes .. math:: - S=\left(1+\beta\right)^2\gamma^2\frac{\xi\gamma_w^2}{\xi\gamma_w^2+\left(1+\beta\right)\gamma^2\left(\xi\beta/2+2\eta\right)} - \label{Eq_scaling1d} + :label: Eq_scaling1d -For low values of :math:`\gamma`, i.e. when :math:`\gamma<<\gamma_w`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) reduces to +For low values of :math:`\gamma`, i.e. when :math:`\gamma<<\gamma_w`, Eq.(:eq:`Eq_scaling1d`) reduces to .. math:: - S_{\gamma<<\gamma_w}=\left(1+\beta\right)^2\gamma^2 - \label{Eq_scaling1d_simpl2} + :label: Eq_scaling1d_simpl2 -Conversely, if :math:`\gamma\rightarrow\infty`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) becomes +Conversely, if :math:`\gamma\rightarrow\infty`, Eq.(`Eq_scaling1d`) becomes .. math:: - S_{\gamma\rightarrow\infty}=\frac{4}{1+4\eta/\xi}\gamma_w^2 - \label{Eq_scaling_gamma_inf} + :label: Eq_scaling_gamma_inf -Finally, in the frame of the wake, i.e. when :math:`\gamma=\gamma_w`, assuming that :math:`\beta_w\approx1`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) gives +Finally, in the frame of the wake, i.e. when :math:`\gamma=\gamma_w`, assuming that :math:`\beta_w\approx1`, Eq.(:eq:`Eq_scaling1d`) gives .. math:: - S_{\gamma=\gamma_w}\approx\frac{2}{1+2\eta/\xi}\gamma_w^2 - \label{Eq_scaling_gamma_wake} + :label: Eq_scaling_gamma_wake -Since :math:`\eta` and :math:`\xi` are of order unity, and the practical regimes of most interest satisfy :math:`\gamma_w^2>>1`, the speedup that is obtained by using the frame of the wake will be near the maximum obtainable value given by Eq.(\ `[Eq_scaling_gamma_inf] <#Eq_scaling_gamma_inf>`__). +Since :math:`\eta` and :math:`\xi` are of order unity, and the practical regimes of most interest satisfy :math:`\gamma_w^2>>1`, the speedup that is obtained by using the frame of the wake will be near the maximum obtainable value given by Eq.(:eq:`Eq_scaling_gamma_inf`). -Note that without the use of a moving window, the relativistic effects that are at play in the time domain would also be at play in the spatial domain (Vay 2007), and the :math:`\gamma^2` scaling would transform to :math:`\gamma^4`. Hence, it is important to use a moving window even in simulations in a Lorentz boosted frame. For very high values of the boosted frame, the optimal velocity of the moving window may vanish (i.e. no moving window) or even reverse. +Note that without the use of a moving window, the relativistic effects that are at play in the time domain would also be at play in the spatial domain :cite:p:`bf-Vayprl07`, and the :math:`\gamma^2` scaling would transform to :math:`\gamma^4`. Hence, it is important to use a moving window even in simulations in a Lorentz boosted frame. For very high values of the boosted frame, the optimal velocity of the moving window may vanish (i.e. no moving window) or even reverse. .. _theory-boostedframe-galilean: Numerical Stability and alternate formulation in a Galilean frame ----------------------------------------------------------------- -The numerical Cherenkov instability (NCI) (Godfrey 1974) +The numerical Cherenkov instability (NCI) :cite:p:`bf-Godfreyjcp74` is the most serious numerical instability affecting multidimensional PIC simulations of relativistic particle beams and streaming plasmas -(Martins et al. 2010; Vay et al. 2010; J L Vay et al. 2011; Sironi and Spitkovsky 2011; Godfrey and Vay 2013; Xu et al. 2013). +:cite:p:`bf-Martinscpc10,bf-VayAAC2010,bf-Vayjcp2011,bf-Spitkovsky:Icnsp2011,bf-GodfreyJCP2013,bf-XuJCP2013`. It arises from coupling between possibly numerically distorted electromagnetic modes and spurious beam modes, the latter due to the mismatch between the Lagrangian -treatment of particles and the Eulerian treatment of fields (Godfrey 1975). +treatment of particles and the Eulerian treatment of fields :cite:p:`bf-Godfreyjcp75`. In recent papers the electromagnetic dispersion -relations for the numerical Cherenkov instability were derived and solved for both FDTD (Godfrey and Vay 2013; Brendan B. Godfrey and Vay 2014) -and PSATD (Brendan B. Godfrey, Vay, and Haber 2014a, 2014b) algorithms. +relations for the numerical Cherenkov instability were derived and solved for both FDTD :cite:p:`bf-GodfreyJCP2013,bf-GodfreyJCP2014_FDTD` +and PSATD :cite:p:`bf-GodfreyJCP2014_PSATD,bf-GodfreyIEEE2014` algorithms. -Several solutions have been proposed to mitigate the NCI (Brendan B Godfrey, Vay, and Haber 2014; Brendan B. Godfrey, Vay, and Haber 2014b, 2014a; Godfrey and Vay 2015; Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015). Although +Several solutions have been proposed to mitigate the NCI :cite:p:`bf-GodfreyJCP2014,bf-GodfreyIEEE2014,bf-GodfreyJCP2014_PSATD,bf-GodfreyCPC2015,bf-YuCPC2015,bf-YuCPC2015-Circ`. Although these solutions efficiently reduce the numerical instability, they typically introduce either strong smoothing of the currents and fields, or arbitrary numerical corrections, which are @@ -137,47 +140,46 @@ it is sometimes unclear to what extent these added corrections could impact the physics at stake for a given resolution. For instance, NCI-specific corrections include periodically smoothing -the electromagnetic field components (Martins et al. 2010), -using a special time step (Vay et al. 2010; J L Vay et al. 2011) or -applying a wide-band smoothing of the current components (Vay et al. 2010; J L Vay et al. 2011; J. Vay et al. 2011). Another set of mitigation methods +the electromagnetic field components :cite:p:`bf-Martinscpc10`, +using a special time step :cite:p:`bf-VayAAC2010,bf-Vayjcp2011` or +applying a wide-band smoothing of the current components :cite:p:`bf-VayAAC2010,bf-Vayjcp2011,bf-VayPOPL2011`. Another set of mitigation methods involve scaling the deposited currents by a carefully-designed wavenumber-dependent factor -(Brendan B. Godfrey and Vay 2014; Brendan B. Godfrey, Vay, and Haber 2014b) or slightly modifying the +:cite:p:`bf-GodfreyJCP2014_FDTD,bf-GodfreyIEEE2014` or slightly modifying the ratio of electric and magnetic fields (:math:`E/B`) before gathering their value onto the macroparticles -(Brendan B. Godfrey, Vay, and Haber 2014a; Godfrey and Vay 2015). +:cite:p:`bf-GodfreyJCP2014_PSATD,bf-GodfreyCPC2015`. Yet another set of NCI-specific corrections -(Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015) consists +:cite:p:`bf-YuCPC2015,bf-YuCPC2015-Circ` consists in combining a small timestep :math:`\Delta t`, a sharp low-pass spatial filter, and a spectral or high-order scheme that is tuned so as to create a small, artificial “bump” in the dispersion relation -(Yu, Xu, Decyk, et al. 2015). While most mitigation methods have only been applied +:cite:p:`bf-YuCPC2015`. While most mitigation methods have only been applied to Cartesian geometry, this last -set of methods ((Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015)) +set of methods :cite:p:`bf-YuCPC2015,bf-YuCPC2015-Circ` has the remarkable property that it can be applied -(Yu, Xu, Tableman, et al. 2015) to both Cartesian geometry and +:cite:p:`bf-YuCPC2015-Circ` to both Cartesian geometry and quasi-cylindrical geometry (i.e. cylindrical geometry with -azimuthal Fourier decomposition (Lifschitz et al. 2009; Davidson et al. 2015; R. Lehe et al. 2016)). However, +azimuthal Fourier decomposition :cite:p:`bf-LifschitzJCP2009,bf-DavidsonJCP2015,bf-Lehe2016`). However, the use of a small timestep proportionally slows down the progress of the simulation, and the artificial “bump” is again an arbitrary correction that departs from the underlying physics. -A new scheme was recently proposed, in (Kirchen et al. 2016; Lehe et al. 2016), which +A new scheme was recently proposed, in :cite:t:`bf-KirchenPOP2016,bf-LehePRE2016`, which completely eliminates the NCI for a plasma drifting at a uniform relativistic velocity – with no arbitrary correction – by simply integrating the PIC equations in *Galilean coordinates* (also known as *comoving coordinates*). More precisely, in the new method, the Maxwell equations *in Galilean coordinates* are integrated analytically, using only natural hypotheses, within the PSATD -framework (Pseudo-Spectral-Analytical-Time-Domain (Haber et al. 1973; Vay, Haber, and Godfrey 2013)). +framework (Pseudo-Spectral-Analytical-Time-Domain :cite:p:`bf-Habericnsp73,bf-VayJCP2013`). The idea of the proposed scheme is to perform a Galilean change of coordinates, and to carry out the simulation in the new coordinates: .. math:: - - \label{eq:change-var} \boldsymbol{x}' = \boldsymbol{x} - \boldsymbol{v}_{gal}t + :label: change-var where :math:`\boldsymbol{x} = x\,\boldsymbol{u}_x + y\,\boldsymbol{u}_y + z\,\boldsymbol{u}_z` and :math:`\boldsymbol{x}' = x'\,\boldsymbol{u}_x + y'\,\boldsymbol{u}_y + z'\,\boldsymbol{u}_z` are the @@ -190,10 +192,10 @@ plasma, the plasma does not move with respect to the grid in the Galilean coordinates :math:`\boldsymbol{x}'` – or, equivalently, in the standard coordinates :math:`\boldsymbol{x}`, the grid moves along with the plasma. The heuristic intuition behind this scheme is that these coordinates should prevent the discrepancy between the Lagrangian and -Eulerian point of view, which gives rise to the NCI (Godfrey 1975). +Eulerian point of view, which gives rise to the NCI :cite:p:`bf-Godfreyjcp75`. An important remark is that the Galilean change of -coordinates (`[eq:change-var] <#eq:change-var>`__) is a simple translation. Thus, when used in +coordinates in Eq. (:eq:`change-var`) is a simple translation. Thus, when used in the context of Lorentz-boosted simulations, it does of course preserve the relativistic dilatation of space and time which gives rise to the characteristic computational speedup of the boosted-frame technique. @@ -206,378 +208,81 @@ translate the boundaries, in the Galilean scheme the gridpoints *themselves* are not only translated but in this case, the physical equations are modified accordingly. Most importantly, the assumed time evolution of the current :math:`\boldsymbol{J}` within one timestep is different in a standard PSATD scheme with moving -window and in a Galilean PSATD scheme (Lehe et al. 2016). +window and in a Galilean PSATD scheme :cite:p:`bf-LehePRE2016`. In the Galilean coordinates :math:`\boldsymbol{x}'`, the equations of particle motion and the Maxwell equations take the form .. math:: + \frac{d\boldsymbol{x}'}{dt} = \frac{\boldsymbol{p}}{\gamma m} - \boldsymbol{v}_{gal} + :label: motion1 - \begin{aligned} - \frac{d\boldsymbol{x}'}{dt} &= \frac{\boldsymbol{p}}{\gamma m} - \boldsymbol{v}_{gal}\label{eq:motion1} \\ - \frac{d\boldsymbol{p}}{dt} &= q \left( \boldsymbol{E} + - \frac{\boldsymbol{p}}{\gamma m} \times \boldsymbol{B} \right) \label{eq:motion2}\\ - \left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{B} &= -\boldsymbol{\nabla'}\times\boldsymbol{E} \label{eq:maxwell1}\\ - \frac{1}{c^2}\left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{E} &= \boldsymbol{\nabla'}\times\boldsymbol{B} - \mu_0\boldsymbol{J} \label{eq:maxwell2}\end{aligned} +.. math:: + \frac{d\boldsymbol{p}}{dt} = q \left( \boldsymbol{E} + \frac{\boldsymbol{p}}{\gamma m} \times \boldsymbol{B} \right) + :label: motion2 + +.. math:: + \left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{B} = -\boldsymbol{\nabla'}\times\boldsymbol{E} + :label: maxwell1 + +.. math:: + \frac{1}{c^2}\left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{E} = \boldsymbol{\nabla'}\times\boldsymbol{B} - \mu_0\boldsymbol{J} + :label: maxwell2 where :math:`\boldsymbol{\nabla'}` denotes a spatial derivative with respect to the Galilean coordinates :math:`\boldsymbol{x}'`. Integrating these equations from :math:`t=n\Delta t` to :math:`t=(n+1)\Delta t` results in the following update equations (see -(Lehe et al. 2016) for the details of the derivation): +:cite:t:`bf-LehePRE2016` for the details of the derivation): .. math:: - \begin{aligned} - \mathbf{\tilde{B}}^{n+1} &= \theta^2 C \mathbf{\tilde{B}}^n - -\frac{\theta^2 S}{ck}i\boldsymbol{k}\times \mathbf{\tilde{E}}^n \nonumber \\ - & + \;\frac{\theta \chi_1}{\epsilon_0c^2k^2}\;i\boldsymbol{k} \times - \mathbf{\tilde{J}}^{n+1/2} \label{eq:disc-maxwell1}\\ - \mathbf{\tilde{E}}^{n+1} &= \theta^2 C \mathbf{\tilde{E}}^n - +\frac{\theta^2 S}{k} \,c i\boldsymbol{k}\times \mathbf{\tilde{B}}^n \nonumber \\ - & +\frac{i\nu \theta \chi_1 - \theta^2S}{\epsilon_0 ck} \; \mathbf{\tilde{J}}^{n+1/2}\nonumber \\ - & - \frac{1}{\epsilon_0k^2}\left(\; \chi_2\;\hat{\mathcal{\rho}}^{n+1} - - \theta^2\chi_3\;\hat{\mathcal{\rho}}^{n} \;\right) i\boldsymbol{k} \label{eq:disc-maxwell2}\end{aligned} - -where we used the short-hand notations :math:`\mathbf{\tilde{E}}^n \equiv -% -\mathbf{\tilde{E}}(\boldsymbol{k}, n\Delta t)`, :math:`\mathbf{\tilde{B}}^n \equiv -\mathbf{\tilde{B}}(\boldsymbol{k}, n\Delta t)` as well as: + \mathbf{\tilde{B}}^{n+1} & = \theta^2 C \mathbf{\tilde{B}}^n -\frac{\theta^2 S}{ck}i\boldsymbol{k}\times \mathbf{\tilde{E}}^n \nonumber + \\ + & + \;\frac{\theta \chi_1}{\epsilon_0c^2k^2}\;i\boldsymbol{k} \times \mathbf{\tilde{J}}^{n+1/2} + \end{aligned} + :label: disc-maxwell1 .. math:: - \begin{aligned} - &C = \cos(ck\Delta t) \quad S = \sin(ck\Delta t) \quad k - = |\boldsymbol{k}| \label{eq:def-C-S}\\& - \nu = \frac{\boldsymbol{k}\cdot\boldsymbol{v}_{gal}}{ck} \quad \theta = - e^{i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2} \quad \theta^* = - e^{-i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2} \label{eq:def-nu-theta}\\& - \chi_1 = \frac{1}{1 -\nu^2} \left( \theta^* - C \theta + i - \nu \theta S \right) \label{eq:def-chi1}\\& - \chi_2 = \frac{\chi_1 - \theta(1-C)}{\theta^*-\theta} \quad - \chi_3 = \frac{\chi_1-\theta^*(1-C)}{\theta^*-\theta} \label{eq:def-chi23}\end{aligned} - -Note that, in the limit :math:`\boldsymbol{v}_{gal}=\boldsymbol{0}`, -(`[eq:disc-maxwell1] <#eq:disc-maxwell1>`__) and (`[eq:disc-maxwell2] <#eq:disc-maxwell2>`__) reduce to the standard PSATD -equations (Haber et al. 1973), as expected. -As shown in (Kirchen et al. 2016; Lehe et al. 2016), -the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. - -.. raw:: html - -
- -.. raw:: html - -
- -Boris, Jp, and R Lee. 1973. “Nonphysical Self Forces in Some Electromagnetic Plasma-Simulation Algorithms.” Note. *Journal of Computational Physics* 12 (1). 525 B St, Ste 1900, San Diego, Ca 92101-4495: Academic Press Inc Jnl-Comp Subscriptions: 131–36. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, B, D Bruhwiler, E Cormier-Michel, E Esarey, C G R Geddes, P Messmer, and K Paul. 2009. “Laser Wakefield Simulation Using A Speed-of-Light Frame Envelope Model.” In *Aip Conference Proceedings*, 1086:309–14. - -.. raw:: html - -
- -.. raw:: html - -
- -Davidson, A., A. Tableman, W. An, F.S. Tsung, W. Lu, J. Vieira, R.A. Fonseca, L.O. Silva, and W.B. Mori. 2015. “Implementation of a hybrid particle code with a PIC description in r–z and a gridless description in :math:`\Phi` into OSIRIS.” *Journal of Computational Physics* 281: 1063–77. https://doi.org/10.1016/j.jcp.2014.10.064. - -.. raw:: html - -
- -.. raw:: html - -
- -Esarey, E, C B Schroeder, and W P Leemans. 2009. “Physics of Laser-Driven Plasma-Based Electron Accelerators.” *Rev. Mod. Phys.* 81 (3): 1229–85. https://doi.org/10.1103/Revmodphys.81.1229. - -.. raw:: html - -
- -.. raw:: html - -
- -Geddes, C G R, D L Bruhwiler, J R Cary, W B Mori, J.-L. Vay, S F Martins, T Katsouleas, et al. 2008. “Computational Studies and Optimization of Wakefield Accelerators.” In *Journal of Physics: Conference Series*, 125:012002 (11 Pp.). - -.. raw:: html - -
- -.. raw:: html - -
- -Geddes et al., C G R. 2009. “Scaled Simulation Design of High Quality Laser Wakefield Accelerator Stages.” In *Proc. Particle Accelerator Conference*. Vancouver, Canada. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Bb. 1974. “Numerical Cherenkov Instabilities in Electromagnetic Particle Codes.” *Journal of Computational Physics* 15 (4): 504–21. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 1975. “Canonical Momenta and Numerical Instabilities in Particle Codes.” *Journal of Computational Physics* 19 (1): 58–76. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B, and Jean-Luc Vay. 2013. “Numerical stability of relativistic beam multidimensional {PIC} simulations employing the Esirkepov algorithm.” *Journal of Computational Physics* 248 (0): 33–46. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2013.04.006. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B., and Jean Luc Vay. 2014. “Suppressing the numerical Cherenkov instability in FDTD PIC codes.” *Journal of Computational Physics* 267: 1–6. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2015. “Improved numerical Cherenkov instability suppression in the generalized PSTD PIC algorithm.” *Computer Physics Communications* 196. Elsevier: 221–25. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B., Jean Luc Vay, and Irving Haber. 2014a. “Numerical stability analysis of the pseudo-spectral analytical time-domain PIC algorithm.” *Journal of Computational Physics* 258: 689–704. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2014b. “Numerical stability improvements for the pseudospectral EM PIC algorithm.” *IEEE Transactions on Plasma Science* 42 (5). Institute of Electrical; Electronics Engineers Inc.: 1339–44. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B, Jean-Luc Vay, and Irving Haber. 2014. “Numerical stability analysis of the pseudo-spectral analytical time-domain {PIC} algorithm.” *Journal of Computational Physics* 258 (0): 689–704. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2013.10.053. - -.. raw:: html + \mathbf{\tilde{E}}^{n+1} & = \theta^2 C \mathbf{\tilde{E}}^n +\frac{\theta^2 S}{k} \,c i\boldsymbol{k}\times \mathbf{\tilde{B}}^n \nonumber + \\ + & + \frac{i\nu \theta \chi_1 - \theta^2S}{\epsilon_0 ck} \; \mathbf{\tilde{J}}^{n+1/2}\nonumber + \\ + & - \frac{1}{\epsilon_0k^2}\left(\; \chi_2\;\hat{\mathcal{\rho}}^{n+1} - \theta^2\chi_3\;\hat{\mathcal{\rho}}^{n} \;\right) i\boldsymbol{k} + \end{aligned} + :label: disc-maxwell2 + +where we used the short-hand notations +:math:`\mathbf{\tilde{E}}^n \equiv \mathbf{\tilde{E}}(\boldsymbol{k}, n\Delta t)`, +:math:`\mathbf{\tilde{B}}^n \equiv \mathbf{\tilde{B}}(\boldsymbol{k}, n\Delta t)` as well as: -
- -.. raw:: html - -
- -Haber, I, R Lee, Hh Klein, and Jp Boris. 1973. “Advances in Electromagnetic Simulation Techniques.” In *Proc. Sixth Conf. Num. Sim. Plasmas*, 46–48. Berkeley, Ca. - -.. raw:: html - -
- -.. raw:: html - -
- -Kirchen, M., R. Lehe, B. B. Godfrey, I. Dornmair, S. Jalas, K. Peters, J.-L. Vay, and A. R. Maier. 2016. “Stable discrete representation of relativistically drifting plasmas.” *arXiv:1608.00215*. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, Rémi, Manuel Kirchen, Igor A. Andriyash, Brendan B. Godfrey, and Jean-Luc Vay. 2016. “A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm.” *Computer Physics Communications* 203: 66–82. https://doi.org/10.1016/j.cpc.2016.02.007. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, R., M. Kirchen, B. B. Godfrey, A. R. Maier, and J.-L. Vay. 2016. “Elimination of Numerical Cherenkov Instability in flowing-plasma Particle-In-Cell simulations by using Galilean coordinates.” *arXiv:1608.00227*. - -.. raw:: html - -
- -.. raw:: html - -
- -Lifschitz, A F, X Davoine, E Lefebvre, J Faure, C Rechatin, and V Malka. 2009. “Particle-in-Cell modelling of laser-plasma interaction using Fourier decomposition.” *Journal of Computational Physics* 228 (5): 1803–14. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2008.11.017. - -.. raw:: html - -
- -.. raw:: html - -
- -Martins, Samuel F, Ricardo A Fonseca, Luis O Silva, Wei Lu, and Warren B Mori. 2010. “Numerical Simulations of Laser Wakefield Accelerators in Optimal Lorentz Frames.” *Computer Physics Communications* 181 (5): 869–75. https://doi.org/10.1016/J.Cpc.2009.12.023. - -.. raw:: html - -
- -.. raw:: html - -
- -Schroeder, C B, C Benedetti, E Esarey, and W P Leemans. 2011. “Nonlinear Pulse Propagation and Phase Velocity of Laser-Driven Plasma Waves.” *Physical Review Letters* 106 (13): 135002. https://doi.org/10.1103/Physrevlett.106.135002. - -.. raw:: html - -
- -.. raw:: html - -
- -Sironi, L, and A Spitkovsky. 2011. “No Title.” - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean Luc, Irving Haber, and Brendan B. Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243: 260–68. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2007. “Noninvariance of Space- and Time-Scale Ranges Under A Lorentz Transformation and the Implications for the Study of Relativistic Interactions.” *Physical Review Letters* 98 (13): 130405/1–4. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -. L, C G R Geddes, C Benedetti, D L Bruhwiler, E Cormier-Michel, B M Cowan, J R Cary, and D P Grote. 2010. “Modeling Laser Wakefield Accelerators in A Lorentz Boosted Frame.” *Aip Conference Proceedings* 1299: 244–49. https://doi.org/10.1063/1.3520322. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Numerical Methods for Instability Mitigation in the Modeling of Laser Wakefield Accelerators in A Lorentz-Boosted Frame.” *Journal of Computational Physics* 230 (15): 5908–29. https://doi.org/10.1016/J.Jcp.2011.04.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jl, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Effects of Hyperbolic Rotation in Minkowski Space on the Modeling of Plasma Accelerators in A Lorentz Boosted Frame.” *Physics of Plasmas* 18 (3): 30701. https://doi.org/10.1063/1.3559483. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -L., C G R Geddes, E Esarey, C B Schroeder, W P Leemans, E Cormier-Michel, and D P Grote. 2011. “Modeling of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations.” *Physics of Plasmas* 18 (12). https://doi.org/10.1063/1.3663841. - -.. raw:: html - -
- -.. raw:: html - -
- -Xu, Xinlu, Peicheng Yu, Samual F Martins, Frank S Tsung, Viktor K Decyk, Jorge Vieira, Ricardo A Fonseca, Wei Lu, Luis O Silva, and Warren B Mori. 2013. “Numerical instability due to relativistic plasma drift in EM-PIC simulations.” *Computer Physics Communications* 184 (11): 2503–14. https://doi.org/http://dx.doi.org/10.1016/j.cpc.2013.07.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Yu, Peicheng, Xinlu Xu, Viktor K. Decyk, Frederico Fiuza, Jorge Vieira, Frank S. Tsung, Ricardo A. Fonseca, Wei Lu, Luis O. Silva, and Warren B. Mori. 2015. “Elimination of the numerical Cerenkov instability for spectral EM-PIC codes.” *Computer Physics Communications* 192 (July). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 32–47. https://doi.org/10.1016/j.cpc.2015.02.018. - -.. raw:: html - -
- -.. raw:: html +.. math:: + C = \cos(ck\Delta t), \quad S = \sin(ck\Delta t), \quad k = |\boldsymbol{k}|, + :label: def-C-S -
+.. math:: + \nu = \frac{\boldsymbol{k}\cdot\boldsymbol{v}_{gal}}{ck}, \quad \theta = e^{i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2}, + :label: def-nu-theta -Yu, Peicheng, Xinlu Xu, Adam Tableman, Viktor K. Decyk, Frank S. Tsung, Frederico Fiuza, Asher Davidson, et al. 2015. “Mitigation of numerical Cerenkov radiation and instability using a hybrid finite difference-FFT Maxwell solver and a local charge conserving current deposit.” *Computer Physics Communications* 197 (December). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 144–52. https://doi.org/10.1016/j.cpc.2015.08.026. +.. math:: + \chi_1 = \frac{1}{1 -\nu^2} \left( \theta^* - C \theta + i \nu \theta S \right), + :label: def-chi1 -.. raw:: html +.. math:: + \chi_2 = \frac{\chi_1 - \theta(1-C)}{\theta^*-\theta} + :label: def-chi2 -
+.. math:: + \chi_3 = \frac{\chi_1-\theta^*(1-C)}{\theta^*-\theta} + :label: def-chi3 -.. raw:: html +Note that, in the limit :math:`\boldsymbol{v}_{gal}=\boldsymbol{0}`, +Eqs. (:eq:`disc-maxwell1`) and (:eq:`disc-maxwell2`) reduce to the standard PSATD +equations :cite:p:`bf-Habericnsp73`, as expected. +As shown in :cite:t:`bf-KirchenPOP2016,bf-LehePRE2016`, +the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. -
+.. bibliography:: + :keyprefix: bf- From 7b87cdb9a88ab1ca7362c6d3b657e50935ee2e8c Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Fri, 15 Dec 2023 17:24:50 -0800 Subject: [PATCH 061/176] Documentation: numbered reference formatting (#4528) * Documentation: custom formatting * Documentation: custom formatting --- .gitignore | 2 +- Docs/source/_static/custom.css | 6 ++++++ Docs/source/conf.py | 8 ++++++++ Docs/source/dataanalysis/sensei.rst | 6 +++--- 4 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 Docs/source/_static/custom.css diff --git a/.gitignore b/.gitignore index d614c1ae2d9..4fa65351331 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,7 @@ Docs/warpx-doxygen-web.tag.xml Docs/openpmd-api-doxygen-web.tag.xml Docs/doxyhtml/ Docs/doxyxml/ -Docs/source/_static/ +Docs/source/_static/doxyhtml #################### # Package Managers # diff --git a/Docs/source/_static/custom.css b/Docs/source/_static/custom.css new file mode 100644 index 00000000000..b31c9188d76 --- /dev/null +++ b/Docs/source/_static/custom.css @@ -0,0 +1,6 @@ +.eqno { + /* As of 2023 Dec, sphinx_rtd_theme has a bug where equation numbers appear above the equations instead of on the right */ + /* The following is a make-shift fix, but comes at the cost of "making the header link invisible," though I'm not sure what that means */ + /* Copied from https://github.com/readthedocs/sphinx_rtd_theme/issues/301#issuecomment-1205755904 */ + float: right; +} diff --git a/Docs/source/conf.py b/Docs/source/conf.py index af9a63ecef7..f21dea15f18 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -134,6 +134,11 @@ def __init__(self, *args, **kwargs): html_theme = 'sphinx_rtd_theme' numfig = True +math_eqref_format = "{number}" +numfig_format = {'figure': 'Fig. %s', + 'table': 'Table %s', + 'code-block': 'Listing %s', + 'section': 'Section %s'} # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -146,6 +151,9 @@ def __init__(self, *args, **kwargs): # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +html_css_files = [ + 'custom.css', +] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/Docs/source/dataanalysis/sensei.rst b/Docs/source/dataanalysis/sensei.rst index 16826c3082c..5e4bfff10fd 100644 --- a/Docs/source/dataanalysis/sensei.rst +++ b/Docs/source/dataanalysis/sensei.rst @@ -30,7 +30,7 @@ and bridge code making it easy to use in AMReX based simulation codes. SENSEI provides a *configurable analysis adaptor* which uses an XML file to select and configure one or more back ends at run time. Run time selection of the back end via XML means one user can access Catalyst, another Libsim, yet -another Python with no changes to the code. This is depicted in figure +another Python with no changes to the code. This is depicted in :numref:`sensei_arch`. On the left side of the figure AMReX produces data, the bridge code pushes the data through the configurable analysis adaptor to the back end that was selected at run time. @@ -99,7 +99,7 @@ The back end is selected and configured at run time using the SENSEI XML file. The XML sets parameters specific to SENSEI and to the chosen back end. Many of the back ends have sophisticated configuration mechanisms which SENSEI makes use of. For example the following XML configuration was used on NERSC's Cori -with WarpX to render 10 iso surfaces, shown in figure :numref:`lpa_visit`, using +with WarpX to render 10 iso surfaces, shown in :numref:`lpa_visit`, using VisIt Libsim. .. code-block:: xml @@ -123,7 +123,7 @@ run of the desired simulation. quadrant has been clipped away to reveal innner structure. The same run and visualization was repeated using ParaView Catalyst, shown in -figure :numref:`lpa_pv`, by providing the following XML configuration. +:numref:`lpa_pv`, by providing the following XML configuration. .. code-block:: xml From e28fd2a083f5755917ff6af0871234eaf9e0b855 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Fri, 15 Dec 2023 17:25:41 -0800 Subject: [PATCH 062/176] Fix CI for ROCm 6.0 (#4527) Need to explicitly install hiprand package in CI because it's now a standalone project, not a submodule for rocRand according to the release notes. --- .github/workflows/dependencies/hip.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index 9d0206efb76..2225e670bb0 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -41,7 +41,8 @@ sudo apt-get install -y --no-install-recommends \ rocm-dev \ rocfft-dev \ rocprim-dev \ - rocrand-dev + rocrand-dev \ + hiprand-dev # ccache $(dirname "$0")/ccache.sh From c7c2b844db8a26cb12190592a67684c67c4b8b53 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Fri, 15 Dec 2023 17:48:18 -0800 Subject: [PATCH 063/176] Spruce up ml_dataset_training.rst (#4512) * Make refs work in ml_dataset_training.rst * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add figures locally and modify captions * Revert figures back to links * Adjust caption on beam 0 figure * Delete extra line breaks * Double-Ref --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- .../usage/workflows/ml_dataset_training.rst | 45 ++++++++++++------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/Docs/source/usage/workflows/ml_dataset_training.rst b/Docs/source/usage/workflows/ml_dataset_training.rst index ace0577c763..6e60a318bee 100644 --- a/Docs/source/usage/workflows/ml_dataset_training.rst +++ b/Docs/source/usage/workflows/ml_dataset_training.rst @@ -15,7 +15,7 @@ For example, a simulation determined by the following input script :language: python In this section we walk through a workflow for data processing and model training. -This workflow was developed and first presented in Refs. :cite:t:`SandbergIPAC23` and :cite:t:`SandbergPASC24`. +This workflow was developed and first presented in :cite:t:`ml-SandbergIPAC23,ml-SandbergPASC24`. This assumes you have an up-to-date environment with PyTorch and openPMD. @@ -25,21 +25,26 @@ Data Cleaning It is important to inspect the data for artifacts to check that input/output data make sense. If we plot the final phase space for beams 1-8, -the particle data is distributed in a single blob. +the particle data is distributed in a single blob, +as shown by :numref:`fig_phase_space_beam_1` for beam 1. +This is as we expect and what is optimal for training neural networks. + +.. _fig_phase_space_beam_1: .. figure:: https://user-images.githubusercontent.com/10621396/290010209-c55baf1c-dd98-4d56-a675-ad3729481eee.png - :alt: Plot comparing model prediction with simulation output. + :alt: Plot showing the final phase space projections for beam 1 of the training data, for a surrogate to stage 1. - Plot showing the final phase space projections of the training data for stage 1. + The final phase space projections for beam 1 of the training data, for a surrogate to stage 1. -This is as we expect and what is optimal for training neural networks. -On the other hand, the final phase space for beam 0 has a halo of outlying particles. +.. _fig_phase_space_beam_0: .. figure:: https://user-images.githubusercontent.com/10621396/290010282-40560ac4-8509-4599-82ca-167bb1739cff.png - :alt: Plot comparing model prediction with simulation output. + :alt: Plot showing the final phase space projections for beam 0 of the training data, for a surrogate to stage 0. - Plot showing the final phase space projections of the training data for stage 1. + The final phase space projections for beam 0 of the training data, for a surrogate to stage 0 +On the other hand, the final phase space for beam 0, shown in :numref:`fig_phase_space_beam_1`, +has a halo of outlying particles. Looking closer at the z-pz space, we see that some particles got caught in a decelerating region of the wake, have slipped back and are much slower than the rest of the beam. To assist our neural network in learning dynamics of interest, we filter out these particles. @@ -126,7 +131,7 @@ use in training and inference. Neural Network Structure ------------------------ -It was found in Ref. :cite:p:`SandbergPASC24` that reasonable surrogate models are obtained with +It was found in :cite:t:`ml-SandbergPASC24` that reasonable surrogate models are obtained with shallow feedforward neural networks consisting of fewer than 10 hidden layers and just under 1000 nodes per layer. The example shown here uses 3 hidden layers and 20 nodes per layer @@ -200,7 +205,7 @@ Save Neural Network Parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The model weights are saved after training to record the updates to the model parameters. -Addtionally, we save some model metainformation with the model for convenience, +Additionally, we save some model metainformation with the model for convenience, including the model hyperparameters, the training and testing losses, and how long the training took. .. literalinclude:: ml_materials/train.py @@ -212,7 +217,7 @@ Evaluate -------- In this section we show two ways to diagnose how well the neural network is learning the data. -First we consider the train-test loss curves, shown in Fig. `[fig:train_test_loss] <#fig:train-test>`__ . +First we consider the train-test loss curves, shown in :numref:`fig_train_test_loss`. This figure shows the model error on the training data (in blue) and testing data (in green) as a function of the number of epochs seen. The training data is used to update the model parameters, so training error should be lower than testing error. A key feature to look for in the train-test loss curve is the inflection point in the test loss trend. @@ -221,22 +226,25 @@ The testing error serves as a metric of model generalizability, indicating how w on data it hasn't seen yet. When the test-loss starts to trend flat or even upward, the neural network is no longer improving its ability to generalize to new data. +.. _fig_train_test_loss: + .. figure:: https://user-images.githubusercontent.com/10621396/290010428-f83725ab-a08f-494c-b075-314b0d26cb9a.png - :alt: Plot of training and testing loss curves + :alt: Plot of training and testing loss curves versus number of training epochs. - Plot of training (in blue) and testing (in green) loss curves versus number of training epochs. + Training (in blue) and testing (in green) loss curves versus number of training epochs. -A visual inspection of the model prediction can be seen in Fig. `[fig:train_evaluation]` . -This plot compares the model prediction, with dots colored by mean-square error, on the testing data with the actual simulation output in black. +.. _fig_train_evaluation: .. figure:: https://user-images.githubusercontent.com/10621396/290010486-4a3541e7-e0be-4cf1-b33b-57d5e5985196.png :alt: Plot comparing model prediction with simulation output. - Plot comparing model prediction (yellow-red dots, colored by mean-squared error) with simulation output (black dots). + A comparison of model prediction (yellow-red dots, colored by mean-squared error) with simulation output (black dots). +A visual inspection of the model prediction can be seen in :numref:`fig_train_evaluation`. +This plot compares the model prediction, with dots colored by mean-square error, on the testing data with the actual simulation output in black. The model obtained with the hyperparameters chosen here trains quickly but is not very accurate. A more accurate model is obtained with 5 hidden layers and 800 nodes per layer, -as discussed in :cite:t:`SandbergPASC24`. +as discussed in :cite:t:`ml-SandbergPASC24`. These figures can be generated with the following Python script. @@ -254,3 +262,6 @@ Surrogate Usage in Accelerator Physics A neural network such as the one we trained here can be incorporated in other BLAST codes. `Consider the example using neural networks in ImpactX `__. + +.. bibliography:: + :keyprefix: ml- From 8eb30ee6e61674c3e05b10f6e4fac2ecd5ac8992 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Fri, 15 Dec 2023 19:24:34 -0800 Subject: [PATCH 064/176] Spruce up PIC theory section (#4536) * Spruce up picsar_theory.rst * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Minor edit * Rename picsar_theory.rst to pic.rst --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/index.rst | 2 +- Docs/source/latex_theory/Makefile | 6 +- Docs/source/theory/pic.rst | 695 +++++++++++++++++++++ Docs/source/theory/picsar_theory.rst | 867 --------------------------- Docs/source/usage/pwfa.rst | 4 +- 5 files changed, 701 insertions(+), 873 deletions(-) create mode 100644 Docs/source/theory/pic.rst delete mode 100644 Docs/source/theory/picsar_theory.rst diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 75287734ff6..e5841da1f2d 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -107,7 +107,7 @@ Theory :hidden: theory/intro - theory/picsar_theory + theory/pic theory/amr theory/PML theory/boosted_frame diff --git a/Docs/source/latex_theory/Makefile b/Docs/source/latex_theory/Makefile index 916bb2399d7..9b2f598f51d 100644 --- a/Docs/source/latex_theory/Makefile +++ b/Docs/source/latex_theory/Makefile @@ -7,14 +7,14 @@ all: $(SRC_FILES) clean pandoc Boosted_frame/Boosted_frame.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o boosted_frame.rst pandoc input_output/input_output.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o input_output.rst mv *.rst ../theory - cd ../../../../picsar/Doxygen/pages/latex_theory/; pandoc theory.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o picsar_theory.rst - mv ../../../../picsar/Doxygen/pages/latex_theory/picsar_theory.rst ../theory + cd ../../../../picsar/Doxygen/pages/latex_theory/; pandoc theory.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o pic.rst + mv ../../../../picsar/Doxygen/pages/latex_theory/pic.rst ../theory cp ../../../../picsar/Doxygen/images/PIC.png ../theory cp ../../../../picsar/Doxygen/images/Yee_grid.png ../theory clean: rm -f ../theory/intro.rst rm -f ../theory/warpx_theory.rst - rm -f ../theory/picsar_theory.rst + rm -f ../theory/pic.rst rm -f ../theory/PIC.png rm -f ../theory/Yee_grid.png diff --git a/Docs/source/theory/pic.rst b/Docs/source/theory/pic.rst new file mode 100644 index 00000000000..820cdba50e6 --- /dev/null +++ b/Docs/source/theory/pic.rst @@ -0,0 +1,695 @@ +.. _theory-pic: + +Particle-in-Cell Method +======================= + +.. _fig-pic: + +.. figure:: PIC.png + :alt: [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). + + The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). + +In the *electromagnetic particle-in-cell method* :cite:p:`pt-Birdsalllangdon,pt-HockneyEastwoodBook`, +the electromagnetic fields are solved on a grid, usually using Maxwell’s +equations + +.. math:: + \frac{\mathbf{\partial B}}{\partial t} = -\nabla\times\mathbf{E} + :label: Faraday-1 + +.. math:: + \frac{\mathbf{\partial E}}{\partial t} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere-1 + +.. math:: + \nabla\cdot\mathbf{E} = \rho + :label: Gauss-1 + +.. math:: + \nabla\cdot\mathbf{B} = 0 + :label: divb-1 + +given here in natural units (:math:`\epsilon_0=\mu_0=c=1`), where :math:`t` is time, :math:`\mathbf{E}` and +:math:`\mathbf{B}` are the electric and magnetic field components, and +:math:`\rho` and :math:`\mathbf{J}` are the charge and current densities. The +charged particles are advanced in time using the Newton-Lorentz equations +of motion + +.. math:: + \frac{d\mathbf{x}}{dt} = \mathbf{v}, + :label: Lorentz_x-1 + +.. math:: + \frac{d\left(\gamma\mathbf{v}\right)}{dt} = \frac{q}{m}\left(\mathbf{E}+\mathbf{v}\times\mathbf{B}\right), + :label: Lorentz_v-1 + +where :math:`m`, :math:`q`, :math:`\mathbf{x}`, :math:`\mathbf{v}` and :math:`\gamma=1/\sqrt{1-v^{2}}` +are respectively the mass, charge, position, velocity and relativistic +factor of the particle given in natural units (:math:`c=1`). The charge and current densities are interpolated +on the grid from the particles’ positions and velocities, while the +electric and magnetic field components are interpolated from the grid +to the particles’ positions for the velocity update. + +.. _theory-pic-push: + +Particle push +------------- + +A centered finite-difference discretization of the Newton-Lorentz +equations of motion is given by + +.. math:: + \frac{\mathbf{x}^{i+1}-\mathbf{x}^{i}}{\Delta t} = \mathbf{v}^{i+1/2}, + :label: leapfrog_x + +.. math:: + \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}-\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{\Delta t} = \frac{q}{m}\left(\mathbf{E}^{i}+\mathbf{\bar{v}}^{i}\times\mathbf{B}^{i}\right). + :label: leapfrog_v + +In order to close the system, :math:`\bar{\mathbf{v}}^{i}` must be +expressed as a function of the other quantities. The two implementations that have become the most popular are presented below. + +.. _theory-pic-push-boris: + +Boris relativistic velocity rotation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The solution proposed by Boris :cite:p:`pt-BorisICNSP70` is given by + +.. math:: + \mathbf{\bar{v}}^{i} = \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}+\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{2\bar{\gamma}^{i}} + :label: boris_v + +where :math:`\bar{\gamma}^{i}` is defined by :math:`\bar{\gamma}^{i} \equiv (\gamma^{i+1/2}+\gamma^{i-1/2} )/2`. + +The system (:eq:`leapfrog_v`, :eq:`boris_v`) is solved very +efficiently following Boris’ method, where the electric field push +is decoupled from the magnetic push. Setting :math:`\mathbf{u}=\gamma\mathbf{v}`, the +velocity is updated using the following sequence: + +.. math:: + + \begin{aligned} + \mathbf{u^{-}} & = \mathbf{u}^{i-1/2}+\left(q\Delta t/2m\right)\mathbf{E}^{i} + \\ + \mathbf{u'} & = \mathbf{u}^{-}+\mathbf{u}^{-}\times\mathbf{t} + \\ + \mathbf{u}^{+} & = \mathbf{u}^{-}+\mathbf{u'}\times2\mathbf{t}/(1+\mathbf{t}^{2}) + \\ + \mathbf{u}^{i+1/2} & = \mathbf{u}^{+}+\left(q\Delta t/2m\right)\mathbf{E}^{i} + \end{aligned} + +where :math:`\mathbf{t}=\left(q\Delta t/2m\right)\mathbf{B}^{i}/\bar{\gamma}^{i}` and where +:math:`\bar{\gamma}^{i}` can be calculated as :math:`\bar{\gamma}^{i}=\sqrt{1+(\mathbf{u}^-/c)^2}`. + +The Boris implementation is second-order accurate, time-reversible and fast. Its implementation is very widespread and used in the vast majority of PIC codes. + +.. _theory-pic-push-vay: + +Vay Lorentz-invariant formulation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It was shown in :cite:t:`pt-Vaypop2008` that the Boris formulation is +not Lorentz invariant and can lead to significant errors in the treatment +of relativistic dynamics. A Lorentz invariant formulation is obtained +by considering the following velocity average + +.. math:: + \mathbf{\bar{v}}^{i} = \frac{\mathbf{v}^{i+1/2}+\mathbf{v}^{i-1/2}}{2}. + :label: new_v + +This gives a system that is solvable analytically (see :cite:t:`pt-Vaypop2008` +for a detailed derivation), giving the following velocity update: + +.. math:: + \mathbf{u^{*}} = \mathbf{u}^{i-1/2}+\frac{q\Delta t}{m}\left(\mathbf{E}^{i}+\frac{\mathbf{v}^{i-1/2}}{2}\times\mathbf{B}^{i}\right), + :label: pusher_gamma + +.. math:: + \mathbf{u}^{i+1/2} = \frac{\mathbf{u^{*}}+\left(\mathbf{u^{*}}\cdot\mathbf{t}\right)\mathbf{t}+\mathbf{u^{*}}\times\mathbf{t}}{1+\mathbf{t}^{2}}, + :label: pusher_upr + +where + +.. math:: + + \begin{align} + \mathbf{t} & = \boldsymbol{\tau}/\gamma^{i+1/2}, + \\ + \boldsymbol{\tau} & = \left(q\Delta t/2m\right)\mathbf{B}^{i}, + \\ + \gamma^{i+1/2} & = \sqrt{\sigma+\sqrt{\sigma^{2}+\left(\boldsymbol{\tau}^{2}+w^{2}\right)}}, + \\ + w & = \mathbf{u^{*}}\cdot\boldsymbol{\tau}, + \\ + \sigma & = \left(\gamma'^{2}-\boldsymbol{\tau}^{2}\right)/2, + \\ + \gamma' & = \sqrt{1+(\mathbf{u}^{*}/c)^{2}}. + \end{align} + +This Lorentz invariant formulation +is particularly well suited for the modeling of ultra-relativistic +charged particle beams, where the accurate account of the cancellation +of the self-generated electric and magnetic fields is essential, as +shown in :cite:t:`pt-Vaypop2008`. + +.. _theory-pic-mwsolve: + +Field solve +----------- + +Various methods are available for solving Maxwell’s equations on a +grid, based on finite-differences, finite-volume, finite-element, +spectral, or other discretization techniques that apply most commonly +on single structured or unstructured meshes and less commonly on multiblock +multiresolution grid structures. In this chapter, we summarize the widespread +second order finite-difference time-domain (FDTD) algorithm, its extension +to non-standard finite-differences as well as the pseudo-spectral +analytical time-domain (PSATD) and pseudo-spectral time-domain (PSTD) +algorithms. Extension to multiresolution (or mesh refinement) PIC +is described in, e.g., :cite:t:`pt-VayCSD12,pt-Vaycpc04`. + +.. _fig_yee_grid: + +.. figure:: Yee_grid.png + :alt: [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. + + (left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. + +.. _theory-pic-mwsolve-fdtd: + +Finite-Difference Time-Domain (FDTD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The most popular algorithm for electromagnetic PIC codes is the Finite-Difference +Time-Domain (or FDTD) solver + +.. math:: + D_{t}\mathbf{B} = -\nabla\times\mathbf{E} + :label: Faraday-2 + +.. math:: + D_{t}\mathbf{E} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere-2 + +.. math:: + \left[\nabla\cdot\mathbf{E} = \rho\right] + :label: Gauss-2 + +.. math:: + \left[\nabla\cdot\mathbf{B} = 0\right]. + :label: divb-2 + +The differential operator is defined as :math:`\nabla=D_{x}\mathbf{\hat{x}}+D_{y}\mathbf{\hat{y}}+D_{z}\mathbf{\hat{z}}` +and the finite-difference operators in time and space are defined +respectively as + +.. math:: + + \begin{align} + D_{t}G|_{i,j,k}^{n} & = \frac{(G|_{i,j,k}^{n+1/2}-G|_{i,j,k}^{n-1/2})}{\Delta t}, + \\ + D_{x}G|_{i,j,k}^{n} & = \frac{G|_{i+1/2,j,k}^{n}-G|_{i-1/2,j,k}^{n}}{\Delta x}, + \end{align} + +where :math:`\Delta t` and :math:`\Delta x` are respectively the time step and +the grid cell size along :math:`x`, :math:`n` is the time index and :math:`i`, :math:`j` +and :math:`k` are the spatial indices along :math:`x`, :math:`y` and :math:`z` respectively. +The difference operators along :math:`y` and :math:`z` are obtained by circular +permutation. The equations in brackets are given for completeness, +as they are often not actually solved, thanks to the usage of a so-called +charge conserving algorithm, as explained below. As shown in :numref:`fig_yee_grid`, +the quantities are given on a staggered (or “Yee”) +grid :cite:p:`pt-Yee`, where the electric field components are located +between nodes and the magnetic field components are located in the +center of the cell faces. Knowing the current densities at half-integer steps, +the electric field components are updated alternately with the magnetic +field components at integer and half-integer steps respectively. + +.. _theory-pic-mwsolve-nsfdtd: + +Non-Standard Finite-Difference Time-Domain (NSFDTD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An implementation of the source-free Maxwell’s wave equations for narrow-band +applications based on non-standard finite-differences (NSFD) +was introduced in :cite:t:`pt-Coleieee1997,pt-Coleieee2002`, and +was adapted for wideband applications in :cite:t:`pt-Karkicap06`. At +the Courant limit for the time step and for a given set of parameters, +the stencil proposed in :cite:t:`pt-Karkicap06` has no numerical dispersion +along the principal axes, provided that the cell size is the same +along each dimension (i.e. cubic cells in 3D). The “Cole-Karkkainen” +(or CK) solver uses the non-standard finite difference formulation +(based on extended stencils) of the Maxwell-Ampere equation and can be +implemented as follows :cite:p:`pt-Vayjcp2011`: + +.. math:: + D_{t}\mathbf{B} = -\nabla^{*}\times\mathbf{E} + :label: Faraday + +.. math:: + D_{t}\mathbf{E} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere + +.. math:: + \left[\nabla\cdot\mathbf{E} = \rho\right] + :label: Gauss + +.. math:: + \left[\nabla^{*}\cdot\mathbf{B}= 0\right] + :label: divb + +Eqs. (:eq:`Gauss`) and (:eq:`divb`) are not being solved explicitly +but verified via appropriate initial conditions and current deposition +procedure. The NSFD differential operator is given by + +.. math:: + + \nabla^{*}=D_{x}^{*}\mathbf{\hat{x}}+D_{y}^{*}\mathbf{\hat{y}}+D_{z}^{*}\mathbf{\hat{z}} + +where + +.. math:: + + D_{x}^{*}=\left(\alpha+\beta S_{x}^{1}+\xi S_{x}^{2}\right)D_{x} + +with + +.. math:: + + \begin{align} + S_{x}^{1}G|_{i,j,k}^{n} & = G|_{i,j+1,k}^{n}+G|_{i,j-1,k}^{n}+G|_{i,j,k+1}^{n}+G|_{i,j,k-1}^{n}, + \\ + S_{x}^{2}G|_{i,j,k}^{n} & = G|_{i,j+1,k+1}^{n}+G|_{i,j-1,k+1}^{n}+G|_{i,j+1,k-1}^{n}+G|_{i,j-1,k-1}^{n}. + \end{align} + +Here :math:`G` is a sample vector component, while :math:`\alpha`, :math:`\beta` and :math:`\xi` +are constant scalars satisfying :math:`\alpha+4\beta+4\xi=1`. As with +the FDTD algorithm, the quantities with half-integer are located between +the nodes (electric field components) or in the center of the cell +faces (magnetic field components). The operators along :math:`y` and :math:`z`, +i.e. :math:`D_{y}`, :math:`D_{z}`, :math:`D_{y}^{*}`, :math:`D_{z}^{*}`, :math:`S_{y}^{1}`, +:math:`S_{z}^{1}`, :math:`S_{y}^{2}`, and :math:`S_{z}^{2}`, are obtained by circular +permutation of the indices. + +Assuming cubic cells (:math:`\Delta x=\Delta y=\Delta z`), the coefficients +given in :cite:t:`pt-Karkicap06` (:math:`\alpha=7/12`, :math:`\beta=1/12` and :math:`\xi=1/48`) +allow for the Courant condition to be at :math:`\Delta t=\Delta x`, which +equates to having no numerical dispersion along the principal axes. +The algorithm reduces to the FDTD algorithm with :math:`\alpha=1` and :math:`\beta=\xi=0`. +An extension to non-cubic cells is provided in 3-D by :cite:t:`pt-CowanPRSTAB13` and in 2-D by +:cite:t:`pt-PukhovJPP99`. An alternative NSFDTD implementation that enables superluminous waves is also +given in :cite:t:`pt-LehePRSTAB13`. + +As mentioned above, a key feature of the algorithms based on NSFDTD +is that some implementations :cite:p:`pt-Karkicap06,pt-CowanPRSTAB13` enable the time step :math:`\Delta t=\Delta x` along one or +more axes and no numerical dispersion along those axes. However, as +shown in :cite:t:`pt-Vayjcp2011`, an instability develops at the Nyquist +wavelength at (or very near) such a timestep. It is also shown in +the same paper that removing the Nyquist component in all the source +terms using a bilinear filter (see description of the filter below) +suppresses this instability. + +.. _theory-pic-mwsolve-psatd: + +Pseudo Spectral Analytical Time Domain (PSATD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Maxwell’s equations in Fourier space are given by + +.. math:: \frac{\partial\mathbf{\tilde{E}}}{\partial t} = i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}} +.. math:: \frac{\partial\mathbf{\tilde{B}}}{\partial t} = -i\mathbf{k}\times\mathbf{\tilde{E}} +.. math:: {}[i\mathbf{k}\cdot\mathbf{\tilde{E}} = \tilde{\rho}] +.. math:: {}[i\mathbf{k}\cdot\mathbf{\tilde{B}} = 0] + +where :math:`\tilde{a}` is the Fourier Transform of the quantity :math:`a`. +As with the real space formulation, provided that the continuity equation +:math:`\partial\tilde{\rho}/\partial t+i\mathbf{k}\cdot\mathbf{\tilde{J}}=0` is satisfied, then +the last two equations will automatically be satisfied at any time +if satisfied initially and do not need to be explicitly integrated. + +Decomposing the electric field and current between longitudinal and +transverse components + +.. math:: + + \begin{aligned} + \mathbf{\tilde{E}} & = \mathbf{\tilde{E}}_{L}+\mathbf{\tilde{E}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{E}}) + \\ + \mathbf{\tilde{J}} & = \mathbf{\tilde{J}}_{L}+\mathbf{\tilde{J}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{J}}) + \end{aligned} + +gives + +.. math:: + + \begin{aligned} + \frac{\partial\mathbf{\tilde{E}}_{T}}{\partial t} & = i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}_{T}} + \\ + \frac{\partial\mathbf{\tilde{E}}_{L}}{\partial t} & = -\mathbf{\tilde{J}_{L}} + \\ + \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = -i\mathbf{k}\times\mathbf{\tilde{E}} + \end{aligned} + +with :math:`\mathbf{\hat{k}}=\mathbf{k}/k`. + +If the sources are assumed to be constant over a time interval :math:`\Delta t`, +the system of equations is solvable analytically and is given by (see :cite:t:`pt-Habericnsp73` for the original formulation and :cite:t:`pt-VayJCP2013` +for a more detailed derivation): + +.. math:: + \mathbf{\tilde{E}}_{T}^{n+1} = C\mathbf{\tilde{E}}_{T}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}_{T}^{n+1/2} + :label: PSATD_transverse_1 + +.. math:: + \mathbf{\tilde{E}}_{L}^{n+1} = \mathbf{\tilde{E}}_{L}^{n}-\Delta t\mathbf{\tilde{J}}_{L}^{n+1/2} + :label: PSATD_longitudinal + +.. math:: + \mathbf{\tilde{B}}^{n+1} = C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n} + i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2} + :label: PSATD_transverse_2 + +with :math:`C=\cos\left(k\Delta t\right)` and :math:`S=\sin\left(k\Delta t\right)`. + +Combining the transverse and longitudinal components, gives + +.. math:: + \begin{aligned} + \mathbf{\tilde{E}}^{n+1} & = C\mathbf{\tilde{E}}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}^{n+1/2} + \\ + & + (1-C)\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}}^{n})\nonumber + \\ + & + \mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}}^{n+1/2})\left(\frac{S}{k}-\Delta t\right), + \end{aligned} + :label: Eq_PSATD_1 + +.. math:: + \begin{aligned} + \mathbf{\tilde{B}}^{n+1} & = C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n} + \\ + & + i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}. + \end{aligned} + :label: Eq_PSATD_2 + +For fields generated by the source terms without the self-consistent +dynamics of the charged particles, this algorithm is free of numerical +dispersion and is not subject to a Courant condition. Furthermore, +this solution is exact for any time step size subject to the assumption +that the current source is constant over that time step. + +As shown in :cite:t:`pt-VayJCP2013`, by expanding the coefficients :math:`S_{h}` +and :math:`C_{h}` in Taylor series and keeping the leading terms, the PSATD +formulation reduces to the perhaps better known pseudo-spectral time-domain +(PSTD) formulation :cite:p:`pt-DawsonRMP83,pt-Liumotl1997`: + +.. math:: + + \begin{aligned} + \mathbf{\tilde{E}}^{n+1} & = \mathbf{\tilde{E}}^{n}+i\Delta t\mathbf{k}\times\mathbf{\tilde{B}}^{n+1/2}-\Delta t\mathbf{\tilde{J}}^{n+1/2}, + \\ + \mathbf{\tilde{B}}^{n+3/2} & = \mathbf{\tilde{B}}^{n+1/2}-i\Delta t\mathbf{k}\times\mathbf{\tilde{E}}^{n+1}. + \end{aligned} + +The dispersion relation of the PSTD solver is given by :math:`\sin(\frac{\omega\Delta t}{2})=\frac{k\Delta t}{2}.` +In contrast to the PSATD solver, the PSTD solver is subject to numerical +dispersion for a finite time step and to a Courant condition that +is given by :math:`\Delta t\leq \frac{2}{\pi}\left(\frac{1}{\Delta x^{2}}+\frac{1}{\Delta y^{2}}+\frac{1}{\Delta z^{2}}\right)^{-1/2}`. + +The PSATD and PSTD formulations that were just given apply to the +field components located at the nodes of the grid. As noted in :cite:t:`pt-Ohmurapiers2010`, +they can also be easily recast on a staggered Yee grid by multiplication +of the field components by the appropriate phase factors to shift +them from the collocated to the staggered locations. The choice between +a collocated and a staggered formulation is application-dependent. + +Spectral solvers used to be very popular in the years 1970s to early 1990s, before being replaced by finite-difference methods with the advent of parallel supercomputers that favored local methods. However, it was shown recently that standard domain decomposition with Fast Fourier Transforms that are local to each subdomain could be used effectively with PIC spectral methods :cite:p:`pt-VayJCP2013`, at the cost of truncation errors in the guard cells that could be neglected. A detailed analysis of the effectiveness of the method with exact evaluation of the magnitude of the effect of the truncation error is given in :cite:t:`pt-VincentiCPC2017a` for stencils of arbitrary order (up-to the infinite “spectral” order). + +WarpX also includes a kinetic-fluid hybrid model in which the electric field is +calculated using Ohm's law instead of directly evolving Maxwell's equations. This +approach allows reduced physics simulations to be done with significantly lower +spatial and temporal resolution than in the standard, fully kinetic, PIC. Details +of this model can be found in the section +:ref:`Kinetic-fluid hybrid model `. + +.. _current_deposition: + +Current deposition +------------------ + +The current densities are deposited on the computational grid from +the particle position and velocities, employing splines of various +orders :cite:p:`pt-Abejcp86`. + +.. math:: + + \begin{aligned} + \rho & = \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_nS_n + \\ + \mathbf{J} & = \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_n\mathbf{v_n}S_n + \end{aligned} + +In most applications, it is essential to prevent the accumulation +of errors resulting from the violation of the discretized Gauss’ Law. +This is accomplished by providing a method for depositing the current +from the particles to the grid that preserves the discretized Gauss’ +Law, or by providing a mechanism for “divergence cleaning” :cite:p:`pt-Birdsalllangdon,pt-Langdoncpc92,pt-Marderjcp87,pt-Vaypop98,pt-Munzjcp2000`. +For the former, schemes that allow a deposition of the current that +is exact when combined with the Yee solver is given in :cite:t:`pt-Villasenorcpc92` +for linear splines and in :cite:t:`pt-Esirkepovcpc01` for splines of arbitrary order. + +The NSFDTD formulations given above and in :cite:t:`pt-PukhovJPP99,pt-Vayjcp2011,pt-CowanPRSTAB13,pt-LehePRSTAB13` +apply to the Maxwell-Faraday +equation, while the discretized Maxwell-Ampere equation uses the FDTD +formulation. Consequently, the charge conserving algorithms developed +for current deposition :cite:p:`pt-Villasenorcpc92,pt-Esirkepovcpc01` apply +readily to those NSFDTD-based formulations. More details concerning +those implementations, including the expressions for the numerical +dispersion and Courant condition are given +in :cite:t:`pt-PukhovJPP99,pt-Vayjcp2011,pt-CowanPRSTAB13,pt-LehePRSTAB13`. + +Current correction +~~~~~~~~~~~~~~~~~~ + +In the case of the pseudospectral solvers, the current deposition +algorithm generally does not satisfy the discretized continuity equation +in Fourier space: + +.. math:: + \tilde{\rho}^{n+1}=\tilde{\rho}^{n}-i\Delta t\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}. + +In this case, a Boris correction :cite:p:`pt-Birdsalllangdon` can be applied +in :math:`k` space in the form + +.. math:: + \mathbf{\tilde{E}}_{c}^{n+1}=\mathbf{\tilde{E}}^{n+1}-\frac{\mathbf{k}\cdot\mathbf{\tilde{E}}^{n+1}+i\tilde{\rho}^{n+1}}{k}\mathbf{\hat{k}}, + +where :math:`\mathbf{\tilde{E}}_{c}` is the corrected field. Alternatively, a correction +to the current can be applied (with some similarity to the current +deposition presented by Morse and Nielson in their potential-based +model in :cite:t:`pt-Morsenielson1971`) using + +.. math:: \mathbf{\tilde{J}}_{c}^{n+1/2}=\mathbf{\tilde{J}}^{n+1/2}-\left[\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}-i\left(\tilde{\rho}^{n+1}-\tilde{\rho}^{n}\right)/\Delta t\right]\mathbf{\hat{k}}/k, + +where :math:`\mathbf{\tilde{J}}_{c}` is the corrected current. In this case, the transverse +component of the current is left untouched while the longitudinal +component is effectively replaced by the one obtained from integration +of the continuity equation, ensuring that the corrected current satisfies +the continuity equation. The advantage of correcting the current rather than +the electric field is that it is more local and thus more compatible with +domain decomposition of the fields for parallel computation :cite:p:`pt-VayJCP2013`. + +Vay deposition +~~~~~~~~~~~~~~ + +Alternatively, an exact current deposition can be written for the pseudo-spectral solvers, following the geometrical interpretation of existing methods in real space :cite:p:`pt-Morsenielson1971,pt-Villasenorcpc92,pt-Esirkepovcpc01`. + +The Vay deposition scheme is the generalization of the Esirkepov deposition scheme for the spectral case with arbitrary-order stencils :cite:p:`pt-VayJCP2013`. +The current density :math:`\widehat{\boldsymbol{J}}^{\,n+1/2}` in Fourier space is computed as :math:`\widehat{\boldsymbol{J}}^{\,n+1/2} = i \, \widehat{\boldsymbol{D}} / \boldsymbol{k}` when :math:`\boldsymbol{k} \neq 0` and set to zero otherwise. +The quantity :math:`\boldsymbol{D}` is deposited in real space by averaging the currents over all possible grid paths between the initial position :math:`\boldsymbol{x}^{\,n}` and the final position :math:`\boldsymbol{x}^{\,n+1}` and is defined as + +- 2D Cartesian geometry: + +.. math:: + \begin{align} + D_x & = \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n+1}) + + \Gamma(x_i^{n+1},z_i^{n}) - \Gamma(x_i^{n},z_i^{n}) + \bigg] + \\[8pt] + D_y & = \sum_i \frac{v_i^y}{\Delta x \Delta z} \frac{q_i w_i}{4} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) + \Gamma(x_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n},z_i^{n+1}) + \Gamma(x_i^{n},z_i^{n}) + \bigg] + \\[8pt] + D_z & = \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n}) + \bigg] + \end{align} + +- 3D Cartesian geometry: + +.. math:: + \begin{align} + \begin{split} + D_x & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \\[8pt] + \begin{split} + D_y & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \\[8pt] + \begin{split} + D_z & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + 2 \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \end{align} + +Here, :math:`w_i` represents the weight of the :math:`i`-th macro-particle and :math:`\Gamma` represents its shape factor. +Note that in 2D Cartesian geometry, :math:`D_y` is effectively :math:`J_y` and does not require additional operations in Fourier space. + +Field gather +------------ + +In general, the field is gathered from the mesh onto the macroparticles +using splines of the same order as for the current deposition :math:`\mathbf{S}=\left(S_{x},S_{y},S_{z}\right)`. +Three variations are considered: + +- “momentum conserving”: fields are interpolated from the grid nodes + to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` + for all field components (if the fields are known at staggered positions, + they are first interpolated to the nodes on an auxiliary grid), + +- “energy conserving (or Galerkin)”: fields are interpolated from + the staggered Yee grid to the macroparticles using :math:`\left(S_{nx-1},S_{ny},S_{nz}\right)` + for :math:`E_{x}`, :math:`\left(S_{nx},S_{ny-1},S_{nz}\right)` for :math:`E_{y}`, + :math:`\left(S_{nx},S_{ny},S_{nz-1}\right)` for :math:`E_{z}`, :math:`\left(S_{nx},S_{ny-1},S_{nz-1}\right)` + for :math:`B_{x}`, :math:`\left(S_{nx-1},S_{ny},S_{nz-1}\right)` for :math:`B{}_{y}` + and\ :math:`\left(S_{nx-1},S_{ny-1},S_{nz}\right)` for :math:`B_{z}` (if the fields + are known at the nodes, they are first interpolated to the staggered + positions on an auxiliary grid), + +- “uniform”: fields are interpolated directly form the Yee grid + to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` + for all field components (if the fields are known at the nodes, they + are first interpolated to the staggered positions on an auxiliary + grid). + +As shown in :cite:t:`pt-Birdsalllangdon,pt-HockneyEastwoodBook,pt-LewisJCP1972`, +the momentum and energy conserving schemes conserve momentum and energy +respectively at the limit of infinitesimal time steps and generally +offer better conservation of the respective quantities for a finite +time step. The uniform scheme does not conserve momentum nor energy +in the sense defined for the others but is given for completeness, +as it has been shown to offer some interesting properties in the modeling +of relativistically drifting plasmas :cite:p:`pt-GodfreyJCP2013`. + +.. _theory-pic-filter: + +Filtering +--------- + +It is common practice to apply digital filtering to the charge or +current density in Particle-In-Cell simulations as a complement or +an alternative to using higher order splines :cite:p:`pt-Birdsalllangdon`. +A commonly used filter in PIC simulations is the three points filter + +.. math:: + \phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-1}+\phi_{j+1}\right)/2 + +where :math:`\phi^{f}` is the filtered quantity. This filter is called +a bilinear filter when :math:`\alpha=0.5`. Assuming :math:`\phi=e^{jkx}` and +:math:`\phi^{f}=g\left(\alpha,k\right)e^{jkx}`, the filter gain :math:`g` is +given as a function of the filtering coefficient :math:`\alpha` and +the wavenumber :math:`k` by + +.. math:: + g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(k\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. + +The total attenuation :math:`G` for :math:`n` successive applications of filters +of coefficients :math:`\alpha_{1}`...\ :math:`\alpha_{n}` is given by + +.. math:: + G=\prod_{i=1}^{n}g\left(\alpha_{i},k\right)\approx1-\left(n-\sum_{i=1}^{n}\alpha_{i}\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. + +A sharper cutoff in :math:`k` space is provided by using :math:`\alpha_{n}=n-\sum_{i=1}^{n-1}\alpha_{i}`, +so that :math:`G\approx1+O\left(k^{4}\right)`. Such step is called a “compensation” +step :cite:p:`pt-Birdsalllangdon`. For the bilinear filter (:math:`\alpha=1/2`), +the compensation factor is :math:`\alpha_{c}=2-1/2=3/2`. For a succession +of :math:`n` applications of the bilinear factor, it is :math:`\alpha_{c}=n/2+1`. + +It is sometimes necessary to filter on a relatively wide band of wavelength, +necessitating the application of a large number of passes of the bilinear +filter or on the use of filters acting on many points. The former +can become very intensive computationally while the latter is problematic +for parallel computations using domain decomposition, as the footprint +of the filter may eventually surpass the size of subdomains. A workaround +is to use a combination of filters of limited footprint. A solution +based on the combination of three point filters with various strides +was proposed in :cite:t:`pt-Vayjcp2011` and operates as follows. + +The bilinear filter provides complete suppression of the signal at +the grid Nyquist wavelength (twice the grid cell size). Suppression +of the signal at integer multiples of the Nyquist wavelength can be +obtained by using a stride :math:`s` in the filter + +.. math:: + \phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-s}+\phi_{j+s}\right)/2 + +for which the gain is given by + +.. math:: + g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(sk\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(sk\Delta x\right)^{2}}{2}+O\left(k^{4}\right). + +For a given stride, the gain is given by the gain of the bilinear +filter shifted in k space, with the pole :math:`g=0` shifted from the wavelength +:math:`\lambda=2/\Delta x` to :math:`\lambda=2s/\Delta x`, with additional poles, +as given by :math:`sk\Delta x=\arccos\left(\frac{\alpha}{\alpha-1}\right)\pmod{2\pi}`. +The resulting filter is pass band between the poles, but since the +poles are spread at different integer values in k space, a wide band +low pass filter can be constructed by combining filters using different +strides. As shown in :cite:t:`pt-Vayjcp2011`, the successive application +of 4-passes + compensation of filters with strides 1, 2 and 4 has +a nearly equivalent fall-off in gain as 80 passes + compensation of +a bilinear filter. Yet, the strided filter solution needs only 15 +passes of a three-point filter, compared to 81 passes for an equivalent +n-pass bilinear filter, yielding a gain of 5.4 in number of operations +in favor of the combination of filters with stride. The width of the +filter with stride 4 extends only on 9 points, compared to 81 points +for a single pass equivalent filter, hence giving a gain of 9 in compactness +for the stride filters combination in comparison to the single-pass +filter with large stencil, resulting in more favorable scaling with the number +of computational cores for parallel calculations. + +.. bibliography:: + :keyprefix: pt- diff --git a/Docs/source/theory/picsar_theory.rst b/Docs/source/theory/picsar_theory.rst deleted file mode 100644 index 63672c0fd0a..00000000000 --- a/Docs/source/theory/picsar_theory.rst +++ /dev/null @@ -1,867 +0,0 @@ -.. raw:: latex - - \markboth{J.-L. Vay, R. Lehe}{Simulations for plasma and laser acceleration.} - -.. raw:: latex - - \maketitle - -.. raw:: latex - - \linenumbers - -.. _theory-pic: - -Particle-in-Cell Method -======================= - -.. figure:: PIC.png - :alt: [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). - - [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). - -In the *electromagnetic particle-in-cell method* (Birdsall and Langdon 1991), -the electromagnetic fields are solved on a grid, usually using Maxwell’s -equations - -.. math:: - - \begin{aligned} - \frac{\mathbf{\partial B}}{\partial t} & = & -\nabla\times\mathbf{E}\label{Eq:Faraday-1}\\ - \frac{\mathbf{\partial E}}{\partial t} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere-1}\\ - \nabla\cdot\mathbf{E} & = & \rho\label{Eq:Gauss-1}\\ - \nabla\cdot\mathbf{B} & = & 0\label{Eq:divb-1}\end{aligned} - -given here in natural units (:math:`\epsilon_0=\mu_0=c=1`), where :math:`t` is time, :math:`\mathbf{E}` and -:math:`\mathbf{B}` are the electric and magnetic field components, and -:math:`\rho` and :math:`\mathbf{J}` are the charge and current densities. The -charged particles are advanced in time using the Newton-Lorentz equations -of motion - -.. math:: - - \begin{aligned} - \frac{d\mathbf{x}}{dt}= & \mathbf{v},\label{Eq:Lorentz_x-1}\\ - \frac{d\left(\gamma\mathbf{v}\right)}{dt}= & \frac{q}{m}\left(\mathbf{E}+\mathbf{v}\times\mathbf{B}\right),\label{Eq:Lorentz_v-1}\end{aligned} - -where :math:`m`, :math:`q`, :math:`\mathbf{x}`, :math:`\mathbf{v}` and :math:`\gamma=1/\sqrt{1-v^{2}}` -are respectively the mass, charge, position, velocity and relativistic -factor of the particle given in natural units (:math:`c=1`). The charge and current densities are interpolated -on the grid from the particles’ positions and velocities, while the -electric and magnetic field components are interpolated from the grid -to the particles’ positions for the velocity update. - -.. _theory-pic-push: - -Particle push -------------- - -A centered finite-difference discretization of the Newton-Lorentz -equations of motion is given by - -.. math:: - - \begin{aligned} - \frac{\mathbf{x}^{i+1}-\mathbf{x}^{i}}{\Delta t}= & \mathbf{v}^{i+1/2},\label{Eq:leapfrog_x}\\ - \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}-\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{\Delta t}= & \frac{q}{m}\left(\mathbf{E}^{i}+\mathbf{\bar{v}}^{i}\times\mathbf{B}^{i}\right).\label{Eq:leapfrog_v}\end{aligned} - -In order to close the system, :math:`\bar{\mathbf{v}}^{i}` must be -expressed as a function of the other quantities. The two implementations that have become the most popular are presented below. - -.. _theory-pic-push-boris: - -Boris relativistic velocity rotation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The solution proposed by Boris (Boris 1970) is given by - -.. math:: - - \begin{aligned} - \mathbf{\bar{v}}^{i}= & \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}+\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{2\bar{\gamma}^{i}}.\label{Eq:boris_v}\end{aligned} - -where :math:`\bar{\gamma}^{i}` is defined by :math:`\bar{\gamma}^{i} \equiv (\gamma^{i+1/2}+\gamma^{i-1/2} )/2`. - -The system (`[Eq:leapfrog_v] <#Eq:leapfrog_v>`__,\ `[Eq:boris_v] <#Eq:boris_v>`__) is solved very -efficiently following Boris’ method, where the electric field push -is decoupled from the magnetic push. Setting :math:`\mathbf{u}=\gamma\mathbf{v}`, the -velocity is updated using the following sequence: - -.. math:: - - \begin{aligned} - \mathbf{u^{-}}= & \mathbf{u}^{i-1/2}+\left(q\Delta t/2m\right)\mathbf{E}^{i}\\ - \mathbf{u'}= & \mathbf{u}^{-}+\mathbf{u}^{-}\times\mathbf{t}\\ - \mathbf{u}^{+}= & \mathbf{u}^{-}+\mathbf{u'}\times2\mathbf{t}/(1+t^{2})\\ - \mathbf{u}^{i+1/2}= & \mathbf{u}^{+}+\left(q\Delta t/2m\right)\mathbf{E}^{i}\end{aligned} - -where :math:`\mathbf{t}=\left(q\Delta t/2m\right)\mathbf{B}^{i}/\bar{\gamma}^{i}` and where -:math:`\bar{\gamma}^{i}` can be calculated as :math:`\bar{\gamma}^{i}=\sqrt{1+(\mathbf{u}^-/c)^2}`. - -The Boris implementation is second-order accurate, time-reversible and fast. Its implementation is very widespread and used in the vast majority of PIC codes. - -.. _theory-pic-push-vay: - -Vay Lorentz-invariant formulation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It was shown in (Vay 2008) that the Boris formulation is -not Lorentz invariant and can lead to significant errors in the treatment -of relativistic dynamics. A Lorentz invariant formulation is obtained -by considering the following velocity average - -.. math:: - - \begin{aligned} - \mathbf{\bar{v}}^{i}= & \frac{\mathbf{v}^{i+1/2}+\mathbf{v}^{i-1/2}}{2},\label{Eq:new_v}\end{aligned} - -This gives a system that is solvable analytically (see (Vay 2008) -for a detailed derivation), giving the following velocity update: - -.. math:: - - \begin{aligned} - \mathbf{u^{*}}= & \mathbf{u}^{i-1/2}+\frac{q\Delta t}{m}\left(\mathbf{E}^{i}+\frac{\mathbf{v}^{i-1/2}}{2}\times\mathbf{B}^{i}\right),\label{pusher_gamma}\\ - \mathbf{u}^{i+1/2}= & \left[\mathbf{u^{*}}+\left(\mathbf{u^{*}}\cdot\mathbf{t}\right)\mathbf{t}+\mathbf{u^{*}}\times\mathbf{t}\right]/\left(1+t^{2}\right),\label{pusher_upr}\end{aligned} - -where :math:`\mathbf{t}=\boldsymbol{\tau}/\gamma^{i+1/2}`, :math:`\boldsymbol{\tau}=\left(q\Delta t/2m\right)\mathbf{B}^{i}`, -:math:`\gamma^{i+1/2}=\sqrt{\sigma+\sqrt{\sigma^{2}+\left(\tau^{2}+w^{2}\right)}}`, -:math:`w=\mathbf{u^{*}}\cdot\boldsymbol{\tau}`, :math:`\sigma=\left(\gamma'^{2}-\tau^{2}\right)/2` -and :math:`\gamma'=\sqrt{1+(\mathbf{u}^{*}/c)^{2}}`. This Lorentz invariant formulation -is particularly well suited for the modeling of ultra-relativistic -charged particle beams, where the accurate account of the cancellation -of the self-generated electric and magnetic fields is essential, as -shown in (Vay 2008). - -.. _theory-pic-mwsolve: - -Field solve ------------ - -Various methods are available for solving Maxwell’s equations on a -grid, based on finite-differences, finite-volume, finite-element, -spectral, or other discretization techniques that apply most commonly -on single structured or unstructured meshes and less commonly on multiblock -multiresolution grid structures. In this chapter, we summarize the widespread -second order finite-difference time-domain (FDTD) algorithm, its extension -to non-standard finite-differences as well as the pseudo-spectral -analytical time-domain (PSATD) and pseudo-spectral time-domain (PSTD) -algorithms. Extension to multiresolution (or mesh refinement) PIC -is described in, e.g. (Vay et al. 2012; Vay, Adam, and Heron 2004). - -.. _theory-pic-mwsolve-fdtd: - -Finite-Difference Time-Domain (FDTD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The most popular algorithm for electromagnetic PIC codes is the Finite-Difference -Time-Domain (or FDTD) solver - -.. math:: - - \begin{aligned} - D_{t}\mathbf{B} & = & -\nabla\times\mathbf{E}\label{Eq:Faraday-2}\\ - D_{t}\mathbf{E} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere-2}\\ - \left[\nabla\cdot\mathbf{E}\right. & = & \left.\rho\right]\label{Eq:Gauss-2}\\ - \left[\nabla\cdot\mathbf{B}\right. & = & \left.0\right].\label{Eq:divb-2}\end{aligned} - -.. figure:: Yee_grid.png - :alt: [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. - - [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. - -The differential operator is defined as :math:`\nabla=D_{x}\mathbf{\hat{x}}+D_{y}\mathbf{\hat{y}}+D_{z}\mathbf{\hat{z}}` -and the finite-difference operators in time and space are defined -respectively as - -.. math:: D_{t}G|_{i,j,k}^{n}=\left(G|_{i,j,k}^{n+1/2}-G|_{i,j,k}^{n-1/2}\right)/\Delta t - -and :math:`D_{x}G|_{i,j,k}^{n}=\left(G|_{i+1/2,j,k}^{n}-G|_{i-1/2,j,k}^{n}\right)/\Delta x`, -where :math:`\Delta t` and :math:`\Delta x` are respectively the time step and -the grid cell size along :math:`x`, :math:`n` is the time index and :math:`i`, :math:`j` -and :math:`k` are the spatial indices along :math:`x`, :math:`y` and :math:`z` respectively. -The difference operators along :math:`y` and :math:`z` are obtained by circular -permutation. The equations in brackets are given for completeness, -as they are often not actually solved, thanks to the usage of a so-called -charge conserving algorithm, as explained below. As shown in Figure -`[fig:yee_grid] <#fig:yee_grid>`__, the quantities are given on a staggered (or “Yee”) -grid (Yee 1966), where the electric field components are located -between nodes and the magnetic field components are located in the -center of the cell faces. Knowing the current densities at half-integer steps, -the electric field components are updated alternately with the magnetic -field components at integer and half-integer steps respectively. - -.. _theory-pic-mwsolve-nsfdtd: - -Non-Standard Finite-Difference Time-Domain (NSFDTD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In (Cole 1997, 2002), Cole introduced an implementation -of the source-free Maxwell’s wave equations for narrow-band applications -based on non-standard finite-differences (NSFD). In (Karkkainen et al. 2006), -Karkkainen *et al.* adapted it for wideband applications. At -the Courant limit for the time step and for a given set of parameters, -the stencil proposed in (Karkkainen et al. 2006) has no numerical dispersion -along the principal axes, provided that the cell size is the same -along each dimension (i.e. cubic cells in 3D). The “Cole-Karkkainnen” -(or CK) solver uses the non-standard finite difference formulation -(based on extended stencils) of the Maxwell-Ampere equation and can be -implemented as follows (Vay et al. 2011): - -.. math:: - - \begin{aligned} - D_{t}\mathbf{B} & = & -\nabla^{*}\times\mathbf{E}\label{Eq:Faraday}\\ - D_{t}\mathbf{E} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere}\\ - \left[\nabla\cdot\mathbf{E}\right. & = & \left.\rho\right]\label{Eq:Gauss}\\ - \left[\nabla^{*}\cdot\mathbf{B}\right. & = & \left.0\right]\label{Eq:divb}\end{aligned} - -Eq. `[Eq:Gauss] <#Eq:Gauss>`__ and `[Eq:divb] <#Eq:divb>`__ are not being solved explicitly -but verified via appropriate initial conditions and current deposition -procedure. The NSFD differential operators is given by :math:`\nabla^{*}=D_{x}^{*}\mathbf{\hat{x}}+D_{y}^{*}\mathbf{\hat{y}}+D_{z}^{*}\mathbf{\hat{z}}` -where :math:`D_{x}^{*}=\left(\alpha+\beta S_{x}^{1}+\xi S_{x}^{2}\right)D_{x}` -with :math:`S_{x}^{1}G|_{i,j,k}^{n}=G|_{i,j+1,k}^{n}+G|_{i,j-1,k}^{n}+G|_{i,j,k+1}^{n}+G|_{i,j,k-1}^{n}`, -:math:`S_{x}^{2}G|_{i,j,k}^{n}=G|_{i,j+1,k+1}^{n}+G|_{i,j-1,k+1}^{n}+G|_{i,j+1,k-1}^{n}+G|_{i,j-1,k-1}^{n}`. -:math:`G` is a sample vector component, while :math:`\alpha`, :math:`\beta` and :math:`\xi` -are constant scalars satisfying :math:`\alpha+4\beta+4\xi=1`. As with -the FDTD algorithm, the quantities with half-integer are located between -the nodes (electric field components) or in the center of the cell -faces (magnetic field components). The operators along :math:`y` and :math:`z`, -i.e. :math:`D_{y}`, :math:`D_{z}`, :math:`D_{y}^{*}`, :math:`D_{z}^{*}`, :math:`S_{y}^{1}`, -:math:`S_{z}^{1}`, :math:`S_{y}^{2}`, and :math:`S_{z}^{2}`, are obtained by circular -permutation of the indices. - -Assuming cubic cells (:math:`\Delta x=\Delta y=\Delta z`), the coefficients -given in (Karkkainen et al. 2006) (:math:`\alpha=7/12`, :math:`\beta=1/12` and :math:`\xi=1/48`) -allow for the Courant condition to be at :math:`\Delta t=\Delta x`, which -equates to having no numerical dispersion along the principal axes. -The algorithm reduces to the FDTD algorithm with :math:`\alpha=1` and :math:`\beta=\xi=0`. -An extension to non-cubic cells is provided by Cowan, *et al.* -in 3-D in (Cowan et al. 2013) and was given by Pukhov in 2-D in -(Pukhov 1999). An alternative NSFDTD implementation that enables superluminous waves is also -given by Lehe et al. in (Lehe et al. 2013). - -As mentioned above, a key feature of the algorithms based on NSFDTD -is that some implementations (Karkkainen et al. 2006; Cowan et al. 2013) enable the time step :math:`\Delta t=\Delta x` along one or -more axes and no numerical dispersion along those axes. However, as -shown in (Vay et al. 2011), an instability develops at the Nyquist -wavelength at (or very near) such a timestep. It is also shown in -the same paper that removing the Nyquist component in all the source -terms using a bilinear filter (see description of the filter below) -suppresses this instability. - -.. _theory-pic-mwsolve-psatd: - -Pseudo Spectral Analytical Time Domain (PSATD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Maxwell’s equations in Fourier space are given by - -.. math:: - - \begin{aligned} - \frac{\partial\mathbf{\tilde{E}}}{\partial t} & = & i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}}\\ - \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = & -i\mathbf{k}\times\mathbf{\tilde{E}}\\ - {}[i\mathbf{k}\cdot\mathbf{\tilde{E}}& = & \tilde{\rho}]\\ - {}[i\mathbf{k}\cdot\mathbf{\tilde{B}}& = & 0]\end{aligned} - -where :math:`\tilde{a}` is the Fourier Transform of the quantity :math:`a`. -As with the real space formulation, provided that the continuity equation -:math:`\partial\tilde{\rho}/\partial t+i\mathbf{k}\cdot\mathbf{\tilde{J}}=0` is satisfied, then -the last two equations will automatically be satisfied at any time -if satisfied initially and do not need to be explicitly integrated. - -Decomposing the electric field and current between longitudinal and -transverse components :math:`\mathbf{\tilde{E}}=\mathbf{\tilde{E}}_{L}+\mathbf{\tilde{E}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{E}})` -and :math:`\mathbf{\tilde{J}}=\mathbf{\tilde{J}}_{L}+\mathbf{\tilde{J}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{J}})` -gives - -.. math:: - - \begin{aligned} - \frac{\partial\mathbf{\tilde{E}}_{T}}{\partial t} & = & i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}_{T}}\\ - \frac{\partial\mathbf{\tilde{E}}_{L}}{\partial t} & = & -\mathbf{\tilde{J}_{L}}\\ - \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = & -i\mathbf{k}\times\mathbf{\tilde{E}}\end{aligned} - -with :math:`\mathbf{\hat{k}}=\mathbf{k}/k`. - -If the sources are assumed to be constant over a time interval :math:`\Delta t`, -the system of equations is solvable analytically and is given by (see -(Haber et al. 1973) for the original formulation and (Jean-Luc Vay, Haber, and Godfrey 2013) -for a more detailed derivation): - -[Eq:PSATD] - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}_{T}^{n+1} & = & C\mathbf{\tilde{E}}_{T}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}_{T}^{n+1/2}\label{Eq:PSATD_transverse_1}\\ - \mathbf{\tilde{E}}_{L}^{n+1} & = & \mathbf{\tilde{E}}_{L}^{n}-\Delta t\mathbf{\tilde{J}}_{L}^{n+1/2}\\ - \mathbf{\tilde{B}}^{n+1} & = & C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n}\\ - &+&i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}\label{Eq:PSATD_transverse_2}\end{aligned} - -with :math:`C=\cos\left(k\Delta t\right)` and :math:`S=\sin\left(k\Delta t\right)`. - -Combining the transverse and longitudinal components, gives - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}^{n+1} & = & C\mathbf{\tilde{E}}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}^{n+1/2}\\ - & + &(1-C)\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}}^{n})\nonumber \\ - & + & \mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}}^{n+1/2})\left(\frac{S}{k}-\Delta t\right),\label{Eq_PSATD_1}\\ - \mathbf{\tilde{B}}^{n+1} & = & C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n}\\ - &+&i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}.\label{Eq_PSATD_2}\end{aligned} - -For fields generated by the source terms without the self-consistent -dynamics of the charged particles, this algorithm is free of numerical -dispersion and is not subject to a Courant condition. Furthermore, -this solution is exact for any time step size subject to the assumption -that the current source is constant over that time step. - -As shown in (Jean-Luc Vay, Haber, and Godfrey 2013), by expanding the coefficients :math:`S_{h}` -and :math:`C_{h}` in Taylor series and keeping the leading terms, the PSATD -formulation reduces to the perhaps better known pseudo-spectral time-domain -(PSTD) formulation (Dawson 1983; Liu 1997): - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}^{n+1} & = & \mathbf{\tilde{E}}^{n}+i\Delta t\mathbf{k}\times\mathbf{\tilde{B}}^{n+1/2}-\Delta t\mathbf{\tilde{J}}^{n+1/2},\\ - \mathbf{\tilde{B}}^{n+3/2} & = & \mathbf{\tilde{B}}^{n+1/2}-i\Delta t\mathbf{k}\times\mathbf{\tilde{E}}^{n+1}.\end{aligned} - -The dispersion relation of the PSTD solver is given by :math:`\sin(\frac{\omega\Delta t}{2})=\frac{k\Delta t}{2}.` -In contrast to the PSATD solver, the PSTD solver is subject to numerical -dispersion for a finite time step and to a Courant condition that -is given by :math:`\Delta t\leq \frac{2}{\pi}\left(\frac{1}{\Delta x^{2}}+\frac{1}{\Delta y^{2}}+\frac{1}{\Delta x^{2}}\right)^{-1/2}.` - -The PSATD and PSTD formulations that were just given apply to the -field components located at the nodes of the grid. As noted in (Ohmura and Okamura 2010), -they can also be easily recast on a staggered Yee grid by multiplication -of the field components by the appropriate phase factors to shift -them from the collocated to the staggered locations. The choice between -a collocated and a staggered formulation is application-dependent. - -Spectral solvers used to be very popular in the years 1970s to early 1990s, before being replaced by finite-difference methods with the advent of parallel supercomputers that favored local methods. However, it was shown recently that standard domain decomposition with Fast Fourier Transforms that are local to each subdomain could be used effectively with PIC spectral methods (Jean-Luc Vay, Haber, and Godfrey 2013), at the cost of truncation errors in the guard cells that could be neglected. A detailed analysis of the effectiveness of the method with exact evaluation of the magnitude of the effect of the truncation error is given in (Vincenti and Vay 2016) for stencils of arbitrary order (up-to the infinite “spectral” order). - -WarpX also includes a kinetic-fluid hybrid model in which the electric field is -calculated using Ohm's law instead of directly evolving Maxwell's equations. This -approach allows reduced physics simulations to be done with significantly lower -spatial and temporal resolution than in the standard, fully kinetic, PIC. Details -of this model can be found in the section -:ref:`Kinetic-fluid hybrid model `. - -.. _current_deposition: - -Current deposition ------------------- - -The current densities are deposited on the computational grid from -the particle position and velocities, employing splines of various -orders (Abe et al. 1986). - -.. math:: - - \begin{aligned} - \rho & = & \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_nS_n\\ - \mathbf{J} & = & \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_n\mathbf{v_n}S_n\end{aligned} - -In most applications, it is essential to prevent the accumulation -of errors resulting from the violation of the discretized Gauss’ Law. -This is accomplished by providing a method for depositing the current -from the particles to the grid that preserves the discretized Gauss’ -Law, or by providing a mechanism for “divergence cleaning” (Birdsall and Langdon 1991; Langdon 1992; Marder 1987; Vay and Deutsch 1998; Munz et al. 2000). -For the former, schemes that allow a deposition of the current that -is exact when combined with the Yee solver is given in (Villasenor and Buneman 1992) -for linear splines and in (Esirkepov 2001) for splines of arbitrary order. - -The NSFDTD formulations given above and in (Pukhov 1999; Vay et al. 2011; Cowan et al. 2013; Lehe et al. 2013) -apply to the Maxwell-Faraday -equation, while the discretized Maxwell-Ampere equation uses the FDTD -formulation. Consequently, the charge conserving algorithms developed -for current deposition (Villasenor and Buneman 1992; Esirkepov 2001) apply -readily to those NSFDTD-based formulations. More details concerning -those implementations, including the expressions for the numerical -dispersion and Courant condition are given -in (Pukhov 1999; Vay et al. 2011; Cowan et al. 2013; Lehe et al. 2013). - -Current correction -~~~~~~~~~~~~~~~~~~ - -In the case of the pseudospectral solvers, the current deposition -algorithm generally does not satisfy the discretized continuity equation -in Fourier space :math:`\tilde{\rho}^{n+1}=\tilde{\rho}^{n}-i\Delta t\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}`. -In this case, a Boris correction (Birdsall and Langdon 1991) can be applied -in :math:`k` space in the form :math:`\mathbf{\tilde{E}}_{c}^{n+1}=\mathbf{\tilde{E}}^{n+1}-\left(\mathbf{k}\cdot\mathbf{\tilde{E}}^{n+1}+i\tilde{\rho}^{n+1}\right)\mathbf{\hat{k}}/k`, -where :math:`\mathbf{\tilde{E}}_{c}` is the corrected field. Alternatively, a correction -to the current can be applied (with some similarity to the current -deposition presented by Morse and Nielson in their potential-based -model in (Morse and Nielson 1971)) using :math:`\mathbf{\tilde{J}}_{c}^{n+1/2}=\mathbf{\tilde{J}}^{n+1/2}-\left[\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}-i\left(\tilde{\rho}^{n+1}-\tilde{\rho}^{n}\right)/\Delta t\right]\mathbf{\hat{k}}/k`, -where :math:`\mathbf{\tilde{J}}_{c}` is the corrected current. In this case, the transverse -component of the current is left untouched while the longitudinal -component is effectively replaced by the one obtained from integration -of the continuity equation, ensuring that the corrected current satisfies -the continuity equation. The advantage of correcting the current rather than -the electric field is that it is more local and thus more compatible with -domain decomposition of the fields for parallel computation (Jean Luc Vay, Haber, and Godfrey 2013). - -Vay deposition -~~~~~~~~~~~~~~ - -Alternatively, an exact current deposition can be written for the pseudo-spectral solvers, following the geometrical interpretation of existing methods in real space (`Morse and Nielson, 1971 `_; `Villasenor and Buneman, 1992 `_; `Esirkepov, 2001 `_). - -The Vay deposition scheme is the generalization of the Esirkepov deposition scheme for the spectral case with arbitrary-order stencils `(Vay et al, 2013) `_. -The current density :math:`\widehat{\boldsymbol{J}}^{\,n+1/2}` in Fourier space is computed as :math:`\widehat{\boldsymbol{J}}^{\,n+1/2} = i \, \widehat{\boldsymbol{D}} / \boldsymbol{k}` when :math:`\boldsymbol{k} \neq 0` and set to zero otherwise. -The quantity :math:`\boldsymbol{D}` is deposited in real space by averaging the currents over all possible grid paths between the initial position :math:`\boldsymbol{x}^{\,n}` and the final position :math:`\boldsymbol{x}^{\,n+1}` and is defined as - -- 2D Cartesian geometry: - -.. math:: - \begin{align} - D_x = & \: \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n+1}) - + \Gamma(x_i^{n+1},z_i^{n}) - \Gamma(x_i^{n},z_i^{n}) - \bigg] - \\[8pt] - D_y = & \: \sum_i \frac{v_i^y}{\Delta x \Delta z} \frac{q_i w_i}{4} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) + \Gamma(x_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n},z_i^{n+1}) + \Gamma(x_i^{n},z_i^{n}) - \bigg] - \\[8pt] - D_z = & \: \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n}) - \bigg] - \end{align} - -- 3D Cartesian geometry: - -.. math:: - \begin{align} - \begin{split} - D_x = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) \\[4pt] - & + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) - + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) \\[4pt] - & - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} \\[8pt] - \begin{split} - D_y = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) \\[4pt] - & + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) - + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) \\[4pt] - & - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} \\[8pt] - \begin{split} - D_z = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) \\[4pt] - & + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) \\[4pt] - & - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + 2 \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} - \end{align} - -Here, :math:`w_i` represents the weight of the :math:`i`-th macro-particle and :math:`\Gamma` represents its shape factor. -Note that in 2D Cartesian geometry, :math:`D_y` is effectively :math:`J_y` and does not require additional operations in Fourier space. - -Field gather ------------- - -In general, the field is gathered from the mesh onto the macroparticles -using splines of the same order as for the current deposition :math:`\mathbf{S}=\left(S_{x},S_{y},S_{z}\right)`. -Three variations are considered: - -- “momentum conserving”: fields are interpolated from the grid nodes - to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` - for all field components (if the fields are known at staggered positions, - they are first interpolated to the nodes on an auxiliary grid), - -- “energy conserving (or Galerkin)”: fields are interpolated from - the staggered Yee grid to the macroparticles using :math:`\left(S_{nx-1},S_{ny},S_{nz}\right)` - for :math:`E_{x}`, :math:`\left(S_{nx},S_{ny-1},S_{nz}\right)` for :math:`E_{y}`, - :math:`\left(S_{nx},S_{ny},S_{nz-1}\right)` for :math:`E_{z}`, :math:`\left(S_{nx},S_{ny-1},S_{nz-1}\right)` - for :math:`B_{x}`, :math:`\left(S_{nx-1},S_{ny},S_{nz-1}\right)` for :math:`B{}_{y}` - and\ :math:`\left(S_{nx-1},S_{ny-1},S_{nz}\right)` for :math:`B_{z}` (if the fields - are known at the nodes, they are first interpolated to the staggered - positions on an auxiliary grid), - -- “uniform”: fields are interpolated directly form the Yee grid - to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` - for all field components (if the fields are known at the nodes, they - are first interpolated to the staggered positions on an auxiliary - grid). - -As shown in :cite:t:`BirdsallLangdon,HockneyEastwoodBook,LewisJCP1972`, -the momentum and energy conserving schemes conserve momentum and energy -respectively at the limit of infinitesimal time steps and generally -offer better conservation of the respective quantities for a finite -time step. The uniform scheme does not conserve momentum nor energy -in the sense defined for the others but is given for completeness, -as it has been shown to offer some interesting properties in the modeling -of relativistically drifting plasmas :cite:p:`GodfreyJCP2013`. - -.. _theory-pic-filter: - -Filtering -========= - -It is common practice to apply digital filtering to the charge or -current density in Particle-In-Cell simulations as a complement or -an alternative to using higher order splines (Birdsall and Langdon 1991). -A commonly used filter in PIC simulations is the three points filter -:math:`\phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-1}+\phi_{j+1}\right)/2` -where :math:`\phi^{f}` is the filtered quantity. This filter is called -a bilinear filter when :math:`\alpha=0.5`. Assuming :math:`\phi=e^{jkx}` and -:math:`\phi^{f}=g\left(\alpha,k\right)e^{jkx}`, the filter gain :math:`g` is -given as a function of the filtering coefficient :math:`\alpha` and -the wavenumber :math:`k` by :math:`g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(k\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -The total attenuation :math:`G` for :math:`n` successive applications of filters -of coefficients :math:`\alpha_{1}`...\ :math:`\alpha_{n}` is given by :math:`G=\prod_{i=1}^{n}g\left(\alpha_{i},k\right)\approx1-\left(n-\sum_{i=1}^{n}\alpha_{i}\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -A sharper cutoff in :math:`k` space is provided by using :math:`\alpha_{n}=n-\sum_{i=1}^{n-1}\alpha_{i}`, -so that :math:`G\approx1+O\left(k^{4}\right)`. Such step is called a “compensation” -step (Birdsall and Langdon 1991). For the bilinear filter (:math:`\alpha=1/2`), -the compensation factor is :math:`\alpha_{c}=2-1/2=3/2`. For a succession -of :math:`n` applications of the bilinear factor, it is :math:`\alpha_{c}=n/2+1`. - -It is sometimes necessary to filter on a relatively wide band of wavelength, -necessitating the application of a large number of passes of the bilinear -filter or on the use of filters acting on many points. The former -can become very intensive computationally while the latter is problematic -for parallel computations using domain decomposition, as the footprint -of the filter may eventually surpass the size of subdomains. A workaround -is to use a combination of filters of limited footprint. A solution -based on the combination of three point filters with various strides -was proposed in (Vay et al. 2011) and operates as follows. - -The bilinear filter provides complete suppression of the signal at -the grid Nyquist wavelength (twice the grid cell size). Suppression -of the signal at integer multiples of the Nyquist wavelength can be -obtained by using a stride :math:`s` in the filter :math:`\phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-s}+\phi_{j+s}\right)/2` -for which the gain is given by :math:`g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(sk\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(sk\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -For a given stride, the gain is given by the gain of the bilinear -filter shifted in k space, with the pole :math:`g=0` shifted from the wavelength -:math:`\lambda=2/\Delta x` to :math:`\lambda=2s/\Delta x`, with additional poles, -as given by :math:`sk\Delta x=\arccos\left(\frac{\alpha}{\alpha-1}\right)\pmod{2\pi}`. -The resulting filter is pass band between the poles, but since the -poles are spread at different integer values in k space, a wide band -low pass filter can be constructed by combining filters using different -strides. As shown in (Vay et al. 2011), the successive application -of 4-passes + compensation of filters with strides 1, 2 and 4 has -a nearly equivalent fall-off in gain as 80 passes + compensation of -a bilinear filter. Yet, the strided filter solution needs only 15 -passes of a three-point filter, compared to 81 passes for an equivalent -n-pass bilinear filter, yielding a gain of 5.4 in number of operations -in favor of the combination of filters with stride. The width of the -filter with stride 4 extends only on 9 points, compared to 81 points -for a single pass equivalent filter, hence giving a gain of 9 in compactness -for the stride filters combination in comparison to the single-pass -filter with large stencil, resulting in more favorable scaling with the number -of computational cores for parallel calculations. - -.. raw:: latex - - \IfFileExists{\jobname.bbl}{} {\typeout{} \typeout{{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}} - \typeout{{*}{*} Please run \textquotedbl{}bibtex \jobname\textquotedbl{} - to optain} \typeout{{*}{*} the bibliography and then re-run LaTeX} - \typeout{{*}{*} twice to fix the references!} \typeout{{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}} - \typeout{} } - -.. raw:: html - -
- -.. raw:: html - -
- -Abe, H, N Sakairi, R Itatani, and H Okuda. 1986. “High-Order Spline Interpolations in the Particle Simulation.” *Journal of Computational Physics* 63 (2): 247–67. - -.. raw:: html - -
- -.. raw:: html - -
- -Birdsall, C K, and A B Langdon. 1991. *Plasma Physics via Computer Simulation*. Adam-Hilger. - -.. raw:: html - -
- -.. raw:: html - -
- -Boris, Jp. 1970. “Relativistic Plasma Simulation-Optimization of a Hybrid Code.” In *Proc. Fourth Conf. Num. Sim. Plasmas*, 3–67. Naval Res. Lab., Wash., D. C. - -.. raw:: html - -
- -.. raw:: html - -
- -Cole, J. B. 1997. “A High-Accuracy Realization of the Yee Algorithm Using Non-Standard Finite Differences.” *Ieee Transactions on Microwave Theory and Techniques* 45 (6): 991–96. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2002. “High-Accuracy Yee Algorithm Based on Nonstandard Finite Differences: New Developments and Verifications.” *Ieee Transactions on Antennas and Propagation* 50 (9): 1185–91. https://doi.org/10.1109/Tap.2002.801268. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, Benjamin M, David L Bruhwiler, John R Cary, Estelle Cormier-Michel, and Cameron G R Geddes. 2013. “Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations.” *Physical Review Special Topics-Accelerators and Beams* 16 (4). https://doi.org/10.1103/PhysRevSTAB.16.041303. - -.. raw:: html - -
- -.. raw:: html - -
- -Dawson, J M. 1983. “Particle Simulation of Plasmas.” *Reviews of Modern Physics* 55 (2): 403–47. https://doi.org/10.1103/RevModPhys.55.403. - -.. raw:: html - -
- -.. raw:: html - -
- -Esirkepov, Tz. 2001. “Exact Charge Conservation Scheme for Particle-in-Cell Simulation with an Arbitrary Form-Factor.” *Computer Physics Communications* 135 (2): 144–53. - -.. raw:: html - -
- -.. raw:: html - -
- -Haber, I, R Lee, Hh Klein, and Jp Boris. 1973. “Advances in Electromagnetic Simulation Techniques.” In *Proc. Sixth Conf. Num. Sim. Plasmas*, 46–48. Berkeley, Ca. - -.. raw:: html - -
- -.. raw:: html - -
- -Karkkainen, M, E Gjonaj, T Lau, and T Weiland. 2006. “Low-Dispersionwake Field Calculation Tools.” In *Proc. Of International Computational Accelerator Physics Conference*, 35–40. Chamonix, France. - -.. raw:: html - -
- -.. raw:: html - -
- -Langdon, A B. 1992. “On Enforcing Gauss Law in Electromagnetic Particle-in-Cell Codes.” *Computer Physics Communications* 70 (3): 447–50. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, R, A Lifschitz, C Thaury, V Malka, and X Davoine. 2013. “Numerical growth of emittance in simulations of laser-wakefield acceleration.” *Physical Review Special Topics-Accelerators and Beams* 16 (2). https://doi.org/10.1103/PhysRevSTAB.16.021301. - -.. raw:: html - -
- -.. raw:: html - -
- -Liu, Qh. 1997. “The PSTD Algorithm: A Time-Domain Method Requiring Only Two Cells Per Wavelength.” *Microwave and Optical Technology Letters* 15 (3): 158–65. `https://doi.org/10.1002/(SICI)1098-2760(19970620)15\:3\<158\:\:AID-MOP11\>3.0.CO\;2-3 3.0.CO\;2-3>`_. - -.. raw:: html - -
- -.. raw:: html - -
- -Marder, B. 1987. “A Method for Incorporating Gauss Law into Electromagnetic Pic Codes.” *Journal of Computational Physics* 68 (1): 48–55. - -.. raw:: html - -
- -.. raw:: html - -
- -Morse, Rl, and Cw Nielson. 1971. “Numerical Simulation of Weibel Instability in One and 2 Dimensions.” *Phys. Fluids* 14 (4): 830 –&. https://doi.org/10.1063/1.1693518. - -.. raw:: html - -
- -.. raw:: html - -
- -Munz, Cd, P Omnes, R Schneider, E Sonnendrucker, and U Voss. 2000. “Divergence Correction Techniques for Maxwell Solvers Based on A Hyperbolic Model.” *Journal of Computational Physics* 161 (2): 484–511. https://doi.org/10.1006/Jcph.2000.6507. - -.. raw:: html - -
- -.. raw:: html - -
- -Ohmura, Y, and Y Okamura. 2010. “Staggered Grid Pseudo-Spectral Time-Domain Method for Light Scattering Analysis.” *Piers Online* 6 (7): 632–35. - -.. raw:: html - -
- -.. raw:: html - -
- -Pukhov, A. 1999. “Three-dimensional electromagnetic relativistic particle-in-cell code VLPL (Virtual Laser Plasma Lab).” *Journal of Plasma Physics* 61 (3): 425–33. https://doi.org/10.1017/S0022377899007515. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean-Luc, Irving Haber, and Brendan B Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243 (June): 260–68. https://doi.org/10.1016/j.jcp.2013.03.010. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean Luc, Irving Haber, and Brendan B. Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243: 260–68. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L. 2008. “Simulation of Beams or Plasmas Crossing at Relativistic Velocity.” *Physics of Plasmas* 15 (5): 56701. https://doi.org/10.1063/1.2837054. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., J.-C. Adam, and A Heron. 2004. “Asymmetric Pml for the Absorption of Waves. Application to Mesh Refinement in Electromagnetic Particle-in-Cell Plasma Simulations.” *Computer Physics Communications* 164 (1-3): 171–77. https://doi.org/10.1016/J.Cpc.2004.06.026. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., and C Deutsch. 1998. “Charge Compensated Ion Beam Propagation in A Reactor Sized Chamber.” *Physics of Plasmas* 5 (4): 1190–7. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Numerical Methods for Instability Mitigation in the Modeling of Laser Wakefield Accelerators in A Lorentz-Boosted Frame.” *Journal of Computational Physics* 230 (15): 5908–29. https://doi.org/10.1016/J.Jcp.2011.04.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., D P Grote, R H Cohen, and A Friedman. 2012. “Novel methods in the particle-in-cell accelerator code-framework warp.” Journal Paper. *Computational Science and Discovery* 5 (1): 014019 (20 pp.). - -.. raw:: html - -
- -.. raw:: html - -
- -Villasenor, J, and O Buneman. 1992. “Rigorous Charge Conservation for Local Electromagnetic-Field Solvers.” *Computer Physics Communications* 69 (2-3): 306–16. - -.. raw:: html - -
- -.. raw:: html - -
- -Vincenti, H., and J.-L. Vay. 2016. “Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver.” *Computer Physics Communications* 200 (March). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 147–67. https://doi.org/10.1016/j.cpc.2015.11.009. - -.. raw:: html - -
- -.. raw:: html - -
- -Yee, Ks. 1966. “Numerical Solution of Initial Boundary Value Problems Involving Maxwells Equations in Isotropic Media.” *Ieee Transactions on Antennas and Propagation* Ap14 (3): 302–7. - -.. raw:: html - -
- -.. raw:: html - -
diff --git a/Docs/source/usage/pwfa.rst b/Docs/source/usage/pwfa.rst index fc26f5db31a..5119184089c 100644 --- a/Docs/source/usage/pwfa.rst +++ b/Docs/source/usage/pwfa.rst @@ -44,7 +44,7 @@ Listed below are the key arguments and best-practices relevant for choosing the Finite Difference Time Domain ----------------------------- - For standard plasma wakefield configurations, it is possible to model the physics correctly using the Particle-In-Cell (PIC) Finite Difference Time Domain (FDTD) algorithms (:doc:`../theory/picsar_theory`). + For standard plasma wakefield configurations, it is possible to model the physics correctly using the :ref:`Particle-In-Cell (PIC) ` Finite Difference Time Domain (FDTD) algorithms. If the simulation contains localised extremely high intensity fields, however, numerical instabilities might arise, such as the numerical Cherenkov instability (:doc:`../theory/boosted_frame`). In that case, it is recommended to use the Pseudo Spectral Analytical Time Domain (PSATD) or the Pseudo-Spectral Time-Domain (PSTD) algorithms. In the example we are describing, it is sufficient to use FDTD. @@ -98,7 +98,7 @@ Time step The time step (:math:`dt`) is used to iterated over the main PIC loop and is computed by WarpX differently depending on the Maxwell field FDTD solvers used: - * **For Yee** is equal to the CFL parameter chosen in the input file (:doc:`parameters`) times the Courant–Friedrichs–Lewy condition (CFL) that follows the analytical expression in :doc:`../theory/picsar_theory` + * **For Yee** is equal to the CFL parameter chosen in the input file (:doc:`parameters`) times the Courant–Friedrichs–Lewy condition (CFL) that follows the analytical expression in :ref:`theory-pic` * **For CKC** is equal to CFL times the minimum between the boosted frame cell dimensions where CFL is chosen to be below unity and set an optimal trade-off between making the simulation faster and avoiding NCI and other spurious effects. From c1ec1024f633cd5ed11a7164cf653a8afc3103e3 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Sat, 16 Dec 2023 18:33:42 -0800 Subject: [PATCH 065/176] Docs: rename and spruce up boundary conditions section (#4537) * Docs: rename and spruce up boundary conditions section --- Docs/source/index.rst | 2 +- Docs/source/theory/PML.rst | 242 ---------------- Docs/source/theory/boundary_conditions.rst | 303 +++++++++++++++++++++ 3 files changed, 304 insertions(+), 243 deletions(-) delete mode 100644 Docs/source/theory/PML.rst create mode 100644 Docs/source/theory/boundary_conditions.rst diff --git a/Docs/source/index.rst b/Docs/source/index.rst index e5841da1f2d..9f2d2ff038e 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -109,7 +109,7 @@ Theory theory/intro theory/pic theory/amr - theory/PML + theory/boundary_conditions theory/boosted_frame theory/input_output theory/collisions diff --git a/Docs/source/theory/PML.rst b/Docs/source/theory/PML.rst deleted file mode 100644 index d60be931c5f..00000000000 --- a/Docs/source/theory/PML.rst +++ /dev/null @@ -1,242 +0,0 @@ -.. _theory-bc: - -Boundary conditions -=================== - -Open boundary condition for electromagnetic waves -------------------------------------------------- - -For the TE case, the original Berenger’s Perfectly Matched Layer (PML) writes - -.. math:: - - \begin{aligned} - \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = & \frac{\partial H_{z}}{\partial y}\label{PML_def_1} \\ - \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = & -\frac{\partial H_{z}}{\partial x}\label{PML_def_2} \\ - \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = & -\frac{\partial E_{y}}{\partial x}\label{PML_def_3} \\ - \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = & \frac{\partial E_{x}}{\partial y}\label{PML_def_4} \\ - H_{z} = & H_{zx}+H_{zy}\label{PML_def_5}\end{aligned} - -This can be generalized to - -.. math:: - - \begin{aligned} - \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = & \frac{c_{y}}{c}\frac{\partial H_{z}}{\partial y}+\overline{\sigma }_{y}H_{z}\label{APML_def_1} \\ - \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = & -\frac{c_{x}}{c}\frac{\partial H_{z}}{\partial x}+\overline{\sigma }_{x}H_{z}\label{APML_def_2} \\ - \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = & -\frac{c^{*}_{x}}{c}\frac{\partial E_{y}}{\partial x}+\overline{\sigma }_{x}^{*}E_{y}\label{APML_def_3} \\ - \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = & \frac{c^{*}_{y}}{c}\frac{\partial E_{x}}{\partial y}+\overline{\sigma }_{y}^{*}E_{x}\label{APML_def_4} \\ - H_{z} = & H_{zx}+H_{zy}\label{APML_def_5}\end{aligned} - -For :math:`c_{x}=c_{y}=c^{*}_{x}=c^{*}_{y}=c` and :math:`\overline{\sigma }_{x}=\overline{\sigma }_{y}=\overline{\sigma }_{x}^{*}=\overline{\sigma }_{y}^{*}=0`, -this system reduces to the Berenger PML medium, while adding the additional -constraint :math:`\sigma _{x}=\sigma _{y}=\sigma _{x}^{*}=\sigma _{y}^{*}=0` -leads to the system of Maxwell equations in vacuum. - -.. _theory-bc-propa-plane-wave: - -Propagation of a Plane Wave in an APML Medium -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -We consider a plane wave of magnitude (:math:`E_{0},H_{zx0},H_{zy0}`) -and pulsation :math:`\omega` propagating in the APML medium with an -angle :math:`\varphi` relative to the x axis - -.. math:: - - \begin{aligned} - E_{x} = & -E_{0}\sin \varphi e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_1} \\ - E_{y} = & E_{0}\cos \varphi e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_2} \\ - H_{zx} = & H_{zx0}e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_AMPL_def_3} \\ - H_{zy} = & H_{zy0}e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_4}\end{aligned} - -where :math:`\alpha` and\ :math:`\beta` are two complex constants to -be determined. - -Introducing (`[Plane_wave_APML_def_1] <#Plane_wave_APML_def_1>`__), (`[Plane_wave_APML_def_2] <#Plane_wave_APML_def_2>`__), -(`[Plane_wave_AMPL_def_3] <#Plane_wave_AMPL_def_3>`__) and (`[Plane_wave_APML_def_4] <#Plane_wave_APML_def_4>`__) -into (`[APML_def_1] <#APML_def_1>`__), (`[APML_def_2] <#APML_def_2>`__), (`[APML_def_3] <#APML_def_3>`__) -and (`[APML_def_4] <#APML_def_4>`__) gives - -.. math:: - - \begin{aligned} - \varepsilon _{0}E_{0}\sin \varphi -i\frac{\sigma _{y}}{\omega }E_{0}\sin \varphi = & \beta \frac{c_{y}}{c}\left( H_{zx0}+H_{zy0}\right) +i\frac{\overline{\sigma }_{y}}{\omega }\left( H_{zx0}+H_{zy0}\right) \label{Plane_wave_APML_1_1} \\ - \varepsilon _{0}E_{0}\cos \varphi -i\frac{\sigma _{x}}{\omega }E_{0}\cos \varphi = & \alpha \frac{c_{x}}{c}\left( H_{zx0}+H_{zy0}\right) -i\frac{\overline{\sigma }_{x}}{\omega }\left( H_{zx0}+H_{zy0}\right) \label{Plane_wave_APML_1_2} \\ - \mu _{0}H_{zx0}-i\frac{\sigma ^{*}_{x}}{\omega }H_{zx0} = & \alpha \frac{c^{*}_{x}}{c}E_{0}\cos \varphi -i\frac{\overline{\sigma }^{*}_{x}}{\omega }E_{0}\cos \varphi \label{Plane_wave_APML_1_3} \\ - \mu _{0}H_{zy0}-i\frac{\sigma ^{*}_{y}}{\omega }H_{zy0} = & \beta \frac{c^{*}_{y}}{c}E_{0}\sin \varphi +i\frac{\overline{\sigma }^{*}_{y}}{\omega }E_{0}\sin \varphi \label{Plane_wave_APML_1_4}\end{aligned} - -Defining :math:`Z=E_{0}/\left( H_{zx0}+H_{zy0}\right)` and using (`[Plane_wave_APML_1_1] <#Plane_wave_APML_1_1>`__) -and (`[Plane_wave_APML_1_2] <#Plane_wave_APML_1_2>`__), we get - -.. math:: - - \begin{aligned} - \beta = & \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi -i\frac{\overline{\sigma }_{y}}{\omega }\right] \frac{c}{c_{y}}\label{Plane_wave_APML_beta_of_g} \\ - \alpha = & \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi +i\frac{\overline{\sigma }_{x}}{\omega }\right] \frac{c}{c_{x}}\label{Plane_wave_APML_alpha_of_g}\end{aligned} - -Adding :math:`H_{zx0}` and :math:`H_{zy0}` from (`[Plane_wave_APML_1_3] <#Plane_wave_APML_1_3>`__) -and (`[Plane_wave_APML_1_4] <#Plane_wave_APML_1_4>`__) and substituting the expressions -for :math:`\alpha` and :math:`\beta` from (`[Plane_wave_APML_beta_of_g] <#Plane_wave_APML_beta_of_g>`__) -and (`[Plane_wave_APML_alpha_of_g] <#Plane_wave_APML_alpha_of_g>`__) yields - -.. math:: - - \begin{aligned} - \frac{1}{Z} = & \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi \frac{c^{*}_{x}}{c_{x}}+i\frac{\overline{\sigma }_{x}}{\omega }\frac{c^{*}_{x}}{c_{x}}-i\frac{\overline{\sigma }^{*}_{x}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{x}}{\omega }}\cos \varphi \nonumber \\ - + & \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi \frac{c^{*}_{y}}{c_{y}}-i\frac{\overline{\sigma }_{y}}{\omega }\frac{c^{*}_{y}}{c_{y}}+i\frac{\overline{\sigma }^{*}_{y}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{y}}{\omega }}\sin \varphi\end{aligned} - -If :math:`c_{x}=c^{*}_{x}`, :math:`c_{y}=c^{*}_{y}`, :math:`\overline{\sigma }_{x}=\overline{\sigma }^{*}_{x}`, :math:`\overline{\sigma }_{y}=\overline{\sigma }^{*}_{y}`, :math:`\frac{\sigma _{x}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{x}}{\mu _{0}}` and :math:`\frac{\sigma _{y}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{y}}{\mu _{0}}` then - -.. math:: - - \begin{aligned} - Z = & \pm \sqrt{\frac{\mu _{0}}{\varepsilon _{0}}}\label{APML_impedance}\end{aligned} - -which is the impedance of vacuum. Hence, like the PML, given some -restrictions on the parameters, the APML does not generate any reflection -at any angle and any frequency. As for the PML, this property is not -retained after discretization, as shown subsequently in this paper. - -Calling :math:`\psi` any component of the field and :math:`\psi _{0}` -its magnitude, we get from (`[Plane_wave_APML_def_1] <#Plane_wave_APML_def_1>`__), (`[Plane_wave_APML_beta_of_g] <#Plane_wave_APML_beta_of_g>`__), -(`[Plane_wave_APML_alpha_of_g] <#Plane_wave_APML_alpha_of_g>`__) and (`[APML_impedance] <#APML_impedance>`__) that - -.. math:: - - \label{Plane_wave_absorption} - \psi =\psi _{0}e^{i\omega \left( t\mp x\cos \varphi /c_{x}\mp y\sin \varphi /c_{y}\right) }e^{-\left( \pm \frac{\sigma _{x}\cos \varphi }{\varepsilon _{0}c_{x}}+\overline{\sigma }_{x}\frac{c}{c_{x}}\right) x}e^{-\left( \pm \frac{\sigma _{y}\sin \varphi }{\varepsilon _{0}c_{y}}+\overline{\sigma }_{y}\frac{c}{c_{y}}\right) y} - -We assume that we have an APML layer of thickness :math:`\delta` (measured -along :math:`x`) and that :math:`\sigma _{y}=\overline{\sigma }_{y}=0` -and :math:`c_{y}=c.` Using (`[Plane_wave_absorption] <#Plane_wave_absorption>`__), we determine -that the coefficient of reflection given by this layer is - -.. math:: - - \begin{aligned} - R_{APML}\left( \theta \right) = & e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}+\overline{\sigma }_{x}c/c_{x}\right) \delta }e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}-\overline{\sigma }_{x}c/c_{x}\right) \delta }\nonumber \\ - = & e^{-2\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}\right) \delta }\end{aligned} - -which happens to be the same as the PML theoretical coefficient of -reflection if we assume :math:`c_{x}=c`. Hence, it follows that for -the purpose of wave absorption, the term :math:`\overline{\sigma }_{x}` -seems to be of no interest. However, although this conclusion is true -at the infinitesimal limit, it does not hold for the discretized counterpart. - -Discretization -~~~~~~~~~~~~~~ - -.. math:: - - \begin{aligned} - \frac{E_x|^{n+1}_{j+1/2,k,l}-E_x|^{n}_{j+1/2,k,l}}{\Delta t} + \sigma_y \frac{E_x|^{n+1}_{j+1/2,k,l}+E_x|^{n}_{j+1/2,k,l}}{2} = & \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}}{\Delta y} \\ - % - \frac{E_y|^{n+1}_{j,k+1/2,l}-E_y|^{n}_{j,k+1/2,l}}{\Delta t} + \sigma_x \frac{E_y|^{n+1}_{j,k+1/2,l}+E_y|^{n}_{j,k+1/2,l}}{2} = & - \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}}{\Delta x} \\ - % - \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zx}|^{n}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_x \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zx}|^{n}_{j+1/2,k+1/2,l}}{2} = & - \frac{E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}}{\Delta x} \\ - % - \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zy}|^{n}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_y \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zy}|^{n}_{j+1/2,k+1/2,l}}{2} = & \frac{E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}}{\Delta y} \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & \left(\frac{1-\sigma_y \Delta t/2}{1+\sigma_y \Delta t/2}\right) E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t/\Delta y}{1+\sigma_y \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & \left(\frac{1-\sigma_x \Delta t/2}{1+\sigma_x \Delta t/2}\right) E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma_x \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & \left(\frac{1-\sigma^*_x \Delta t/2}{1+\sigma^*_x \Delta t/2}\right) H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma^*_x \Delta t/2} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & \left(\frac{1-\sigma^*_y \Delta t/2}{1+\sigma^*_y \Delta t/2}\right) H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t/\Delta y}{1+\sigma^*_y \Delta t/2} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y}\frac{c_y}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x}\frac{c_x}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x}\frac{c^*_x}{c} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y}\frac{c^*_y}{c} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - c_x = & c e^{-\sigma_x\Delta t} \frac{\sigma_x \Delta x}{1-e^{-\sigma_x\Delta t}} \\ - c_y = & c e^{-\sigma_y\Delta t} \frac{\sigma_y \Delta y}{1-e^{-\sigma_y\Delta t}} \\ - c^*_x = & c e^{-\sigma^*_x\Delta t} \frac{\sigma^*_x \Delta x}{1-e^{-\sigma^*_x\Delta t}} \\ - c^*_y = & c e^{-\sigma^*_y\Delta t} \frac{\sigma^*_y \Delta y}{1-e^{-\sigma^*_y\Delta t}}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} \left[ E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \right] \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} \left[ E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \right] \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} \left[ H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \right] \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} \left[ H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \right] \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. _theory-bc-pec: - -Perfect Electrical Conductor ----------------------------- - -This boundary can be used to model a dielectric or metallic surface. -For the electromagnetic solve, at PEC, the tangential electric field and the normal magnetic -field are set to 0. In the guard-cell region, the tangential electric field is set equal and -opposite to the respective field component in the mirror location across the PEC -boundary, and the normal electric field is set equal to the field component in the -mirror location in the domain across the PEC boundary. Similarly, the tangential -(and normal) magnetic field components are set equal (and opposite) to the respective -magnetic field components in the mirror locations across the PEC boundary. - -The PEC boundary condition also impacts the deposition of charge and current density. -On the boundary the charge density and parallel current density is set to zero. If -a reflecting boundary condition is used for the particles, density overlapping -with the PEC will be reflected back into the domain (for both charge and current -density). If absorbing boundaries are used, an image charge (equal weight but -opposite charge) is considered in the mirror location accross the boundary, and -the density from that charge is also deposited in the simulation domain. The -figure below shows the effect of this. The left boundary is absorbing while -the right boundary is reflecting. - -.. figure:: https://user-images.githubusercontent.com/40245517/221491318-b0a2bcbc-b04f-4b8c-8ec5-e9c92e55ee53.png - :alt: PEC boundary deposition - :width: 80% diff --git a/Docs/source/theory/boundary_conditions.rst b/Docs/source/theory/boundary_conditions.rst new file mode 100644 index 00000000000..395b072ccbe --- /dev/null +++ b/Docs/source/theory/boundary_conditions.rst @@ -0,0 +1,303 @@ +.. _theory-bc: + +Boundary conditions +=================== + +.. _theory-bc-PML: + +Perfectly Matched Layer: open boundary condition for electromagnetic waves +-------------------------------------------------------------------------- + +For the transverse electric (TE) case, the original Berenger’s Perfectly Matched Layer (PML) paper :cite:p:`bc-Berengerjcp94` writes + +.. math:: + \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = \frac{\partial H_{z}}{\partial y} + :label: PML_def_1 + +.. math:: + \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = -\frac{\partial H_{z}}{\partial x} + :label: PML_def_2 + +.. math:: + \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = -\frac{\partial E_{y}}{\partial x} + :label: PML_def_3 + +.. math:: + \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = \frac{\partial E_{x}}{\partial y} + :label: PML_def_4 + +.. math:: + H_{z} = H_{zx}+H_{zy} + :label: PML_def_5 + +This can be generalized to + +.. math:: + \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = \frac{c_{y}}{c}\frac{\partial H_{z}}{\partial y}+\overline{\sigma }_{y}H_{z} + :label: APML_def_1 + +.. math:: + \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = -\frac{c_{x}}{c}\frac{\partial H_{z}}{\partial x}+\overline{\sigma }_{x}H_{z} + :label: APML_def_2 + +.. math:: + \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = -\frac{c^{*}_{x}}{c}\frac{\partial E_{y}}{\partial x}+\overline{\sigma }_{x}^{*}E_{y} + :label: APML_def_3 + +.. math:: + \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = \frac{c^{*}_{y}}{c}\frac{\partial E_{x}}{\partial y}+\overline{\sigma }_{y}^{*}E_{x} + :label: APML_def_4 + +.. math:: + H_{z} = H_{zx}+H_{zy} + :label: APML_def_5 + +For :math:`c_{x}=c_{y}=c^{*}_{x}=c^{*}_{y}=c` and :math:`\overline{\sigma }_{x}=\overline{\sigma }_{y}=\overline{\sigma }_{x}^{*}=\overline{\sigma }_{y}^{*}=0`, +this system reduces to the Berenger PML medium, while adding the additional +constraint :math:`\sigma _{x}=\sigma _{y}=\sigma _{x}^{*}=\sigma _{y}^{*}=0` +leads to the system of Maxwell equations in vacuum. + +.. _theory-bc-propa-plane-wave: + +Propagation of a Plane Wave in an APML Medium +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We consider a plane wave of magnitude (:math:`E_{0},H_{zx0},H_{zy0}`) +and pulsation :math:`\omega` propagating in the APML medium with an +angle :math:`\varphi` relative to the x axis + +.. math:: + E_{x} = -E_{0}\sin \varphi \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_1 + +.. math:: + E_{y} = E_{0}\cos \varphi \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_2 + +.. math:: + H_{zx} = H_{zx0} \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_AMPL_def_3 + +.. math:: + H_{zy} = H_{zy0} \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_4 + +where :math:`\alpha` and :math:`\beta` are two complex constants to +be determined. + +Introducing Eqs. (:eq:`Plane_wave_APML_def_1`), (:eq:`Plane_wave_APML_def_2`), +(:eq:`Plane_wave_AMPL_def_3`) and (:eq:`Plane_wave_APML_def_4`) +into Eqs. (:eq:`APML_def_1`), (:eq:`APML_def_2`), (:eq:`APML_def_3`) +and (:eq:`APML_def_4`) gives + +.. math:: + \varepsilon _{0}E_{0}\sin \varphi -i\frac{\sigma _{y}}{\omega }E_{0}\sin \varphi = \beta \frac{c_{y}}{c}\left( H_{zx0}+H_{zy0}\right) +i\frac{\overline{\sigma }_{y}}{\omega }\left( H_{zx0}+H_{zy0}\right) + :label: Plane_wave_APML_1_1 + +.. math:: + \varepsilon _{0}E_{0}\cos \varphi -i\frac{\sigma _{x}}{\omega }E_{0}\cos \varphi = \alpha \frac{c_{x}}{c}\left( H_{zx0}+H_{zy0}\right) -i\frac{\overline{\sigma }_{x}}{\omega }\left( H_{zx0}+H_{zy0}\right) + :label: Plane_wave_APML_1_2 + +.. math:: + \mu _{0}H_{zx0}-i\frac{\sigma ^{*}_{x}}{\omega }H_{zx0} = \alpha \frac{c^{*}_{x}}{c}E_{0}\cos \varphi -i\frac{\overline{\sigma }^{*}_{x}}{\omega }E_{0}\cos \varphi + :label: Plane_wave_APML_1_3 + +.. math:: + \mu _{0}H_{zy0}-i\frac{\sigma ^{*}_{y}}{\omega }H_{zy0} = \beta \frac{c^{*}_{y}}{c}E_{0}\sin \varphi +i\frac{\overline{\sigma }^{*}_{y}}{\omega }E_{0}\sin \varphi + :label: Plane_wave_APML_1_4 + +Defining :math:`Z=E_{0}/\left( H_{zx0}+H_{zy0}\right)` and using Eqs. (:eq:`Plane_wave_APML_1_1`) +and (:eq:`Plane_wave_APML_1_2`), we get + +.. math:: + \beta = \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi -i\frac{\overline{\sigma }_{y}}{\omega }\right] \frac{c}{c_{y}} + :label: Plane_wave_APML_beta_of_g + +.. math:: + \alpha = \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi +i\frac{\overline{\sigma }_{x}}{\omega }\right] \frac{c}{c_{x}} + :label: Plane_wave_APML_alpha_of_g + +Adding :math:`H_{zx0}` and :math:`H_{zy0}` from Eqs. (:eq:`Plane_wave_APML_1_3`) +and (:eq:`Plane_wave_APML_1_4`) and substituting the expressions +for :math:`\alpha` and :math:`\beta` from Eqs. (:eq:`Plane_wave_APML_beta_of_g`) +and (:eq:`Plane_wave_APML_alpha_of_g`) yields + +.. math:: + + \begin{aligned} + \frac{1}{Z} & = \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi \frac{c^{*}_{x}}{c_{x}}+i\frac{\overline{\sigma }_{x}}{\omega }\frac{c^{*}_{x}}{c_{x}}-i\frac{\overline{\sigma }^{*}_{x}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{x}}{\omega }}\cos \varphi \nonumber + \\ + & + \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi \frac{c^{*}_{y}}{c_{y}}-i\frac{\overline{\sigma }_{y}}{\omega }\frac{c^{*}_{y}}{c_{y}}+i\frac{\overline{\sigma }^{*}_{y}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{y}}{\omega }}\sin \varphi + \end{aligned} + +If :math:`c_{x}=c^{*}_{x}`, :math:`c_{y}=c^{*}_{y}`, :math:`\overline{\sigma }_{x}=\overline{\sigma }^{*}_{x}`, :math:`\overline{\sigma }_{y}=\overline{\sigma }^{*}_{y}`, :math:`\frac{\sigma _{x}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{x}}{\mu _{0}}` and :math:`\frac{\sigma _{y}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{y}}{\mu _{0}}` then + +.. math:: + Z = \pm \sqrt{\frac{\mu _{0}}{\varepsilon _{0}}} + :label: APML_impedance + +which is the impedance of vacuum. Hence, like the PML, given some +restrictions on the parameters, the APML does not generate any reflection +at any angle and any frequency. As for the PML, this property is not +retained after discretization, as shown subsequently. + +Calling :math:`\psi` any component of the field and :math:`\psi _{0}` +its magnitude, we get from Eqs. (:eq:`Plane_wave_APML_def_1`), (:eq:`Plane_wave_APML_beta_of_g`), +(:eq:`Plane_wave_APML_alpha_of_g`) and (:eq:`APML_impedance`) that + +.. math:: + \psi =\psi _{0} \: e^{i\omega \left( t\mp x\cos \varphi /c_{x}\mp y\sin \varphi /c_{y}\right) }e^{-\left( \pm \frac{\sigma _{x}\cos \varphi }{\varepsilon _{0}c_{x}}+\overline{\sigma }_{x}\frac{c}{c_{x}}\right) x} e^{-\left( \pm \frac{\sigma _{y}\sin \varphi }{\varepsilon _{0}c_{y}}+\overline{\sigma }_{y}\frac{c}{c_{y}}\right) y}. + :label: Plane_wave_absorption + +We assume that we have an APML layer of thickness :math:`\delta` (measured +along :math:`x`) and that :math:`\sigma _{y}=\overline{\sigma }_{y}=0` +and :math:`c_{y}=c.` Using (:eq:`Plane_wave_absorption`), we determine +that the coefficient of reflection given by this layer is + +.. math:: + + \begin{aligned} + R_{\mathrm{APML}}\left( \theta \right) & = e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}+\overline{\sigma }_{x}c/c_{x}\right) \delta }e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}-\overline{\sigma }_{x}c/c_{x}\right) \delta },\nonumber + \\ + & = e^{-2\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}\right) \delta }, + \end{aligned} + +which happens to be the same as the PML theoretical coefficient of +reflection if we assume :math:`c_{x}=c`. Hence, it follows that for +the purpose of wave absorption, the term :math:`\overline{\sigma }_{x}` +seems to be of no interest. However, although this conclusion is true +at the infinitesimal limit, it does not hold for the discretized counterpart. + +Discretization +~~~~~~~~~~~~~~ + +In the following we set :math:`\varepsilon_0 = \mu_0 = 1`. We discretize Eqs. (:eq:`PML_def_1`), (:eq:`PML_def_2`), (:eq:`PML_def_3`), and (:eq:`PML_def_4`) to obtain + +.. math:: + \frac{E_x|^{n+1}_{j+1/2,k,l}-E_x|^{n}_{j+1/2,k,l}}{\Delta t} + \sigma_y \frac{E_x|^{n+1}_{j+1/2,k,l}+E_x|^{n}_{j+1/2,k,l}}{2} = \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}}{\Delta y} + +.. math:: + \frac{E_y|^{n+1}_{j,k+1/2,l}-E_y|^{n}_{j,k+1/2,l}}{\Delta t} + \sigma_x \frac{E_y|^{n+1}_{j,k+1/2,l}+E_y|^{n}_{j,k+1/2,l}}{2} = - \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}}{\Delta x} + +.. math:: + \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_x \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l}}{2} = - \frac{E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}}{\Delta x} + +.. math:: + \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_y \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l}}{2} = \frac{E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}}{\Delta y} + +and this can be solved to obtain the following leapfrog integration equations + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = \left(\frac{1-\sigma_y \Delta t/2}{1+\sigma_y \Delta t/2}\right) E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t/\Delta y}{1+\sigma_y \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = \left(\frac{1-\sigma_x \Delta t/2}{1+\sigma_x \Delta t/2}\right) E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma_x \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = \left(\frac{1-\sigma^*_x \Delta t/2}{1+\sigma^*_x \Delta t/2}\right) H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma^*_x \Delta t/2} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = \left(\frac{1-\sigma^*_y \Delta t/2}{1+\sigma^*_y \Delta t/2}\right) H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t/\Delta y}{1+\sigma^*_y \Delta t/2} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +If we account for higher order :math:`\Delta t` terms, a better approximation is given by + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +More generally, this becomes + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y}\frac{c_y}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x}\frac{c_x}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x}\frac{c^*_x}{c} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y}\frac{c^*_y}{c} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +If we set + +.. math:: + + \begin{aligned} + c_x & = c \: e^{-\sigma_x\Delta t} \frac{\sigma_x \Delta t}{1-e^{-\sigma_x\Delta t}} \\ + c_y & = c \: e^{-\sigma_y\Delta t} \frac{\sigma_y \Delta t}{1-e^{-\sigma_y\Delta t}} \\ + c^*_x & = c \: e^{-\sigma^*_x\Delta t} \frac{\sigma^*_x \Delta t}{1-e^{-\sigma^*_x\Delta t}} \\ + c^*_y & = c \: e^{-\sigma^*_y\Delta t} \frac{\sigma^*_y \Delta t}{1-e^{-\sigma^*_y\Delta t}}\end{aligned} + +then this becomes + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} \left[ E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \right] + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} \left[ E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \right] + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} \left[ H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \right] + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} \left[ H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \right] + \end{aligned} + +When the generalized conductivities are zero, the update equations are + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +as expected. + +.. _theory-bc-pec: + +Perfect Electrical Conductor +---------------------------- + +This boundary can be used to model a dielectric or metallic surface. +For the electromagnetic solve, at PEC, the tangential electric field and the normal magnetic +field are set to 0. In the guard-cell region, the tangential electric field is set equal and +opposite to the respective field component in the mirror location across the PEC +boundary, and the normal electric field is set equal to the field component in the +mirror location in the domain across the PEC boundary. Similarly, the tangential +(and normal) magnetic field components are set equal (and opposite) to the respective +magnetic field components in the mirror locations across the PEC boundary. + +The PEC boundary condition also impacts the deposition of charge and current density. +On the boundary the charge density and parallel current density is set to zero. If +a reflecting boundary condition is used for the particles, density overlapping +with the PEC will be reflected back into the domain (for both charge and current +density). If absorbing boundaries are used, an image charge (equal weight but +opposite charge) is considered in the mirror location accross the boundary, and +the density from that charge is also deposited in the simulation domain. :numref:`fig_PEC_boundary_deposition` +shows the effect of this. The left boundary is absorbing while +the right boundary is reflecting. + +.. _fig_PEC_boundary_deposition: + +.. figure:: https://user-images.githubusercontent.com/40245517/221491318-b0a2bcbc-b04f-4b8c-8ec5-e9c92e55ee53.png + :alt: Plot of PEC boundary current deposition showing current vs position along the ``x``-axis. + :width: 100% + + PEC boundary current deposition along the ``x``-axis. The left boundary is absorbing while the right boundary is reflecting. + +.. bibliography:: + :keyprefix: bc- From 9af087d70c5104b8c7150ef8a01aee66e885bf56 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Sun, 17 Dec 2023 11:13:11 -0800 Subject: [PATCH 066/176] Spruce up kinetic_fluid_hybrid_model.rst (#4534) --- Docs/source/refs.bib | 22 ++++++++++--------- .../theory/kinetic_fluid_hybrid_model.rst | 6 ++--- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 65545dfca87..8d001cf971e 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -60,20 +60,22 @@ @article{Turner2013 year = {2013} } -@article{winske2022hybrid, -archivePrefix = {arXiv}, -author = {D. Winske and Homa Karimabadi and Ari Le and N. Omidi and Vadim Roytershteyn and Adam Stanier}, -eprint = {2204.01676}, -journal = {arXiv}, -primaryClass = {physics.plasm-ph}, -title = {{Hybrid codes (massless electron fluid)}}, -year = {2022} +@Inbook{WinskeInBook2023, +author = {Winske, Dan and Karimabadi, Homa and Le, Ari Yitzchak and Omidi, Nojan Nick and Roytershteyn, Vadim and Stanier, Adam John}, +bookTitle = {Space and Astrophysical Plasma Simulation: Methods, Algorithms, and Applications}, +doi = {10.1007/978-3-031-11870-8_3}, +editor = {B{\"u}chner, J{\"o}rg}, +isbn = {978-3-031-11870-8}, +pages = {63--91}, +publisher = {Springer International Publishing}, +title = {{Hybrid-Kinetic Approach: Massless Electrons}}, +year = {2023} } -@incollection{NIELSON1976, +@incollection{Nielson1976, author = {Clair W. Nielson and H. Ralph Lewis}, booktitle = {Controlled Fusion}, -doi = {https://doi.org/10.1016/B978-0-12-460816-0.50015-4}, +doi = {10.1016/B978-0-12-460816-0.50015-4}, editor = {John Killeen}, issn = {0076-6860}, pages = {367--388}, diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index 69797352bee..55bce704fbb 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -19,7 +19,7 @@ of light. Many authors have described variations of the kinetic ion & fluid electron model, generally referred to as particle-fluid hybrid or just hybrid-PIC models. The implementation -in WarpX follows the outline from :cite:t:`c-winske2022hybrid`. +in WarpX follows the outline from :cite:t:`kfhm-WinskeInBook2023`. This description follows mostly from that reference. Model @@ -29,7 +29,7 @@ The basic justification for the hybrid model is that the system to which it is applied is dominated by ion kinetics, with ions moving much slower than electrons and photons. In this scenario two critical approximations can be made, namely, neutrality (:math:`n_e=n_i`) and the Maxwell-Ampere equation can be simplified by -neglecting the displacement current term :cite:p:`c-NIELSON1976`, giving, +neglecting the displacement current term :cite:p:`kfhm-Nielson1976`, giving, .. math:: @@ -168,4 +168,4 @@ The isothermal limit is given by :math:`\gamma = 1` while :math:`\gamma = 5/3` (default) produces the adiabatic limit. .. bibliography:: - :keyprefix: c- + :keyprefix: kfhm- From 8c307c712f26a47c6775eb45b30e3b71c023936f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 17 Dec 2023 13:07:34 -0800 Subject: [PATCH 067/176] Doc: CUDA 11.7+ (#4538) We know of a few compiler issues for C++17 CUDA code in CUDA <11.7 for a bit, updating Summit and general recommendations. --- Docs/source/install/dependencies.rst | 2 +- Tools/machines/summit-olcf/summit_warpx.profile.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index 7bd6c7dc8d5..a939db840da 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -18,7 +18,7 @@ Optional dependencies include: - for on-node accelerated compute *one of either*: - `OpenMP 3.1+ `__: for threaded CPU execution or - - `CUDA Toolkit 11.0+ (11.3+ recommended) `__: for Nvidia GPU support (see `matching host-compilers `_) or + - `CUDA Toolkit 11.7+ `__: for Nvidia GPU support (see `matching host-compilers `_) or - `ROCm 5.2+ (5.5+ recommended) `__: for AMD GPU support - `FFTW3 `_: for spectral solver (PSATD) support when running on CPU or SYCL diff --git a/Tools/machines/summit-olcf/summit_warpx.profile.example b/Tools/machines/summit-olcf/summit_warpx.profile.example index 5d88bb9aeed..e41521b5815 100644 --- a/Tools/machines/summit-olcf/summit_warpx.profile.example +++ b/Tools/machines/summit-olcf/summit_warpx.profile.example @@ -11,7 +11,7 @@ module load nano # required dependencies module load cmake/3.20.2 module load gcc/9.3.0 -module load cuda/11.3.1 +module load cuda/11.7.1 # optional: faster re-builds module load ccache From 244ec1cf43bab800d107440ea730389c8486a617 Mon Sep 17 00:00:00 2001 From: Andrei Berceanu Date: Sun, 17 Dec 2023 23:09:42 +0200 Subject: [PATCH 068/176] Add updated instructions for WarpX on Karolina (#4477) * Add updated instructions for WarpX on Karolina Use spack for installing all dependencies, and the new slurm scheduler. * Update Tools/machines/karolina-it4i/install_dependencies.sh End for loop Co-authored-by: Axel Huebl --- Docs/source/install/hpc/karolina.rst | 195 +++++------------- Tools/machines/karolina-it4i/cleanup.sh | 3 + .../karolina-it4i/install_cpu_dependencies.sh | 141 ------------- .../karolina-it4i/install_dependencies.sh | 48 +++++ .../karolina-it4i/install_gpu_dependencies.sh | 141 ------------- .../karolina_cpu_warpx.profile.example | 61 ------ .../machines/karolina-it4i/karolina_gpu.qsub | 29 --- .../karolina-it4i/karolina_gpu.sbatch | 39 ++++ .../karolina_gpu_warpx.profile.example | 65 ------ .../karolina_warpx.profile.example | 69 +++++++ .../karolina-it4i/spack-karolina-cuda.yaml | 80 +++++++ 11 files changed, 293 insertions(+), 578 deletions(-) create mode 100755 Tools/machines/karolina-it4i/cleanup.sh delete mode 100755 Tools/machines/karolina-it4i/install_cpu_dependencies.sh create mode 100755 Tools/machines/karolina-it4i/install_dependencies.sh delete mode 100755 Tools/machines/karolina-it4i/install_gpu_dependencies.sh delete mode 100644 Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example delete mode 100644 Tools/machines/karolina-it4i/karolina_gpu.qsub create mode 100644 Tools/machines/karolina-it4i/karolina_gpu.sbatch delete mode 100644 Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example create mode 100644 Tools/machines/karolina-it4i/karolina_warpx.profile.example create mode 100644 Tools/machines/karolina-it4i/spack-karolina-cuda.yaml diff --git a/Docs/source/install/hpc/karolina.rst b/Docs/source/install/hpc/karolina.rst index 7a8bf561986..fffb917df1c 100644 --- a/Docs/source/install/hpc/karolina.rst +++ b/Docs/source/install/hpc/karolina.rst @@ -12,83 +12,60 @@ Introduction If you are new to this system, **please see the following resources**: * `IT4I user guide `__ -* Batch system: `PBS `__ +* Batch system: `SLURM `__ * Jupyter service: not provided/documented (yet) * `Filesystems `__: * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (25GB default quota) - * ``/scratch/``: `production directory `__; very fast for parallel jobs (20TB default) + * ``/scratch/``: `production directory `__; very fast for parallel jobs (10TB default) + * ``/mnt/proj/``: per-project work directory, used for long term data storage (20TB default) .. _building-karolina-preparation: -Preparation ------------ - -Use the following commands to download the WarpX source code: +Installation +------------ -.. code-block:: bash +We show how to install from scratch all the dependencies using `Spack `__. - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx +For size reasons it is not advisable to install WarpX in the ``$HOME`` directory, it should be installed in the "work directory". For this purpose we set an environment variable ``$WORK`` with the path to the "work directory". On Karolina, you can run either on GPU nodes with fast A100 GPUs (recommended) or CPU nodes. -.. tab-set:: - - .. tab-item:: A100 GPUs - - We use system software modules, add environment hints and further dependencies via the file ``$HOME/karolina_gpu_warpx.profile``. - Create it now: - - .. code-block:: bash - - cp $HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example $HOME/karolina_gpu_warpx.profile - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example - :language: bash - - Edit the 2nd line of this script, which sets the ``export proj=""`` variable. - For example, if you are member of the project ``DD-23-83``, then run ``vi $HOME/karolina_gpu_warpx.profile``. - Enter the edit mode by typing ``i`` and edit line 2 to read: - - .. code-block:: bash +Profile file +^^^^^^^^^^^^ - export proj="DD-23-83" +One can use the pre-prepared ``karolina_warpx.profile`` script below, +which you can copy to ``${HOME}/karolina_warpx.profile``, edit as required and then ``source``. - Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. important:: + .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_warpx.profile.example + :language: bash + :caption: Copy the contents of this file to ``${HOME}/karolina_warpx.profile``. - Now, and as the first step on future logins to Karolina, activate these environment settings: +To have the environment activated on every login, add the following line to ``${HOME}/.bashrc``: - .. code-block:: bash - - source $HOME/karolina_gpu_warpx.profile - - Finally, since Karolina does not yet provide software modules for some of our dependencies, install them once: - - .. code-block:: bash +.. code-block:: bash - bash $HOME/src/warpx/Tools/machines/karolina-it4i/install_gpu_dependencies.sh - source $HOME/sw/karolina/gpu/venvs/warpx-gpu/bin/activate + source $HOME/karolina_warpx.profile - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down +To install the ``spack`` environment and Python packages: - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_gpu_dependencies.sh - :language: bash +.. code-block:: bash + bash $WORK/src/warpx/Tools/machines/karolina-it4i/install_dependencies.sh - .. tab-item:: CPU Nodes +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - CPU usage is documentation is TODO. + .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_dependencies.sh + :language: bash .. _building-karolina-compilation: @@ -98,117 +75,53 @@ Compilation Use the following :ref:`cmake commands ` to compile the application executable: -.. tab-set:: - - .. tab-item:: A100 GPUs - - .. code-block:: bash - - cd $HOME/src/warpx - rm -rf build_gpu - - cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_gpu -j 12 - - The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``. - Additionally, the following commands will install WarpX as a Python module: - - .. code-block:: bash - - rm -rf build_gpu_py - - cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_gpu_py -j 12 --target pip_install - - .. tab-item:: CPU Nodes - - .. code-block:: bash +.. code-block:: bash - cd $HOME/src/warpx - rm -rf build_cpu + cd $WORK/src/warpx + rm -rf build_gpu - cmake -S . -B build_cpu -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_cpu -j 12 + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu -j 48 - The WarpX application executables are now in ``$HOME/src/warpx/build_cpu/bin/``. - Additionally, the following commands will install WarpX as a Python module: +The WarpX application executables are now in ``$WORK/src/warpx/build_gpu/bin/``. +Additionally, the following commands will install WarpX as a Python module: - .. code-block:: bash +.. code-block:: bash - cd $HOME/src/warpx - rm -rf build_cpu_py + cd $WORK/src/warpx + rm -rf build_gpu_py - cmake -S . -B build_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_cpu_py -j 12 --target pip_install + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu_py -j 48 --target pip_install Now, you can :ref:`submit Karolina compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Karolina jobs (:ref:`example inputs `). For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch/``. -.. _building-karolina-update: - -Update WarpX & Dependencies ---------------------------- - -If you already installed WarpX in the past and want to update it, start by getting the latest source code: - -.. code-block:: bash - - cd $HOME/src/warpx - - # read the output of this command - does it look ok? - git status - - # get the latest WarpX source code - git fetch - git pull - - # read the output of these commands - do they look ok? - git status - git log # press q to exit - -And, if needed, - -- :ref:`update the karolina_gpu_warpx.profile or karolina_cpu_warpx.profile files `, -- log out and into the system, activate the now updated environment profile as usual, -- :ref:`execute the dependency install scripts `. - -As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX. - - .. _running-cpp-karolina: Running ------- -.. tab-set:: +The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#SBATCH --nodes=`` accordingly) on the supercomputer Karolina at IT4I. +This partition has up to `72 nodes `__. +Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors. - .. tab-item:: A100 (40GB) GPUs +Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``DD-23-83``. +Note that we run one MPI rank per GPU. - The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#PBS -l select=`` accordingly) on the supercomputer Karolina at IT4I. - This partition as up to `72 nodes `__. - Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors. +.. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.sbatch + :language: bash + :caption: You can copy this file from ``$WORK/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.sbatch``. - Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``DD-23-83``. - Note that we run one MPI rank per GPU. - - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.qsub - :language: bash - :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.qsub``. - - To run a simulation, copy the lines above to a file ``karolina_gpu.qsub`` and run - - .. code-block:: bash - - qsub karolina_gpu.qsub - - to submit the job. +To run a simulation, copy the lines above to a file ``karolina_gpu.sbatch`` and run +.. code-block:: bash - .. tab-item:: CPU Nodes + sbatch karolina_gpu.sbatch - CPU usage is documentation is TODO. +to submit the job. .. _post-processing-karolina: diff --git a/Tools/machines/karolina-it4i/cleanup.sh b/Tools/machines/karolina-it4i/cleanup.sh new file mode 100755 index 00000000000..b4d46edd1e4 --- /dev/null +++ b/Tools/machines/karolina-it4i/cleanup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +rm -rf ${HOME}/.spack ${WORK}/src/warpx ${WORK}/spack ${HOME}/.local/lib/python3.11 diff --git a/Tools/machines/karolina-it4i/install_cpu_dependencies.sh b/Tools/machines/karolina-it4i/install_cpu_dependencies.sh deleted file mode 100755 index 2695435738f..00000000000 --- a/Tools/machines/karolina-it4i/install_cpu_dependencies.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was karolina_cpu_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your karolina_cpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SW_DIR="${HOME}/sw/karolina/cpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# c-blosc (I/O compression) -if [ -d $HOME/src/c-blosc ] -then - cd $HOME/src/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc -fi -rm -rf $HOME/src/c-blosc-cpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-cpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-cpu-build - -# HDF5 -if [ -d $HOME/src/hdf5 ] -then - cd $HOME/src/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 -fi -rm -rf $HOME/src/hdf5-build -cmake -S $HOME/src/hdf5 -B $HOME/src/hdf5-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build $HOME/src/hdf5-build --target install --parallel 16 -rm -rf $HOME/src/hdf5-build - -# ADIOS2 -if [ -d $HOME/src/adios2 ] -then - cd $HOME/src/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 -fi -rm -rf $HOME/src/adios2-cpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-cpu-build --target install --parallel 16 -rm -rf $HOME/src/adios2-cpu-build - -# BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] -then - cd $HOME/src/blaspp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp -fi -rm -rf $HOME/src/blaspp-cpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-cpu-build -Duse_openmp=ON -Dcpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-cpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-cpu-build - -# LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] -then - cd $HOME/src/lapackpp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp -fi -rm -rf $HOME/src/lapackpp-cpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-cpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-cpu-build - - -# Python ###################################################################### -# -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade virtualenv -python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx-cpu -python3 -m venv ${SW_DIR}/venvs/warpx-cpu -source ${SW_DIR}/venvs/warpx-cpu/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade build -python3 -m pip install --upgrade packaging -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade setuptools -python3 -m pip install --upgrade cython -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -python3 -m pip install --upgrade matplotlib -python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/cpu -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh new file mode 100755 index 00000000000..c9c084d5aa7 --- /dev/null +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl, Andrei Berceanu +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ################################# +# +set -eu -o pipefail + +# Check: ########################################################## +# +# Was karolina_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your karolina_warpx.profile file!" + echo "Please edit its line 2 to continue!" + return +fi + +# download and activate spack +# this might take about ~ 1 hour +if [ ! -d "$WORK/spack" ] +then + git clone -c feature.manyFiles=true -b v0.21.0 https://github.com/spack/spack.git $WORK/spack + source $WORK/spack/share/spack/setup-env.sh + + # create and activate the spack environment + spack env create warpx-karolina-cuda $WORK/src/warpx/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml + spack env activate warpx-karolina-cuda + spack install +fi + +# Python ########################################################## +# +python -m pip install --user --upgrade pandas +python -m pip install --user --upgrade matplotlib +# optional +#python -m pip install --user --upgrade yt + +# install or update WarpX dependencies +python -m pip install --user --upgrade picmistandard==0.28.0 +python -m pip install --user --upgrade lasy + +# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +# python -m pip install --user --upgrade -r $WORK/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/install_gpu_dependencies.sh b/Tools/machines/karolina-it4i/install_gpu_dependencies.sh deleted file mode 100755 index a03bb008816..00000000000 --- a/Tools/machines/karolina-it4i/install_gpu_dependencies.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was karolina_gpu_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your karolina_gpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SW_DIR="${HOME}/sw/karolina/gpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# c-blosc (I/O compression) -if [ -d $HOME/src/c-blosc ] -then - cd $HOME/src/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc -fi -rm -rf $HOME/src/c-blosc-gpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-gpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-gpu-build - -# HDF5 -if [ -d $HOME/src/hdf5 ] -then - cd $HOME/src/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 -fi -rm -rf $HOME/src/hdf5-build -cmake -S $HOME/src/hdf5 -B $HOME/src/hdf5-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build $HOME/src/hdf5-build --target install --parallel 16 -rm -rf $HOME/src/hdf5-build - -# ADIOS2 -if [ -d $HOME/src/adios2 ] -then - cd $HOME/src/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 -fi -rm -rf $HOME/src/adios2-gpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-gpu-build --target install --parallel 12 -rm -rf $HOME/src/adios2-gpu-build - -# BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] -then - cd $HOME/src/blaspp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp -fi -rm -rf $HOME/src/blaspp-gpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-gpu-build --target install --parallel 12 -rm -rf $HOME/src/blaspp-gpu-build - -# LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] -then - cd $HOME/src/lapackpp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp -fi -rm -rf $HOME/src/lapackpp-gpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-gpu-build --target install --parallel 12 -rm -rf $HOME/src/lapackpp-gpu-build - - -# Python ###################################################################### -# -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade virtualenv -python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx-gpu -python3 -m venv ${SW_DIR}/venvs/warpx-gpu -source ${SW_DIR}/venvs/warpx-gpu/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade build -python3 -m pip install --upgrade packaging -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade setuptools -python3 -m pip install --upgrade cython -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -python3 -m pip install --upgrade matplotlib -python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch # CUDA 11.7 compatible wheel -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example deleted file mode 100644 index c0a3ed53ee3..00000000000 --- a/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example +++ /dev/null @@ -1,61 +0,0 @@ -# please set your project account -export proj="" # change me! - -# remembers the location of this script -export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi - -# required dependencies -module load GCCcore/11.3.0 -module load CMake/3.23.1-GCCcore-11.3.0 -module load OpenMPI/4.1.4-GCC-11.3.0 - -# optional: for QED support with detailed tables -module load Boost/1.79.0-GCC-11.3.0 - -# optional: for openPMD and PSATD+RZ support -module load OpenBLAS/0.3.20-GCC-11.3.0 -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/lapackpp-master:$CMAKE_PREFIX_PATH - -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH - -export PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2/bin:${PATH} -export PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3/bin:${PATH} - -# optional: CCache (not found) -#module load ccache - -# optional: for Python bindings or libEnsemble -module load Python/3.10.4-GCCcore-11.3.0-bare - -if [ -d "${HOME}/sw/karolina/cpu/venvs/warpx-cpu" ] -then - source ${HOME}/sw/karolina/cpu/venvs/warpx-cpu/bin/activate -fi - -# an alias to request an interactive batch node for one hour (TODO) -# for parallel execution, start on the batch node: srun -#alias getNode="salloc -N 1 --ntasks-per-node=4 -t 1:00:00 -q interactive -C gpu --gpu-bind=single:1 -c 32 -G 4 -A $proj" -# an alias to run a command on a batch node for up to 30min -# usage: runNode -#alias runNode="srun -N 1 --ntasks-per-node=4 -t 0:30:00 -q interactive -C gpu --gpu-bind=single:1 -c 32 -G 4 -A $proj" - -# optimize CUDA compilation for A100 -export AMREX_CUDA_ARCH=8.0 - -# optimize CPU microarchitecture for ... (TODO) -#export CXXFLAGS="-march=abc" -#export CFLAGS="-march=def" - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) diff --git a/Tools/machines/karolina-it4i/karolina_gpu.qsub b/Tools/machines/karolina-it4i/karolina_gpu.qsub deleted file mode 100644 index 274184ed1ca..00000000000 --- a/Tools/machines/karolina-it4i/karolina_gpu.qsub +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -l - -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Authors: Axel Huebl, Andrei Berceanu -# License: BSD-3-Clause-LBNL - -#PBS -q qgpu -#PBS -N WarpX -# Use two full nodes, 8 GPUs per node, 16 GPUs total -#PBS -l select=2:ncpus=128:ngpus=8:mpiprocs=8:ompthreads=16,walltime=00:10:00 -#PBS -A - -cd ${PBS_O_WORKDIR} - -# executable & inputs file or python interpreter & PICMI script here -EXE=./warpx.rz -INPUTS=inputs_rz - -# OpenMP threads per MPI rank -export OMP_NUM_THREADS=16 - -# run -mpirun -np ${PBS_NP} bash -c " - export CUDA_VISIBLE_DEVICES=\${OMPI_COMM_WORLD_LOCAL_RANK}; - ${EXE} ${INPUTS}" \ - > output.txt diff --git a/Tools/machines/karolina-it4i/karolina_gpu.sbatch b/Tools/machines/karolina-it4i/karolina_gpu.sbatch new file mode 100644 index 00000000000..94af0fa5b0e --- /dev/null +++ b/Tools/machines/karolina-it4i/karolina_gpu.sbatch @@ -0,0 +1,39 @@ +#!/bin/bash -l + +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Axel Huebl, Andrei Berceanu +# License: BSD-3-Clause-LBNL + +#SBATCH --account= +#SBATCH --partition=qgpu +#SBATCH --time=00:10:00 +#SBATCH --job-name=WarpX +#SBATCH --nodes=2 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=16 +#SBATCH --gpus-per-node=8 +#SBATCH --gpu-bind=single:1 + +#SBATCH --mail-type=ALL +# change me! +#SBATCH --mail-user=someone@example.com +#SBATCH --chdir=/scratch/project//it4i-/runs/warpx + +#SBATCH -o stdout_%j +#SBATCH -e stderr_%j + +# OpenMP threads per MPI rank +export OMP_NUM_THREADS=16 + +# set user rights to u=rwx;g=r-x;o=--- +umask 0027 + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx.rz +INPUTS=./inputs_rz + +# run +srun -K1 ${EXE} ${INPUTS} diff --git a/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example deleted file mode 100644 index 174598acaac..00000000000 --- a/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example +++ /dev/null @@ -1,65 +0,0 @@ -# please set your project account -export proj="" # change me! - -# remembers the location of this script -export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi - -# required dependencies -module purge -ml GCCcore/11.3.0 -ml CUDA/11.7.0 -ml OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 -ml CMake/3.23.1-GCCcore-11.3.0 - -# optional: for QED support with detailed tables -ml Boost/1.79.0-GCC-11.3.0 - -# optional: for openPMD and PSATD+RZ support -ml OpenBLAS/0.3.20-GCC-11.3.0 -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/lapackpp-master:$CMAKE_PREFIX_PATH - -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH - -export PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2/bin:${PATH} -export PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3/bin:${PATH} - -# optional: CCache (not found) -#ml ccache - -# optional: for Python bindings or libEnsemble -ml Python/3.10.4-GCCcore-11.3.0-bare - -if [ -d "${HOME}/sw/karolina/gpu/venvs/warpx-gpu" ] -then - source ${HOME}/sw/karolina/gpu/venvs/warpx-gpu/bin/activate -fi - -# an alias to request an interactive batch node for one hour (TODO) -# for parallel execution, start on the batch node: srun -alias getNode="qsub -q qgpu -A $proj -l select=1:ncpus=32:ngpus=4 -l walltime=1:00:00 -I" -# an alias to run a command on a batch node for up to 1hr -# usage: runNode -alias runNode='echo -e "#!/bin/bash\nmpirun -n 4 $1" | qsub -q qgpu -A $proj -l select=1:ncpus=32:ngpus=4 -l walltime=1:00:00' - -# optimize CUDA compilation for A100 -export AMREX_CUDA_ARCH=8.0 - -# optimize CPU microarchitecture for ... (TODO) -#export CXXFLAGS="-march=abc" -#export CFLAGS="-march=def" - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/karolina-it4i/karolina_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_warpx.profile.example new file mode 100644 index 00000000000..5fbd144300f --- /dev/null +++ b/Tools/machines/karolina-it4i/karolina_warpx.profile.example @@ -0,0 +1,69 @@ +# please set your project account, ie DD-N-N +export proj="" # change me! + +# Name and Path of this Script ################### (DO NOT change!) +export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) + +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file!" + echo "Please edit its line 2 to continue!" + return +fi + +# set env variable storing the path to the work directory +# please check if your project ID belongs to proj1, proj2, proj3 etc +export WORK="/mnt/proj/${proj,,}/${USER}" # change me! +mkdir -p WORK + +# clone warpx +# you can also clone your own fork here, eg git@github.com:/WarpX.git +if [ ! -d "$WORK/src/warpx" ] +then + git clone https://github.com/ECP-WarpX/WarpX.git $WORK/src/warpx +fi + +# load required modules +module purge +module load OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 + +# download and activate spack +if [ ! -d "$WORK/spack" ] +then + echo "WARNING: Spack installation not detected, please run install_dependencies.sh" +else + # activate the spack environment + source $WORK/spack/share/spack/setup-env.sh + spack env activate warpx-karolina-cuda +fi + +# Text Editor for Tools ########################## (edit this line) +# examples: "nano", "vim", "emacs -nw" or without terminal: "gedit" +#export EDITOR="nano" # change me! + +# allocate an interactive shell for one hour +# usage: getNode 2 # allocates two interactive nodes (default: 1) +function getNode() { + if [ -z "$1" ] ; then + numNodes=1 + else + numNodes=$1 + fi + export OMP_NUM_THREADS=16 + srun --time=1:00:00 --nodes=$numNodes --ntasks=$((8 * $numNodes)) --ntasks-per-node=8 --cpus-per-task=16 --exclusive --gpus-per-node=8 -p qgpu -A $proj --pty bash +} + +# Environment ##################################################### +# optimize CUDA compilation for A100 +export AMREX_CUDA_ARCH="8.0" +export SCRATCH="/scratch/project/${proj,,}/${USER}" + +# optimize CPU microarchitecture for AMD EPYC 7763 (zen3) +export CFLAGS="-march=znver3" +export CXXFLAGS="-march=znver3" + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml new file mode 100644 index 00000000000..1cb6a4ac209 --- /dev/null +++ b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml @@ -0,0 +1,80 @@ +spack: + specs: +# Karolina's openssl version is deprecated + - openssl certs=system + - pkgconfig + - ccache + - cmake@3.26.5 + - cuda@11.7.0 + - openmpi@4.1.4 +atomics + - fftw + - hdf5@1.14.0 + - adios2@2.9.2 ~mgard + - blaspp + - lapackpp + - boost@1.81.0 +program_options +atomic ~python + - python@3.11.6 + - py-pip + - py-setuptools + - py-wheel + - py-cython + - py-mpi4py + - py-numpy@1.24.2 + - openpmd-api@0.15.2 +python + - py-periodictable@1.5.0 + - py-h5py +# optional +# - py-libensemble +nlopt + + packages: + openssh: + externals: + - spec: openssh@7.4p1 + prefix: /usr + buildable: False + cuda: + externals: + - spec: cuda@11.7.0 + modules: + - CUDA/11.7.0 + buildable: False + mpi: + buildable: False + openmpi: + externals: + - spec: openmpi@4.1.4 +atomics +cuda %gcc@11.3.0 + modules: + - OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 + libfabric: + externals: + - spec: libfabric@1.15.1 %gcc@11.3.0 + modules: + - libfabric/1.15.1-GCCcore-11.3.0 + buildable: False + all: + target: [zen3] + compiler: [gcc@11.3.0] + variants: +mpi ~fortran +cuda cuda_arch=80 + providers: + mpi: [openmpi@4.1.4] + cuda: [cuda@11.7.0] + + compilers: + - compiler: + modules: [GCCcore/11.3.0] + operating_system: centos7 + paths: + cc: /apps/all/GCCcore/11.3.0/bin/gcc + cxx: /apps/all/GCCcore/11.3.0/bin/g++ + f77: /apps/all/GCCcore/11.3.0/bin/gfortran + fc: /apps/all/GCCcore/11.3.0/bin/gfortran + spec: gcc@=11.3.0 + target: x86_64 + flags: {} + environment: {} + extra_rpaths: [] + + view: true + concretizer: + reuse: false + unify: true From e8f54f2986033ef6e981861760aea1414ef53d47 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 23:49:25 -0800 Subject: [PATCH 069/176] [pre-commit.ci] pre-commit autoupdate (#4539) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/isort: 5.13.0 → 5.13.2](https://github.com/pycqa/isort/compare/5.13.0...5.13.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 21577b0604a..35c38ac1ad6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,7 +76,7 @@ repos: # Sorts Python imports according to PEP8 # https://www.python.org/dev/peps/pep-0008/#imports - repo: https://github.com/pycqa/isort - rev: 5.13.0 + rev: 5.13.2 hooks: - id: isort name: isort (python) From e460e608a303b642023b2c8418bf5ae2ec64a5f5 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 19 Dec 2023 12:11:04 +0100 Subject: [PATCH 070/176] AMReX/pyAMReX/PICSAR: Weekly Update (#4541) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index ff50edc428e..2874b9dea7c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach ecaa46d0be4b5c79b8806e48e3469000d8bb7252 && cd - + cd ../amrex && git checkout --detach ef38229189e3213f992a2e89dbe304fb49db9287 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index ceba435d251..40eb6cd4802 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = ecaa46d0be4b5c79b8806e48e3469000d8bb7252 +branch = ef38229189e3213f992a2e89dbe304fb49db9287 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 2934c686c4d..7fa7162891a 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = ecaa46d0be4b5c79b8806e48e3469000d8bb7252 +branch = ef38229189e3213f992a2e89dbe304fb49db9287 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 2b044f3a485..bc04048e2fe 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "ecaa46d0be4b5c79b8806e48e3469000d8bb7252" +set(WarpX_amrex_branch "ef38229189e3213f992a2e89dbe304fb49db9287" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 57e378c26a0..9bb9cf427cd 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "0b2d3f6b160991834534164b7391080fabc48ddb" +set(WarpX_pyamrex_branch "91bbbbab13a0651781b6830f64d362dc3d15a3ea" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 8da087155ba..8ddca959861 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach ecaa46d0be4b5c79b8806e48e3469000d8bb7252 && cd - +cd amrex && git checkout --detach ef38229189e3213f992a2e89dbe304fb49db9287 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From b41869c8b9f344f1a00d4ee6ac43761376b8c59e Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 20 Dec 2023 19:14:33 +0100 Subject: [PATCH 071/176] WarpXAMReXInit: move include directive from header to cpp file (#4543) --- Source/Initialization/WarpXAMReXInit.H | 2 -- Source/Initialization/WarpXAMReXInit.cpp | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Source/Initialization/WarpXAMReXInit.H b/Source/Initialization/WarpXAMReXInit.H index 50c90d40ae6..613250b5c2f 100644 --- a/Source/Initialization/WarpXAMReXInit.H +++ b/Source/Initialization/WarpXAMReXInit.H @@ -7,8 +7,6 @@ #ifndef WARPX_AMREX_INIT_H_ #define WARPX_AMREX_INIT_H_ -#include - #include namespace warpx::initialization diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index c153595604f..d1dd0bbe90b 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -8,6 +8,7 @@ #include "Initialization/WarpXAMReXInit.H" #include +#include #include #include From 8885be5345f8a82b0886bbbfc18d1d5893067b62 Mon Sep 17 00:00:00 2001 From: "S. Eric Clark" <25495882+clarkse@users.noreply.github.com> Date: Wed, 20 Dec 2023 12:05:01 -0800 Subject: [PATCH 072/176] Adding user defined keywords in Hybrid PIC Python class to allow for addition of constants when defining functions for resistivity and current. (#4550) --- Python/pywarpx/picmi.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 168ad8508f6..6e85bb97a97 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1159,10 +1159,21 @@ def __init__(self, grid, Te=None, n0=None, gamma=None, self.Jy_external_function = Jy_external_function self.Jz_external_function = Jz_external_function + # Handle keyword arguments used in expressions + self.user_defined_kw = {} + for k in list(kw.keys()): + self.user_defined_kw[k] = kw[k] + del kw[k] + self.handle_init(kw) def initialize_inputs(self): + # Add the user defined keywords to my_constants + # The keywords are mangled if there is a conflicting variable already + # defined in my_constants with the same name but different value. + self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) + self.grid.initialize_inputs() pywarpx.algo.maxwell_solver = self.method @@ -1172,17 +1183,21 @@ def initialize_inputs(self): pywarpx.hybridpicmodel.gamma = self.gamma pywarpx.hybridpicmodel.n_floor = self.n_floor pywarpx.hybridpicmodel.__setattr__( - 'plasma_resistivity(rho)', self.plasma_resistivity + 'plasma_resistivity(rho)', + pywarpx.my_constants.mangle_expression(self.plasma_resistivity, self.mangle_dict) ) pywarpx.hybridpicmodel.substeps = self.substeps pywarpx.hybridpicmodel.__setattr__( - 'Jx_external_grid_function(x,y,z,t)', self.Jx_external_function + 'Jx_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jx_external_function, self.mangle_dict) ) pywarpx.hybridpicmodel.__setattr__( - 'Jy_external_grid_function(x,y,z,t)', self.Jy_external_function + 'Jy_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jy_external_function, self.mangle_dict) ) pywarpx.hybridpicmodel.__setattr__( - 'Jz_external_grid_function(x,y,z,t)', self.Jz_external_function + 'Jz_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jz_external_function, self.mangle_dict) ) From 522cbf0ae35040c1d6e287c55678a4504f319448 Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 20 Dec 2023 17:32:36 -0800 Subject: [PATCH 073/176] Clean up doc to improve readability (#4555) --- Docs/source/usage/parameters.rst | 90 ++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index ec360135f6f..16f4b58e0e6 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -713,29 +713,48 @@ Particle initialization * ``SingleParticle``: Inject a single macroparticle. This requires the additional parameters: - ``.single_particle_pos`` (`3 doubles`, particle 3D position [meter]) - ``.single_particle_u`` (`3 doubles`, particle 3D normalized momentum, i.e. :math:`\gamma \beta`) - ``.single_particle_weight`` ( `double`, macroparticle weight, i.e. number of physical particles it represents) + + * ``.single_particle_pos`` (`3 doubles`, particle 3D position [meter]) + + * ``.single_particle_u`` (`3 doubles`, particle 3D normalized momentum, i.e. :math:`\gamma \beta`) + + * ``.single_particle_weight`` ( `double`, macroparticle weight, i.e. number of physical particles it represents) * ``MultipleParticles``: Inject multiple macroparticles. This requires the additional parameters: - ``.multiple_particles_pos_x`` (list of `doubles`, X positions of the particles [meter]) - ``.multiple_particles_pos_y`` (list of `doubles`, Y positions of the particles [meter]) - ``.multiple_particles_pos_z`` (list of `doubles`, Z positions of the particles [meter]) - ``.multiple_particles_ux`` (list of `doubles`, X normalized momenta of the particles, i.e. :math:`\gamma \beta_x`) - ``.multiple_particles_uy`` (list of `doubles`, Y normalized momenta of the particles, i.e. :math:`\gamma \beta_y`) - ``.multiple_particles_uz`` (list of `doubles`, Z normalized momenta of the particles, i.e. :math:`\gamma \beta_z`) - ``.multiple_particles_weight`` (list of `doubles`, macroparticle weights, i.e. number of physical particles each represents) + + * ``.multiple_particles_pos_x`` (list of `doubles`, X positions of the particles [meter]) + + * ``.multiple_particles_pos_y`` (list of `doubles`, Y positions of the particles [meter]) + + * ``.multiple_particles_pos_z`` (list of `doubles`, Z positions of the particles [meter]) + + * ``.multiple_particles_ux`` (list of `doubles`, X normalized momenta of the particles, i.e. :math:`\gamma \beta_x`) + + * ``.multiple_particles_uy`` (list of `doubles`, Y normalized momenta of the particles, i.e. :math:`\gamma \beta_y`) + + * ``.multiple_particles_uz`` (list of `doubles`, Z normalized momenta of the particles, i.e. :math:`\gamma \beta_z`) + + * ``.multiple_particles_weight`` (list of `doubles`, macroparticle weights, i.e. number of physical particles each represents) * ``gaussian_beam``: Inject particle beam with gaussian distribution in space in all directions. This requires additional parameters: - ``.q_tot`` (beam charge), - ``.npart`` (number of particles in the beam), - ``.x/y/z_m`` (average position in `x/y/z`), - ``.x/y/z_rms`` (standard deviation in `x/y/z`), - ``.x/y/z_cut`` (optional, particles with ``abs(x-x_m) > x_cut*x_rms`` are not injected, same for y and z. ``.q_tot`` is the charge of the un-cut beam, so that cutting the distribution is likely to result in a lower total charge), - and optional arguments ``.do_symmetrize`` (whether to - symmetrize the beam) and ``.symmetrization_order`` (order of symmetrization, default is 4, can be 4 or 8). + + * ``.q_tot`` (beam charge), + + * ``.npart`` (number of particles in the beam), + + * ``.x/y/z_m`` (average position in `x/y/z`), + + * ``.x/y/z_rms`` (standard deviation in `x/y/z`), + + There are additional optional parameters: + + * ``.x/y/z_cut`` (optional, particles with ``abs(x-x_m) > x_cut*x_rms`` are not injected, same for y and z. ``.q_tot`` is the charge of the un-cut beam, so that cutting the distribution is likely to result in a lower total charge), + * ``.do_symmetrize`` (optional, whether to symmetrize the beam) + + * ``.symmetrization_order`` (order of symmetrization, default is 4, can be 4 or 8). + If ``.do_symmetrize`` is 0, no symmetrization occurs. If ``.do_symmetrize`` is 1, then the beam is symmetrized according to the value of ``.symmetrization_order``. If set to 4, symmetrization is in the x and y direction, (x,y) (-x,y) (x,-y) (-x,-y). @@ -743,11 +762,17 @@ Particle initialization * ``external_file``: Inject macroparticles with properties (mass, charge, position, and momentum - :math:`\gamma \beta m c`) read from an external openPMD file. With it users can specify the additional arguments: - ``.injection_file`` (`string`) openPMD file name and - ``.charge`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. - ``.mass`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. - ``.z_shift`` (`double`) optional (default is no shift) when set this value will be added to the longitudinal, ``z``, position of the particles. - ``.impose_t_lab_from_file`` (`bool`) optional (default is false) only read if warpx.gamma_boost > 1., it allows to set t_lab for the Lorentz Transform as being the time stored in the openPMD file. + + * ``.injection_file`` (`string`) openPMD file name and + + * ``.charge`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. + + * ``.mass`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. + + * ``.z_shift`` (`double`) optional (default is no shift) when set this value will be added to the longitudinal, ``z``, position of the particles. + + * ``.impose_t_lab_from_file`` (`bool`) optional (default is false) only read if warpx.gamma_boost > 1., it allows to set t_lab for the Lorentz Transform as being the time stored in the openPMD file. + Warning: ``q_tot!=0`` is not supported with the ``external_file`` injection style. If a value is provided, it is ignored and no re-scaling is done. The external file must include the species ``openPMD::Record`` labeled ``position`` and ``momentum`` (`double` arrays), with dimensionality and units set via ``openPMD::setUnitDimension`` and ``setUnitSI``. If the external file also contains ``openPMD::Records`` for ``mass`` and ``charge`` (constant `double` scalars) then the species will use these, unless overwritten in the input file (see ``.mass``, ``.charge`` or ``.species_type``). @@ -756,13 +781,20 @@ Particle initialization * ``NFluxPerCell``: Continuously inject a flux of macroparticles from a planar surface. This requires the additional parameters: - ``.flux_profile`` (see the description of this parameter further below) - ``.surface_flux_pos`` (`double`, location of the injection plane [meter]) - ``.flux_normal_axis`` (`x`, `y`, or `z` for 3D, `x` or `z` for 2D, or `r`, `t`, or `z` for RZ. When `flux_normal_axis` is `r` or `t`, the `x` and `y` components of the user-specified momentum distribution are interpreted as the `r` and `t` components respectively) - ``.flux_direction`` (`-1` or `+1`, direction of flux relative to the plane) - ``.num_particles_per_cell`` (`double`) - ``.flux_tmin`` (`double`, Optional time at which the flux will be turned on. Ignored when negative.) - ``.flux_tmax`` (`double`, Optional time at which the flux will be turned off. Ignored when negative.) + + * ``.flux_profile`` (see the description of this parameter further below) + + * ``.surface_flux_pos`` (`double`, location of the injection plane [meter]) + + * ``.flux_normal_axis`` (`x`, `y`, or `z` for 3D, `x` or `z` for 2D, or `r`, `t`, or `z` for RZ. When `flux_normal_axis` is `r` or `t`, the `x` and `y` components of the user-specified momentum distribution are interpreted as the `r` and `t` components respectively) + + * ``.flux_direction`` (`-1` or `+1`, direction of flux relative to the plane) + + * ``.num_particles_per_cell`` (`double`) + + * ``.flux_tmin`` (`double`, Optional time at which the flux will be turned on. Ignored when negative.) + + * ``.flux_tmax`` (`double`, Optional time at which the flux will be turned off. Ignored when negative.) * ``none``: Do not inject macro-particles (for example, in a simulation that starts with neutral, ionizable atoms, one may want to create the electrons species -- where ionized electrons can be stored later on -- without injecting electron macro-particles). From bf3303cc8502a044823eb7722d4151b46b351f05 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 21 Dec 2023 02:50:02 -0800 Subject: [PATCH 074/176] add hybrid-PIC PoP reference to docs (#4551) --- Docs/source/highlights.rst | 7 +++++- Docs/source/refs.bib | 25 ++++++++++--------- .../theory/kinetic_fluid_hybrid_model.rst | 4 +-- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 420bb376d67..98b0d85391e 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -149,7 +149,12 @@ Please see :ref:`this section `. Nuclear Fusion - Magnetically Confined Plasmas ********************************************** -#. Nicks, B. S., Putvinski, S. and Tajima, T. +#. Nicks B. S., Putvinski S. and Tajima T. **Stabilization of the Alfvén-ion cyclotron instability through short plasmas: Fully kinetic simulations in a high-beta regime**. Physics of Plasmas **30**, 102108, 2023. `DOI:10.1063/5.0163889 `__ + +#. Groenewald R. E., Veksler A., Ceccherini F., Necas A., Nicks B. S., Barnes D. C., Tajima T. and Dettrick S. A. + **Accelerated kinetic model for global macro stability studies of high-beta fusion reactors**. + Physics of Plasmas **30**, 122508, 2023. + `DOI:10.1063/5.0178288 `__ diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 8d001cf971e..1aab3fb8645 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -60,18 +60,6 @@ @article{Turner2013 year = {2013} } -@Inbook{WinskeInBook2023, -author = {Winske, Dan and Karimabadi, Homa and Le, Ari Yitzchak and Omidi, Nojan Nick and Roytershteyn, Vadim and Stanier, Adam John}, -bookTitle = {Space and Astrophysical Plasma Simulation: Methods, Algorithms, and Applications}, -doi = {10.1007/978-3-031-11870-8_3}, -editor = {B{\"u}chner, J{\"o}rg}, -isbn = {978-3-031-11870-8}, -pages = {63--91}, -publisher = {Springer International Publishing}, -title = {{Hybrid-Kinetic Approach: Massless Electrons}}, -year = {2023} -} - @incollection{Nielson1976, author = {Clair W. Nielson and H. Ralph Lewis}, booktitle = {Controlled Fusion}, @@ -288,3 +276,16 @@ @Inbook{VanLeerBookChapter1997 url = {https://doi.org/10.1007/978-3-642-60543-7_3}, year = {1997} } + +@article{Groenewald2023, +author = {Groenewald, R. E. and Veksler, A. and Ceccherini, F. and Necas, A. and Nicks, B. S. and Barnes, D. C. and Tajima, T. and Dettrick, S. A.}, +title = "{Accelerated kinetic model for global macro stability studies of high-beta fusion reactors}", +journal = {Physics of Plasmas}, +volume = {30}, +number = {12}, +pages = {122508}, +year = {2023}, +month = {12}, +issn = {1070-664X}, +doi = {10.1063/5.0178288}, +} diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index 55bce704fbb..854c4d5ada1 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -18,8 +18,8 @@ has to resolve the electron Debye length and CFL-condition based on the speed of light. Many authors have described variations of the kinetic ion & fluid electron model, -generally referred to as particle-fluid hybrid or just hybrid-PIC models. The implementation -in WarpX follows the outline from :cite:t:`kfhm-WinskeInBook2023`. +generally referred to as particle-fluid hybrid or just hybrid-PIC models. The +implementation in WarpX is described in detail in :cite:t:`kfhm-Groenewald2023`. This description follows mostly from that reference. Model From 1603278776b63cff41560fba2770af16d60bbae3 Mon Sep 17 00:00:00 2001 From: David Grote Date: Thu, 21 Dec 2023 11:12:03 -0800 Subject: [PATCH 075/176] Fix reference (#4552) --- Docs/source/developers/profiling.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/developers/profiling.rst b/Docs/source/developers/profiling.rst index a1c610eed4a..5acea786920 100644 --- a/Docs/source/developers/profiling.rst +++ b/Docs/source/developers/profiling.rst @@ -121,7 +121,7 @@ Perlmutter Example """""""""""""""""" Example on how to create traces on a multi-GPU system that uses the Slurm scheduler (e.g., NERSC's Perlmutter system). -You can either run this on an interactive node or use the Slurm batch script header :ref:`documented here `. +You can either run this on an interactive node or use the Slurm batch script header :ref:`documented here `. .. code-block:: bash From 3fd5b5f61f320e8a49c5b71fd20cfd23ef29b769 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 22 Dec 2023 09:02:03 +0100 Subject: [PATCH 076/176] AMReX/pyAMReX/PICSAR: Weekly Update (#4559) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 2874b9dea7c..29a434dc579 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach ef38229189e3213f992a2e89dbe304fb49db9287 && cd - + cd ../amrex && git checkout --detach 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 40eb6cd4802..0566184848b 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = ef38229189e3213f992a2e89dbe304fb49db9287 +branch = 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 7fa7162891a..b00916463c7 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = ef38229189e3213f992a2e89dbe304fb49db9287 +branch = 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index bc04048e2fe..886ee9c951f 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "ef38229189e3213f992a2e89dbe304fb49db9287" +set(WarpX_amrex_branch "75571e2dcbf2417529c5ed8e24113580e8e1f3f1" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 9bb9cf427cd..2ad63917965 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "91bbbbab13a0651781b6830f64d362dc3d15a3ea" +set(WarpX_pyamrex_branch "eb24d03fac522d36fb27d20c6e026d8b59dfa908" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 8ddca959861..deb4939da51 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach ef38229189e3213f992a2e89dbe304fb49db9287 && cd - +cd amrex && git checkout --detach 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From bb42b2867e230f683d71b86842f41254da1ae6d3 Mon Sep 17 00:00:00 2001 From: Andrei Berceanu Date: Fri, 22 Dec 2023 10:03:25 +0200 Subject: [PATCH 077/176] Add updated instructions for WarpX on Karolina (#4545) * Add updated instructions for WarpX on Karolina Use spack for installing all dependencies, and the new slurm scheduler. * Update Tools/machines/karolina-it4i/install_dependencies.sh fix short circuit Co-authored-by: Axel Huebl --------- Co-authored-by: Axel Huebl --- .../karolina-it4i/install_dependencies.sh | 20 +++++++++++++++---- .../karolina-it4i/karolina_gpu.sbatch | 1 + .../karolina_warpx.profile.example | 14 +++++-------- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh index c9c084d5aa7..0435b5e2926 100755 --- a/Tools/machines/karolina-it4i/install_dependencies.sh +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -26,13 +26,25 @@ if [ ! -d "$WORK/spack" ] then git clone -c feature.manyFiles=true -b v0.21.0 https://github.com/spack/spack.git $WORK/spack source $WORK/spack/share/spack/setup-env.sh +else + # If the directory exists, checkout v0.21.0 branch + cd $WORK/spack + git checkout v0.21.0 + git pull origin v0.21.0 + source $WORK/spack/share/spack/setup-env.sh + + # Delete spack env if present + spack env deactivate || true + spack env rm -y warpx-karolina-cuda || true - # create and activate the spack environment - spack env create warpx-karolina-cuda $WORK/src/warpx/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml - spack env activate warpx-karolina-cuda - spack install + cd - fi +# create and activate the spack environment +spack env create warpx-karolina-cuda $WORK/src/warpx/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml +spack env activate warpx-karolina-cuda +spack install + # Python ########################################################## # python -m pip install --user --upgrade pandas diff --git a/Tools/machines/karolina-it4i/karolina_gpu.sbatch b/Tools/machines/karolina-it4i/karolina_gpu.sbatch index 94af0fa5b0e..6171ff03abc 100644 --- a/Tools/machines/karolina-it4i/karolina_gpu.sbatch +++ b/Tools/machines/karolina-it4i/karolina_gpu.sbatch @@ -27,6 +27,7 @@ # OpenMP threads per MPI rank export OMP_NUM_THREADS=16 +export SRUN_CPUS_PER_TASK=16 # set user rights to u=rwx;g=r-x;o=--- umask 0027 diff --git a/Tools/machines/karolina-it4i/karolina_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_warpx.profile.example index 5fbd144300f..1a4eda19a23 100644 --- a/Tools/machines/karolina-it4i/karolina_warpx.profile.example +++ b/Tools/machines/karolina-it4i/karolina_warpx.profile.example @@ -26,15 +26,11 @@ fi module purge module load OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 -# download and activate spack -if [ ! -d "$WORK/spack" ] -then - echo "WARNING: Spack installation not detected, please run install_dependencies.sh" -else - # activate the spack environment - source $WORK/spack/share/spack/setup-env.sh - spack env activate warpx-karolina-cuda -fi +source $WORK/spack/share/spack/setup-env.sh && spack env activate warpx-karolina-cuda && { + echo "Spack environment 'warpx-karolina-cuda' activated successfully." +} || { + echo "Failed to activate Spack environment 'warpx-karolina-cuda'. Please run install_dependencies.sh." +} # Text Editor for Tools ########################## (edit this line) # examples: "nano", "vim", "emacs -nw" or without terminal: "gedit" From f05cf71596b2cb23671fe4a879b2637484e5f35b Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Fri, 22 Dec 2023 07:38:28 -0800 Subject: [PATCH 078/176] Add DSMC module (#4125) * moved MCCProcess to ScatteringProcess * initial commit of DSMC - not yet working * partially working DSMC implementation * reuse kinematic functionality * fix bug in setting new particle ids * code cleanup * move DSMC filter functions to different file * fixed issue with some negative id particles not being removed - needs clean up * add a redistribute call after resampling to remove negative id particles * update CMakeLists * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * code cleanup and build fixes * only print resampling message if `verbose` is on * properly use target species velocity * add DSMC CI test * fix CI issues * WIP update collision documentation * increase rtol for DSMC checksum test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * `import os` in analysis script for DSMC test * reduce tolerance for DSMC test further * added `get_charge_density` to pybind `WarpXParticleContainer` and exposed useful fields.py functionality directly for a mf * fix outdated libwarpx call in CI test * fix clang-tidy and CI issues * use `double` in calculation of collision energy * fix clang-tidy issues * apply code review changes Co-authored-by: RemiLehe * apply second round of code review changes * use `amrex::Scan::PrefixSum` to calculate offsets from collision mask; fix for GPU executability --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: RemiLehe --- Docs/source/refs.bib | 28 ++ Docs/source/theory/collisions.rst | 90 +++++- Docs/source/usage/parameters.rst | 37 +-- Docs/source/usage/python.rst | 2 + .../capacitive_discharge/PICMI_inputs_1d.py | 133 ++++++-- .../capacitive_discharge/analysis_dsmc.py | 60 ++++ Python/pywarpx/picmi.py | 43 +++ .../benchmarks_json/Python_dsmc_1d.json | 27 ++ Regression/WarpX-tests.ini | 19 ++ .../BackgroundMCC/BackgroundMCCCollision.H | 12 +- .../BackgroundMCC/BackgroundMCCCollision.cpp | 24 +- .../Collision/BackgroundMCC/CMakeLists.txt | 1 - .../BackgroundMCC/ImpactIonization.H | 14 +- .../Collision/BackgroundMCC/Make.package | 1 - .../BinaryCollision/BinaryCollisionUtils.H | 88 +++++ .../Collision/BinaryCollision/CMakeLists.txt | 2 + .../BinaryCollision/DSMC/CMakeLists.txt | 7 + .../DSMC/CollisionFilterFunc.H | 232 ++++++++++++++ .../Collision/BinaryCollision/DSMC/DSMC.H | 85 +++++ .../Collision/BinaryCollision/DSMC/DSMC.cpp | 300 ++++++++++++++++++ .../BinaryCollision/DSMC/Make.package | 3 + .../DSMC/SplitAndScatterFunc.H | 165 ++++++++++ .../Collision/BinaryCollision/Make.package | 2 + .../NuclearFusion/SingleNuclearFusionEvent.H | 79 +---- .../BinaryCollision/ParticleCreationFunc.cpp | 66 ++-- Source/Particles/Collision/CMakeLists.txt | 1 + .../Particles/Collision/CollisionHandler.cpp | 4 + Source/Particles/Collision/Make.package | 1 + .../MCCProcess.H => ScatteringProcess.H} | 34 +- .../MCCProcess.cpp => ScatteringProcess.cpp} | 32 +- 30 files changed, 1363 insertions(+), 229 deletions(-) create mode 100755 Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py create mode 100644 Regression/Checksum/benchmarks_json/Python_dsmc_1d.json create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/Make.package create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H rename Source/Particles/Collision/{BackgroundMCC/MCCProcess.H => ScatteringProcess.H} (81%) rename Source/Particles/Collision/{BackgroundMCC/MCCProcess.cpp => ScatteringProcess.cpp} (83%) diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 1aab3fb8645..4b1cd1bd1a1 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -289,3 +289,31 @@ @article{Groenewald2023 issn = {1070-664X}, doi = {10.1063/5.0178288}, } + +@article{Perez2012, + author = {Pérez, F. and Gremillet, L. and Decoster, A. and Drouin, M. and Lefebvre, E.}, + title = "{Improved modeling of relativistic collisions and collisional ionization in particle-in-cell codes}", + journal = {Physics of Plasmas}, + volume = {19}, + number = {8}, + pages = {083104}, + year = {2012}, + month = {08}, + issn = {1070-664X}, + doi = {10.1063/1.4742167}, + url = {https://doi.org/10.1063/1.4742167}, + eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4742167/13891570/083104\_1\_online.pdf}, +} + +@article{Higginson2019, + doi = {10.1016/j.jcp.2019.03.020}, + url = {https://doi.org/10.1016/j.jcp.2019.03.020}, + year = {2019}, + month = jul, + publisher = {Elsevier {BV}}, + volume = {388}, + pages = {439--453}, + author = {Drew Pitney Higginson and Anthony Link and Andrea Schmidt}, + title = {A pairwise nuclear fusion algorithm for weighted particle-in-cell plasma simulations}, + journal = {Journal of Computational Physics} +} diff --git a/Docs/source/theory/collisions.rst b/Docs/source/theory/collisions.rst index 3f24cd8f331..52e36521125 100644 --- a/Docs/source/theory/collisions.rst +++ b/Docs/source/theory/collisions.rst @@ -3,18 +3,36 @@ Collisions ========== -Monte Carlo Collisions ----------------------- - -The Monte Carlo collisions (MCC) module can be found in *Source/Particles/Collisions/BackgroundMCC/*. -Several types of collisions between simulation particles and a neutral background gas are supported including elastic scattering, back scattering, charge exchange, excitation collisions and impact ionization. -An instance of the class :cpp:class:`MCCProcess` is created for each type of collision included in a simulation. This class saves information about the type of collision and the collision cross-section as a function of energy. - -The so-called null collision strategy is used in order to minimize the computational burden of the MCC module. -This strategy is standard in PIC-MCC and a detailed description can be found elsewhere, for example in :cite:t:`b-Birdsall1991`. -In short the maximum collision probability is found over a sensible range of energies and is used to pre-select the appropriate number of macroparticles for collision consideration. Only these pre-selected particles are then individually considered for a collision based on their energy and the cross-sections of all the different collisional processes included. - -The MCC implementation assumes that the background neutral particles are **thermal**, and are moving at non-relativistic velocities in the lab frame. For each simulation particle considered for a collision, a velocity vector for a neutral particle is randomly chosen. The particle velocity is then boosted to the stationary frame of the neutral through a Galilean transformation. The energy of the collision is calculated using the particle utility function, ``ParticleUtils::getCollisionEnergy()``, as +WarpX includes several different models to capture collisional processes +including collisions between kinetic particles (Coulomb collisions, DSMC, +nuclear fusion) as well as collisions between kinetic particles and a fixed +(i.e. non-evolving) background species (MCC, background stopping). + +.. _theory-collisions-mcc: + +Background Monte Carlo Collisions (MCC) +--------------------------------------- + +Several types of collisions between simulation particles and a neutral +background gas are supported including elastic scattering, back scattering, +charge exchange, excitation collisions and impact ionization. + +The so-called null collision strategy is used in order to minimize the +computational burden of the MCC module. This strategy is standard in PIC-MCC and +a detailed description can be found elsewhere, for example in :cite:t:`b-Birdsall1991`. +In short the maximum collision probability is found over a sensible range of +energies and is used to pre-select the appropriate number of macroparticles for +collision consideration. Only these pre-selected particles are then individually +considered for a collision based on their energy and the cross-sections of all +the different collisional processes included. + +The MCC implementation assumes that the background neutral particles are **thermal**, +and are moving at non-relativistic velocities in the lab frame. For each +simulation particle considered for a collision, a velocity vector for a neutral +particle is randomly chosen given the user specified neutral temperature. The +particle velocity is then boosted to the stationary frame of the neutral through +a Galilean transformation. The energy of the collision is calculated using the +particle utility function, ``ParticleUtils::getCollisionEnergy()``, as .. math:: @@ -23,19 +41,55 @@ The MCC implementation assumes that the background neutral particles are **therm &= \frac{2Mmu^2}{M + m + \sqrt{M^2+m^2+2\gamma mM}}\frac{1}{\gamma + 1} \end{aligned} -where :math:`u` is the speed of the particle as tracked in WarpX (i.e. :math:`u = \gamma v` with :math:`v` the particle speed), while :math:`m` and :math:`M` are the rest masses of the simulation and background species, respectively. The Lorentz factor is defined in the usual way, :math:`\gamma = \sqrt{1 + u^2/c^2}`. Note that if :math:`\gamma\to1` the above expression clearly reduces to the classical equation :math:`E_{coll} = \frac{1}{2}\frac{Mm}{M+m} u^2`. The collision cross-sections for all scattering processes are evaluated at the energy as calculated above. +where :math:`u` is the speed of the particle as tracked in WarpX (i.e. +:math:`u = \gamma v` with :math:`v` the particle speed), while :math:`m` and +:math:`M` are the rest masses of the simulation and background species, +respectively. The Lorentz factor is defined in the usual way, +:math:`\gamma \def \sqrt{1 + u^2/c^2}`. Note that if :math:`\gamma\to1` the above +expression reduces to the classical equation +:math:`E_{coll} = \frac{1}{2}\frac{Mm}{M+m} u^2`. The collision cross-sections +for all scattering processes are evaluated at the energy as calculated above. Once a particle is selected for a specific collision process, that process determines how the particle is scattered as outlined below. +.. _theory-collisions-dsmc: + +Direct Simulation Monte Carlo (DSMC) +------------------------------------ + +The algorithm by which binary collisions are treated is outlined below. The +description assumes collisions between different species. + +1. Particles from both species are sorted by grid-cells. +2. The order of the particles in each cell is shuffled. +3. Within each cell, particles are paired to form collision partners. Particles + of the species with fewer members in a given cell is split in half so that + each particle has exactly one partner of the other species. +4. Each collision pair is considered for a collision using the same logic as in + the MCC description above. +5. Particles that are chosen for collision are scattered according to the + selected collision process. + +Scattering processes +-------------------- + Charge exchange ^^^^^^^^^^^^^^^ -This is the simplest scattering process. Under charge exchange the simulation particle's velocity is simply replaced with the sampled velocity of the neutral particle. Charge exchange usually has a cooling effect on the ions in the simulation by exchanging energetic ions for lower energy neutrals. +This process can occur when an ion and neutral (of the same species) collide +and results in the exchange of an electron. The ion velocity is simply replaced +with the neutral velocity and vice-versa. Elastic scattering ^^^^^^^^^^^^^^^^^^ -This scattering process as well as the ones below that relate to it, are all performed in the center-of-momentum (COM) frame. Designating the COM velocity of the particle as :math:`\vec{u}_c` and its labframe velocity as :math:`\vec{u}_l`, the transformation from lab frame to COM frame is done with a general Lorentz boost (see function ``ParticleUtils::doLorentzTransform()``): +The ``elastic`` option uses isotropic scattering, i.e., with a differential +cross section that is independent of angle. +This scattering process as well as the ones below that relate to it, are all +performed in the center-of-momentum (COM) frame. Designating the COM velocity of +the particle as :math:`\vec{u}_c` and its labframe velocity as :math:`\vec{u}_l`, +the transformation from lab frame to COM frame is done with a general Lorentz +boost (see function ``ParticleUtils::doLorentzTransform()``): .. math:: \begin{bmatrix} @@ -74,6 +128,12 @@ Excitation The process is also the same as for elastic scattering except the excitation energy cost is subtracted from the particle energy. This is done by reducing the velocity before a scattering angle is chosen. +Benchmarks +---------- + +See the :ref:`MCC example ` for a benchmark of the MCC +implementation against literature results. + Particle cooling due to elastic collisions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 16f4b58e0e6..3ff9c4e62ee 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1713,6 +1713,7 @@ Collision models ---------------- WarpX provides several particle collision models, using varying degrees of approximation. +Details about the collision models can be found in the :ref:`theory section `. * ``collisions.collision_names`` (`strings`, separated by spaces) The name of each collision type. @@ -1722,29 +1723,29 @@ WarpX provides several particle collision models, using varying degrees of appro * ``.type`` (`string`) optional The type of collision. The types implemented are: - - ``pairwisecoulomb`` for pairwise Coulomb collisions, the default if unspecified. + - ``pairwisecoulomb`` for pair-wise Coulomb collisions, the default if unspecified. This provides a pair-wise relativistic elastic Monte Carlo binary Coulomb collision model, - following the algorithm given by `Perez et al. (Phys. Plasmas 19, 083104, 2012) `__. + following the algorithm given by :cite:t:`param-Perez2012`. When the RZ mode is used, `warpx.n_rz_azimuthal_modes` must be set to 1 at the moment, since the current implementation of the collision module assumes axisymmetry. - ``nuclearfusion`` for fusion reactions. - This implements the pair-wise fusion model by `Higginson et al. (JCP 388, 439-453, 2019) `__. + This implements the pair-wise fusion model by :cite:t:`param-Higginson2019`. Currently, WarpX supports deuterium-deuterium, deuterium-tritium, deuterium-helium and proton-boron fusion. When initializing the reactant and product species, you need to use ``species_type`` (see the documentation for this parameter), so that WarpX can identify the type of reaction to use. (e.g. ``.species_type = 'deuterium'``) + - ``dsmc`` for pair-wise, non-Coulomb collisions between kinetic species. + This is a "direct simulation Monte Carlo" treatment of collisions between + kinetic species. See :ref:`DSMC section `. - ``background_mcc`` for collisions between particles and a neutral background. This is a relativistic Monte Carlo treatment for particles colliding - with a neutral background gas. The implementation follows the so-called - null collision strategy discussed for example in `Birdsall (IEEE Transactions on - Plasma Science, vol. 19, no. 2, pp. 65-85, 1991) `_. - See also :ref:`collisions section `. + with a neutral background gas. See :ref:`MCC section `. - ``background_stopping`` for slowing of ions due to collisions with electrons or ions. This implements the approximate formulae as derived in Introduction to Plasma Physics, from Goldston and Rutherford, section 14.2. * ``.species`` (`strings`) - If using ``pairwisecoulomb`` or ``nuclearfusion``, this should be the name(s) of the species, + If using ``dsmc``, ``pairwisecoulomb`` or ``nuclearfusion``, this should be the name(s) of the species, between which the collision will be considered. (Provide only one name for intra-species collisions.) If using ``background_mcc`` or ``background_stopping`` type this should be the name of the species for which collisions with a background will be included. @@ -1767,7 +1768,7 @@ WarpX provides several particle collision models, using varying degrees of appro :math:`A` is the mass number. If this is not provided, or if a non-positive value is provided, a Coulomb logarithm will be computed automatically according to the algorithm in - `Perez et al. (Phys. Plasmas 19, 083104, 2012) `__. + :cite:t:`param-Perez2012`. * ``.fusion_multiplier`` (`float`) optional. Only for ``nuclearfusion``. @@ -1778,8 +1779,8 @@ WarpX provides several particle collision models, using varying degrees of appro More specifically, in a fusion reaction between two macroparticles with weight ``w_1`` and ``w_2``, the weight of the product macroparticles will be ``min(w_1,w_2)/fusion_multiplier``. (And the weights of the reactant macroparticles are reduced correspondingly after the reaction.) - See `Higginson et al. (JCP 388, 439-453, 2019) `__ - for more details. The default value of ``fusion_multiplier`` is 1. + See :cite:t:`param-Higginson2019` for more details. + The default value of ``fusion_multiplier`` is 1. * ``.fusion_probability_threshold`` (`float`) optional. Only for ``nuclearfusion``. @@ -1849,25 +1850,21 @@ WarpX provides several particle collision models, using varying degrees of appro where :math:`\beta` is the term on the r.h.s except :math:`W_b`. * ``.scattering_processes`` (`strings` separated by spaces) - Only for ``background_mcc``. The MCC scattering processes that should be + Only for ``dsmc`` and ``background_mcc``. The scattering processes that should be included. Available options are ``elastic``, ``back`` & ``charge_exchange`` for ions and ``elastic``, ``excitationX`` & ``ionization`` for electrons. - The ``elastic`` option uses hard-sphere scattering, with a differential - cross section that is independent of angle. - With ``charge_exchange``, the ion velocity is replaced with the neutral - velocity, chosen from a Maxwellian based on the value of - ``.background_temperature``. Multiple excitation events can be included for electrons corresponding to excitation to different levels, the ``X`` above can be changed to a unique identifier for each excitation process. For each scattering process specified - a path to a cross-section data file must also be given. We use + a path to a cross-section data file must also be given. We use ```` as a placeholder going forward. * ``._cross_section`` (`string`) - Only for ``background_mcc``. Path to the file containing cross-section data + Only for ``dsmc`` and ``background_mcc``. Path to the file containing cross-section data for the given scattering processes. The cross-section file must have exactly 2 columns of data, the first containing equally spaced energies in eV and the - second the corresponding cross-section in :math:`m^2`. + second the corresponding cross-section in :math:`m^2`. The energy column should + represent the kinetic energy of the colliding particles in the center-of-mass frame. * ``._energy`` (`float`) Only for ``background_mcc``. If the scattering process is either diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 2cf7982c69c..448800f0d4d 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -124,6 +124,8 @@ Other operations related to particles: .. autoclass:: pywarpx.picmi.CoulombCollisions +.. autoclass:: pywarpx.picmi.DSMCCollisions + .. autoclass:: pywarpx.picmi.MCCCollisions .. autoclass:: pywarpx.picmi.FieldIonization diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index 0c6f9828543..dd970cc15ea 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# --- Copyright 2021 Modern Electron +# --- Copyright 2021 Modern Electron (DSMC test added in 2023 by TAE Technologies) # --- Monte-Carlo Collision script to reproduce the benchmark tests from # --- Turner et al. (2013) - https://doi.org/10.1063/1.4775084 @@ -46,9 +46,7 @@ def initialize_inputs(self): WarpX parser. """ # grab the boundary potentials from the grid object - self.right_voltage = ( - self.grid.potential_zmax.replace('sin', 'np.sin').replace('pi', 'np.pi') - ) + self.right_voltage = self.grid.potential_zmax # set WarpX boundary potentials to None since we will handle it # ourselves in this solver @@ -108,8 +106,12 @@ def solve(self): calculating phi from rho.""" left_voltage = 0.0 - t = self.sim.extension.warpx.gett_new(0) - right_voltage = eval(self.right_voltage) + right_voltage = eval( + self.right_voltage, { + 't': self.sim.extension.warpx.gett_new(0), + 'sin': np.sin, 'pi': np.pi + } + ) # Construct b vector rho = -self.rho_data / constants.ep0 @@ -158,11 +160,12 @@ class CapacitiveDischargeExample(object): # Time (in seconds) between diagnostic evaluations diag_interval = 32 / freq - def __init__(self, n=0, test=False, pythonsolver=False): + def __init__(self, n=0, test=False, pythonsolver=False, dsmc=False): """Get input parameters for the specific case (n) desired.""" self.n = n self.test = test self.pythonsolver = pythonsolver + self.dsmc = dsmc # Case specific input parameters self.voltage = f"{self.voltage[n]}*sin(2*pi*{self.freq:.5e}*t)" @@ -236,13 +239,26 @@ def setup_run(self): rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)]*3, ) ) + if self.dsmc: + self.neutrals = picmi.Species( + particle_type='He', name='neutrals', + charge=0, mass=self.m_ion, + warpx_reflection_model_zlo=1.0, + warpx_reflection_model_zhi=1.0, + warpx_do_resampling=True, + warpx_resampling_trigger_max_avg_ppc=int(self.seed_nppc*1.5), + initial_distribution=picmi.UniformDistribution( + density=self.gas_density, + rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)]*3, + ) + ) ####################################################################### # Collision initialization # ####################################################################### cross_sec_direc = '../../../../warpx-data/MCC_cross_sections/He/' - mcc_electrons = picmi.MCCCollisions( + electron_colls = picmi.MCCCollisions( name='coll_elec', species=self.electrons, background_density=self.gas_density, @@ -269,24 +285,26 @@ def setup_run(self): } ) - mcc_ions = picmi.MCCCollisions( - name='coll_ion', - species=self.ions, - background_density=self.gas_density, - background_temperature=self.gas_temp, - ndt=self.mcc_subcycling_steps, - scattering_processes={ - 'elastic' : { - 'cross_section' : cross_sec_direc+'ion_scattering.dat' - }, - 'back' : { - 'cross_section' : cross_sec_direc+'ion_back_scatter.dat' - }, - # 'charge_exchange' : { - # 'cross_section' : cross_sec_direc+'charge_exchange.dat' - # } - } - ) + ion_scattering_processes={ + 'elastic': {'cross_section': cross_sec_direc+'ion_scattering.dat'}, + 'back': {'cross_section': cross_sec_direc+'ion_back_scatter.dat'}, + # 'charge_exchange': {'cross_section': cross_sec_direc+'charge_exchange.dat'} + } + if self.dsmc: + ion_colls = picmi.DSMCCollisions( + name='coll_ion', + species=[self.ions, self.neutrals], + ndt=5, scattering_processes=ion_scattering_processes + ) + else: + ion_colls = picmi.MCCCollisions( + name='coll_ion', + species=self.ions, + background_density=self.gas_density, + background_temperature=self.gas_temp, + ndt=self.mcc_subcycling_steps, + scattering_processes=ion_scattering_processes + ) ####################################################################### # Initialize simulation # @@ -296,8 +314,7 @@ def setup_run(self): solver=self.solver, time_step_size=self.dt, max_steps=self.max_steps, - warpx_collisions=[mcc_electrons, mcc_ions], - warpx_load_balance_intervals=self.max_steps//5000, + warpx_collisions=[electron_colls, ion_colls], verbose=self.test ) self.solver.sim = self.sim @@ -314,18 +331,36 @@ def setup_run(self): n_macroparticle_per_cell=[self.seed_nppc], grid=self.grid ) ) + if self.dsmc: + self.sim.add_species( + self.neutrals, + layout = picmi.GriddedLayout( + n_macroparticle_per_cell=[self.seed_nppc//2], grid=self.grid + ) + ) + self.solver.sim_ext = self.sim.extension + + if self.dsmc: + # Periodically reset neutral density to starting temperature + callbacks.installbeforecollisions(self.rethermalize_neutrals) ####################################################################### # Add diagnostics for the CI test to be happy # ####################################################################### - if self.pythonsolver: - file_prefix = 'Python_background_mcc_1d_plt' + if self.dsmc: + file_prefix = 'Python_dsmc_1d_plt' else: - file_prefix = 'Python_background_mcc_1d_tridiag_plt' - + if self.pythonsolver: + file_prefix = 'Python_background_mcc_1d_plt' + else: + file_prefix = 'Python_background_mcc_1d_tridiag_plt' + + species = [self.electrons, self.ions] + if self.dsmc: + species.append(self.neutrals) particle_diag = picmi.ParticleDiagnostic( - species=[self.electrons, self.ions], + species=species, name='diag1', period=0, write_dir='.', @@ -342,6 +377,30 @@ def setup_run(self): self.sim.add_diagnostic(particle_diag) self.sim.add_diagnostic(field_diag) + def rethermalize_neutrals(self): + # When using DSMC the neutral temperature will change due to collisions + # with the ions. This is not captured in the original MCC test. + # Re-thermalize the neutrals every 1000 steps + step = self.sim.extension.warpx.getistep(lev=0) + if step % 1000 != 10: + return + + if not hasattr(self, 'neutral_cont'): + self.neutral_cont = particle_containers.ParticleContainerWrapper( + self.neutrals.name + ) + + ux_arrays = self.neutral_cont.uxp + uy_arrays = self.neutral_cont.uyp + uz_arrays = self.neutral_cont.uzp + + vel_std = np.sqrt(constants.kb * self.gas_temp / self.m_ion) + for ii in range(len(ux_arrays)): + nps = len(ux_arrays[ii]) + ux_arrays[ii][:] = vel_std * np.random.normal(size=nps) + uy_arrays[ii][:] = vel_std * np.random.normal(size=nps) + uz_arrays[ii][:] = vel_std * np.random.normal(size=nps) + def _get_rho_ions(self): # deposit the ion density in rho_fp he_ions_wrapper = particle_containers.ParticleContainerWrapper('he_ions') @@ -389,11 +448,17 @@ def run_sim(self): '--pythonsolver', help='toggle whether to use the Python level solver', action='store_true' ) +parser.add_argument( + '--dsmc', help='toggle whether to use DSMC for ions in place of MCC', + action='store_true' +) args, left = parser.parse_known_args() sys.argv = sys.argv[:1]+left if args.n < 1 or args.n > 4: raise AttributeError('Test number must be an integer from 1 to 4.') -run = CapacitiveDischargeExample(n=args.n-1, test=args.test, pythonsolver=args.pythonsolver) +run = CapacitiveDischargeExample( + n=args.n-1, test=args.test, pythonsolver=args.pythonsolver, dsmc=args.dsmc +) run.run_sim() diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py new file mode 100755 index 00000000000..df773dd9deb --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +# 2023 TAE Technologies + +import os +import sys + +import numpy as np + +sys.path.append('../../../../warpx/Regression/Checksum/') + +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] + +my_check = checksumAPI.evaluate_checksum( + test_name, fn, do_particles=True, rtol=0.01 +) + +ref_density = np.array([ + 1.27953969e+14, 2.23553999e+14, 2.55384510e+14, 2.55663110e+14, + 2.55805760e+14, 2.55812087e+14, 2.55813911e+14, 2.55754104e+14, + 2.55929601e+14, 2.56085472e+14, 2.55932867e+14, 2.55828121e+14, + 2.55901711e+14, 2.55985074e+14, 2.56182697e+14, 2.56446847e+14, + 2.56483696e+14, 2.56301187e+14, 2.56245301e+14, 2.56797584e+14, + 2.57257907e+14, 2.57023627e+14, 2.56500876e+14, 2.56106851e+14, + 2.56283546e+14, 2.56723967e+14, 2.56960855e+14, 2.56825486e+14, + 2.56674669e+14, 2.56567191e+14, 2.56310927e+14, 2.56361171e+14, + 2.56692197e+14, 2.56743606e+14, 2.56653108e+14, 2.56883854e+14, + 2.56763228e+14, 2.56343726e+14, 2.56385489e+14, 2.56570110e+14, + 2.56538112e+14, 2.56472179e+14, 2.56322922e+14, 2.56195384e+14, + 2.56474576e+14, 2.56764233e+14, 2.56533016e+14, 2.56257170e+14, + 2.56362463e+14, 2.56363962e+14, 2.56311292e+14, 2.56678788e+14, + 2.57061138e+14, 2.56785892e+14, 2.56406603e+14, 2.56334908e+14, + 2.56120051e+14, 2.56003269e+14, 2.56132187e+14, 2.56329572e+14, + 2.56535713e+14, 2.56708950e+14, 2.56661860e+14, 2.56448986e+14, + 2.56386823e+14, 2.56233660e+14, 2.56137632e+14, 2.56206263e+14, + 2.56364996e+14, 2.56483536e+14, 2.56308741e+14, 2.56447231e+14, + 2.56896301e+14, 2.56691405e+14, 2.56170780e+14, 2.56122216e+14, + 2.56427399e+14, 2.56897558e+14, 2.56928868e+14, 2.56659033e+14, + 2.56749993e+14, 2.56952497e+14, 2.56798907e+14, 2.56377081e+14, + 2.56453057e+14, 2.56796632e+14, 2.56944576e+14, 2.57248469e+14, + 2.57279426e+14, 2.56849516e+14, 2.56601834e+14, 2.56850545e+14, + 2.56953072e+14, 2.56442586e+14, 2.56329006e+14, 2.56790661e+14, + 2.57083582e+14, 2.57075550e+14, 2.56719615e+14, 2.56220486e+14, + 2.56222323e+14, 2.56547365e+14, 2.56499423e+14, 2.56434041e+14, + 2.56378587e+14, 2.56249892e+14, 2.56380492e+14, 2.56504513e+14, + 2.56337631e+14, 2.56204891e+14, 2.56325116e+14, 2.56297798e+14, + 2.56112782e+14, 2.56054218e+14, 2.56320120e+14, 2.56580938e+14, + 2.56446800e+14, 2.56267011e+14, 2.56372853e+14, 2.56617592e+14, + 2.56630745e+14, 2.56615242e+14, 2.56625259e+14, 2.56561320e+14, + 2.56640072e+14, 2.56693273e+14, 2.56613237e+14, 2.24169847e+14, + 1.27683197e+14 +]) + +density_data = np.load( 'ion_density_case_1.npy' ) +print(repr(density_data)) +assert np.allclose(density_data, ref_density, rtol=0.01) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 6e85bb97a97..7d7e150f111 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1545,6 +1545,49 @@ def initialize_inputs(self): collision.add_new_attr(process+'_'+key, val) +class DSMCCollisions(picmistandard.base._ClassWithInit): + """ + Custom class to handle setup of DSMC collisions in WarpX. If collision + initialization is added to picmistandard this can be changed to inherit + that functionality. + + Parameters + ---------- + name: string + Name of instance (used in the inputs file) + + species: species instance + The species involved in the collision + + scattering_processes: dictionary + The scattering process to use and any needed information + + ndt: integer, optional + The collisions will be applied every "ndt" steps. Must be 1 or larger. + """ + + def __init__(self, name, species, scattering_processes, ndt=None, **kw): + self.name = name + self.species = species + self.scattering_processes = scattering_processes + self.ndt = ndt + + self.handle_init(kw) + + def initialize_inputs(self): + collision = pywarpx.Collisions.newcollision(self.name) + collision.type = 'dsmc' + collision.species = [species.name for species in self.species] + collision.ndt = self.ndt + + collision.scattering_processes = self.scattering_processes.keys() + for process, kw in self.scattering_processes.items(): + for key, val in kw.items(): + if key == 'species': + val = val.name + collision.add_new_attr(process+'_'+key, val) + + class EmbeddedBoundary(picmistandard.base._ClassWithInit): """ Custom class to handle set up of embedded boundaries specific to WarpX. diff --git a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json new file mode 100644 index 00000000000..4d30f741bfb --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json @@ -0,0 +1,27 @@ +{ + "lev=0": { + "rho_electrons": 0.004436602398896733, + "rho_he_ions": 0.0052003262664415285 + }, + "he_ions": { + "particle_momentum_x": 2.7735512966774165e-19, + "particle_momentum_y": 2.7574111491186894e-19, + "particle_momentum_z": 3.620520352986572e-19, + "particle_position_x": 2201.236370518716, + "particle_weight": 17190734375000.002 + }, + "electrons": { + "particle_momentum_x": 3.50212700099208e-20, + "particle_momentum_y": 3.5368926859820716e-20, + "particle_momentum_z": 1.2588108956625115e-19, + "particle_position_x": 2139.6498177543617, + "particle_weight": 14582968750000.002 + }, + "neutrals": { + "particle_momentum_x": 1.405588503355727e-19, + "particle_momentum_y": 1.408077689882847e-19, + "particle_momentum_z": 1.4024616940779626e-19, + "particle_position_x": 1121.2330379095083, + "particle_weight": 6.4588e+19 + } +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index b00916463c7..3fba702fbdc 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3078,6 +3078,25 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py +[Python_dsmc_1d] +buildDir = . +inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +runtime_params = +customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc +dim = 1 +addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py + [Python_ElectrostaticSphereEB] buildDir = . inputFile = Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H index be92e42d5fc..1db3922d178 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H @@ -7,9 +7,9 @@ #ifndef WARPX_PARTICLES_COLLISION_BACKGROUNDMCCCOLLISION_H_ #define WARPX_PARTICLES_COLLISION_BACKGROUNDMCCCOLLISION_H_ -#include "MCCProcess.H" #include "Particles/MultiParticleContainer.H" #include "Particles/Collision/CollisionBase.H" +#include "Particles/Collision/ScatteringProcess.H" #include #include @@ -32,7 +32,7 @@ public: BackgroundMCCCollision ( BackgroundMCCCollision&& ) = delete; BackgroundMCCCollision& operator= ( BackgroundMCCCollision&& ) = delete; - amrex::ParticleReal get_nu_max (amrex::Vector const& mcc_processes); + amrex::ParticleReal get_nu_max (amrex::Vector const& mcc_processes); /** Perform the collisions * @@ -70,10 +70,10 @@ public: private: - amrex::Vector m_scattering_processes; - amrex::Vector m_ionization_processes; - amrex::Gpu::DeviceVector m_scattering_processes_exe; - amrex::Gpu::DeviceVector m_ionization_processes_exe; + amrex::Vector m_scattering_processes; + amrex::Vector m_ionization_processes; + amrex::Gpu::DeviceVector m_scattering_processes_exe; + amrex::Gpu::DeviceVector m_ionization_processes_exe; bool init_flag = false; bool ionization_flag = false; diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index 98ecdf70e5b..ece9b073339 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -90,7 +90,7 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name amrex::Vector scattering_process_names; pp_collision_name.queryarr("scattering_processes", scattering_process_names); - // create a vector of MCCProcess objects from each scattering + // create a vector of ScatteringProcess objects from each scattering // process name for (const auto& scattering_process : scattering_process_names) { const std::string kw_cross_section = scattering_process + "_cross_section"; @@ -107,17 +107,17 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name pp_collision_name, kw_energy.c_str(), energy); } - MCCProcess process(scattering_process, cross_section_file, energy); + ScatteringProcess process(scattering_process, cross_section_file, energy); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != MCCProcessType::INVALID, - "Cannot add an unknown MCC process type"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, + "Cannot add an unknown scattering process type"); // if the scattering process is ionization get the secondary species // only one ionization process is supported, the vector // m_ionization_processes is only used to make it simple to calculate // the maximum collision frequency with the same function used for // particle conserving processes - if (process.type() == MCCProcessType::IONIZATION) { + if (process.type() == ScatteringProcessType::IONIZATION) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(!ionization_flag, "Background MCC only supports a single ionization process"); ionization_flag = true; @@ -133,8 +133,8 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name } #ifdef AMREX_USE_GPU - amrex::Gpu::HostVector h_scattering_processes_exe; - amrex::Gpu::HostVector h_ionization_processes_exe; + amrex::Gpu::HostVector h_scattering_processes_exe; + amrex::Gpu::HostVector h_ionization_processes_exe; for (auto const& p : m_scattering_processes) { h_scattering_processes_exe.push_back(p.executor()); } @@ -162,7 +162,7 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name * ranges from 1e-4 to 5000 eV in 0.2 eV increments */ amrex::ParticleReal -BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processes) +BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processes) { using namespace amrex::literals; amrex::ParticleReal nu, nu_max = 0.0; @@ -404,7 +404,7 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile // and target velocities which doesn't require any of the Lorentz // transformations below; note that if the projectile and target // have the same mass this is identical to back scattering - if (scattering_process.m_type == MCCProcessType::CHARGE_EXCHANGE) { + if (scattering_process.m_type == ScatteringProcessType::CHARGE_EXCHANGE) { ux[ip] = ua_x; uy[ip] = ua_y; uz[ip] = ua_z; @@ -433,13 +433,13 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile // transform to COM frame ParticleUtils::doLorentzTransform(vx, vy, vz, uCOM_x, uCOM_y, uCOM_z); - if ((scattering_process.m_type == MCCProcessType::ELASTIC) - || (scattering_process.m_type == MCCProcessType::EXCITATION)) { + if ((scattering_process.m_type == ScatteringProcessType::ELASTIC) + || (scattering_process.m_type == ScatteringProcessType::EXCITATION)) { ParticleUtils::RandomizeVelocity( vx, vy, vz, sqrt(vx*vx + vy*vy + vz*vz), engine ); } - else if (scattering_process.m_type == MCCProcessType::BACK) { + else if (scattering_process.m_type == ScatteringProcessType::BACK) { // elastic scattering with cos(chi) = -1 (i.e. 180 degrees) vx *= -1.0_prt; vy *= -1.0_prt; diff --git a/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt b/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt index 15742af465f..ca8db0c53ec 100644 --- a/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt +++ b/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt @@ -3,6 +3,5 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE BackgroundMCCCollision.cpp - MCCProcess.cpp ) endforeach() diff --git a/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H b/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H index 6cd3b0c20f7..73ed60d0ad7 100644 --- a/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H +++ b/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H @@ -4,10 +4,10 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ -#define WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ +#ifndef WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ +#define WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ -#include "MCCProcess.H" +#include "Particles/Collision/ScatteringProcess.H" #include "Utils/ParticleUtils.H" #include "Utils/WarpXConst.H" @@ -39,7 +39,7 @@ public: * ensure accurate energy calculations (otherwise errors occur with single * or mixed precision builds of WarpX). * - * @param[in] mcc_process an MCCProcess object associated with the ionization + * @param[in] mcc_process an ScatteringProcess object associated with the ionization * @param[in] mass colliding particle's mass (could also assume electron) * @param[in] total_collision_prob total probability for a collision to occur * @param[in] nu_max maximum collision frequency @@ -48,7 +48,7 @@ public: * @param[in] t the current simulation time */ ImpactIonizationFilterFunc( - MCCProcess const& mcc_process, + ScatteringProcess const& mcc_process, double const mass, amrex::ParticleReal const total_collision_prob, amrex::ParticleReal const nu_max, @@ -108,7 +108,7 @@ public: } private: - MCCProcess::Executor m_mcc_process; + ScatteringProcess::Executor m_mcc_process; double m_mass; amrex::ParticleReal m_total_collision_prob = 0; amrex::ParticleReal m_nu_max; @@ -227,4 +227,4 @@ private: amrex::ParserExecutor<4> m_T_a_func; amrex::Real m_t; }; -#endif // WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ +#endif // WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ diff --git a/Source/Particles/Collision/BackgroundMCC/Make.package b/Source/Particles/Collision/BackgroundMCC/Make.package index 0b007e64e6d..242e85a8764 100644 --- a/Source/Particles/Collision/BackgroundMCC/Make.package +++ b/Source/Particles/Collision/BackgroundMCC/Make.package @@ -1,4 +1,3 @@ CEXE_sources += BackgroundMCCCollision.cpp -CEXE_sources += MCCProcess.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BackgroundMCC diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index 0a7aae9835e..b8a390ddb93 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -12,6 +12,8 @@ #include "Particles/MultiParticleContainer.H" +#include + enum struct CollisionType { DeuteriumTritiumToNeutronHeliumFusion, DeuteriumDeuteriumToProtonTritiumFusion, DeuteriumDeuteriumToNeutronHeliumFusion, @@ -36,6 +38,92 @@ namespace BinaryCollisionUtils{ MultiParticleContainer const * mypc); CollisionType nuclear_fusion_type_to_collision_type (NuclearFusionType fusion_type); + + /** + * \brief Return (relativistic) collision energy, collision speed and + * Lorentz factor for transforming between the lab and center-of-momentum + * frames. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void get_collision_parameters ( + const amrex::ParticleReal& u1x, const amrex::ParticleReal& u1y, + const amrex::ParticleReal& u1z, const amrex::ParticleReal& u2x, + const amrex::ParticleReal& u2y, const amrex::ParticleReal& u2z, + const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, + amrex::ParticleReal& E_kin_COM, amrex::ParticleReal& v_rel_COM, + amrex::ParticleReal& lab_to_COM_lorentz_factor ) + { + // General notations in this function: + // x_sq denotes the square of x + // x_star denotes the value of x in the center of mass frame + + using namespace amrex::literals; + using namespace amrex::Math; + + constexpr auto one_pr = amrex::ParticleReal(1.); + constexpr auto inv_four_pr = amrex::ParticleReal(1./4.); + constexpr double c_sq = PhysConst::c * PhysConst::c; + constexpr double inv_csq = 1.0 / c_sq; + + const amrex::ParticleReal m1_sq = m1*m1; + const amrex::ParticleReal m2_sq = m2*m2; + + // Compute Lorentz factor gamma in the lab frame + const double g1 = std::sqrt( 1.0 + static_cast(u1x*u1x+u1y*u1y+u1z*u1z)*inv_csq ); + const double g2 = std::sqrt( 1.0 + static_cast(u2x*u2x+u2y*u2y+u2z*u2z)*inv_csq ); + + // Compute momenta + const amrex::ParticleReal p1x = u1x * m1; + const amrex::ParticleReal p1y = u1y * m1; + const amrex::ParticleReal p1z = u1z * m1; + const amrex::ParticleReal p2x = u2x * m2; + const amrex::ParticleReal p2y = u2y * m2; + const amrex::ParticleReal p2z = u2z * m2; + // Square norm of the total (sum between the two particles) momenta in the lab frame + const auto p_total_sq = static_cast( + powi<2>(p1x + p2x) + powi<2>(p1y + p2y) + powi<2>(p1z + p2z) + ); + + // Total energy in the lab frame + // Note the use of `double` for energy since this calculation is + // prone to error with single precision. + const auto m1_dbl = static_cast(m1); + const auto m2_dbl = static_cast(m2); + const double E_lab = (m1_dbl * g1 + m2_dbl * g2) * c_sq; + // Total energy squared in the center of mass frame, calculated using the Lorentz invariance + // of the four-momentum norm + const double E_star_sq = E_lab*E_lab - c_sq*p_total_sq; + + // Kinetic energy in the center of mass frame + const double E_star = std::sqrt(E_star_sq); + E_kin_COM = static_cast(E_star - (m1_dbl + m2_dbl)*c_sq); + + // Square of the norm of the momentum of one of the particles in the center of mass frame + // Formula obtained by inverting E^2 = p^2*c^2 + m^2*c^4 in the COM frame for each particle + // The expression below is specifically written in a form that avoids returning + // small negative numbers due to machine precision errors, for low-energy particles + const auto E_ratio = static_cast(E_star/((m1 + m2)*c_sq)); + const auto p_star_sq = static_cast( + m1*m2*c_sq * ( powi<2>(E_ratio) - one_pr ) + + powi<2>(m1 - m2)*c_sq*inv_four_pr * powi<2>( E_ratio - 1._prt/E_ratio) + ); + + // Lorentz factors in the center of mass frame + const auto g1_star = std::sqrt(one_pr + p_star_sq / static_cast(m1_sq*c_sq)); + const auto g2_star = std::sqrt(one_pr + p_star_sq / static_cast(m2_sq*c_sq)); + + // relative velocity in the center of mass frame + v_rel_COM = std::sqrt(p_star_sq) * (one_pr/(m1*g1_star) + one_pr/(m2*g2_star)); + + // Cross sections and relative velocity are computed in the center of mass frame. + // On the other hand, the particle densities (weight over volume) in the lab frame are used. + // To take this disrepancy into account, it is needed to multiply the + // collision probability by the ratio between the Lorentz factors in the + // COM frame and the Lorentz factors in the lab frame (see + // Perez et al., Phys.Plasmas.19.083104 (2012)). The correction factor + // is calculated here. + lab_to_COM_lorentz_factor = g1_star*g2_star/static_cast(g1*g2); + } } #endif // BINARY_COLLISION_UTILS_H_ diff --git a/Source/Particles/Collision/BinaryCollision/CMakeLists.txt b/Source/Particles/Collision/BinaryCollision/CMakeLists.txt index 55bbf21e310..60933c83a21 100644 --- a/Source/Particles/Collision/BinaryCollision/CMakeLists.txt +++ b/Source/Particles/Collision/BinaryCollision/CMakeLists.txt @@ -6,3 +6,5 @@ foreach(D IN LISTS WarpX_DIMS) ParticleCreationFunc.cpp ) endforeach() + +add_subdirectory(DSMC) diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt new file mode 100644 index 00000000000..3575d53ad29 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt @@ -0,0 +1,7 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + DSMC.cpp + ) +endforeach() diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H new file mode 100644 index 00000000000..a847b841ee2 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H @@ -0,0 +1,232 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies), Neil Zaim + * + * License: BSD-3-Clause-LBNL + */ +#ifndef COLLISION_FILTER_FUNC_H_ +#define COLLISION_FILTER_FUNC_H_ + +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" +#include "Particles/Collision/ScatteringProcess.H" + +#include + +/** + * \brief This function determines whether a collision occurs for a given + * pair of particles. + * + * @param[in] u1x,u1y,u1z momenta of the first colliding particle + * @param[in] u2x,u2y,u2z momenta of the second colliding particle + * @param[in] m1,m2 masses + * @param[in] w1,w2 effective weight of the colliding particles + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] pair_index is the index of the colliding pair + * @param[out] p_mask is a mask that will be set to a non-zero integer if a + * collision occurs. The integer encodes the scattering process. + * @param[out] p_pair_reaction_weight stores the weight of the product particles + * @param[in] process_count number of scattering processes to consider + * @param[in] scattering processes an array of scattering processes included for consideration + * @param[in] engine the random engine. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleReal& u1y, + const amrex::ParticleReal& u1z, const amrex::ParticleReal& u2x, + const amrex::ParticleReal& u2y, const amrex::ParticleReal& u2z, + const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, + amrex::ParticleReal w1, amrex::ParticleReal w2, + const amrex::Real& dt, const amrex::ParticleReal& dV, const int& pair_index, + index_type* AMREX_RESTRICT p_mask, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + const int& multiplier_ratio, + int const process_count, + ScatteringProcess::Executor* scattering_processes, + const amrex::RandomEngine& engine) +{ + using namespace amrex::literals; + + amrex::ParticleReal E_coll, v_coll, lab_to_COM_factor; + + const amrex::ParticleReal w_min = amrex::min(w1, w2); + const amrex::ParticleReal w_max = amrex::max(w1, w2); + + BinaryCollisionUtils::get_collision_parameters( + u1x, u1y, u1z, u2x, u2y, u2z, m1, m2, + E_coll, v_coll, lab_to_COM_factor); + + // convert E_coll to eV + E_coll /= PhysConst::q_e; + + amrex::ParticleReal sigma_tot = 0._prt; + for (int ii = 0; ii < process_count; ii++) { + auto const& scattering_process = *(scattering_processes + ii); + sigma_tot += scattering_process.getCrossSection(E_coll); + } + + // calculate total collision probability + amrex::ParticleReal exponent = ( + lab_to_COM_factor * multiplier_ratio * w_max + * sigma_tot * v_coll * dt / dV + ); + + // Compute actual collision probability that is always between zero and one + // In principle this is obtained by computing 1 - exp(-probability_estimate) + // However, the computation of this quantity can fail numerically when probability_estimate is + // too small (e.g. exp(-probability_estimate) returns 1 and the computation returns 0). + // In this case, we simply use "probability_estimate" instead of 1 - exp(-probability_estimate) + // The threshold exp_threshold at which we switch between the two formulas is determined by the + // fact that computing the exponential is only useful if it can resolve the x^2/2 term of its + // Taylor expansion, i.e. the square of probability_estimate should be greater than the + // machine epsilon. +#ifdef AMREX_SINGLE_PRECISION_PARTICLES + constexpr auto exp_threshold = amrex::ParticleReal(1.e-3); +#else + constexpr auto exp_threshold = amrex::ParticleReal(5.e-8); +#endif + const amrex::ParticleReal probability = (exponent < exp_threshold) ? + exponent: 1._prt - std::exp(-exponent); + + // Now we determine if a collision should occur + if (amrex::Random(engine) < probability) + { + const amrex::ParticleReal random_number = amrex::Random(engine); + amrex::ParticleReal sigma = 0._prt; + for (int ii = 0; ii < process_count; ii++) { + auto const& scattering_process = *(scattering_processes + ii); + sigma += scattering_process.getCrossSection(E_coll); + if (random_number <= sigma / sigma_tot) + { + p_mask[pair_index] = int(scattering_process.m_type); + p_pair_reaction_weight[pair_index] = w_min; + break; + } + } + } + else + { + p_mask[pair_index] = false; + } +} + +/** + * \brief Function that determines if a collision occurs and if so, what + * type. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in] ptd_1,ptd_2 contain the particle data of the two species + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] cell_start_pair is the start index of the pairs in that cell. + * @param[out] p_mask is a mask that will be set to a non-zero integer if a + * collision occurs. The integer encodes the scattering process. + * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the + * particles of a given pair. They are only needed here to store information that will be used + * later on when actually creating the product particles. + * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only + * needed here to store information that will be used later on when actually creating the + * product particles. + * @param[in] process_count number of scattering processes to consider + * @param[in] scattering processes an array of scattering processes included for consideration + * @param[in] engine the random engine. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void CollisionFilter ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + PData ptd_1, PData ptd_2, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, + index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, + index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + int const process_count, + ScatteringProcess::Executor* scattering_processes, + amrex::RandomEngine const& engine) +{ + + amrex::ParticleReal * const AMREX_RESTRICT w1 = ptd_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u1x = ptd_1.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u1y = ptd_1.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u1z = ptd_1.m_rdata[PIdx::uz]; + + amrex::ParticleReal * const AMREX_RESTRICT w2 = ptd_2.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u2x = ptd_2.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u2y = ptd_2.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u2z = ptd_2.m_rdata[PIdx::uz]; + + // Number of macroparticles of each species + const int NI1 = I1e - I1s; + const int NI2 = I2e - I2s; + const int max_N = amrex::max(NI1,NI2); + + int i1 = I1s; + int i2 = I2s; + int pair_index = cell_start_pair; + + // Because the number of particles of each species is not always equal (NI1 != NI2 + // in general), some macroparticles will be paired with multiple macroparticles of the + // other species and we need to decrease their weight accordingly. + // c1 corresponds to the minimum number of times a particle of species 1 will be paired + // with a particle of species 2. Same for c2. + const int c1 = amrex::max(NI2/NI1,1); + const int c2 = amrex::max(NI1/NI2,1); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal * const AMREX_RESTRICT theta1 = ptd_1.m_rdata[PIdx::theta]; + amrex::ParticleReal * const AMREX_RESTRICT theta2 = ptd_2.m_rdata[PIdx::theta]; +#endif + + for (int k = 0; k < max_N; ++k) + { + // c1k : how many times the current particle of species 1 is paired with a particle + // of species 2. Same for c2k. + const int c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; + const int c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; + +#if (defined WARPX_DIM_RZ) + /* In RZ geometry, macroparticles can collide with other macroparticles + * in the same *cylindrical* cell. For this reason, collisions between macroparticles + * are actually not local in space. In this case, the underlying assumption is that + * particles within the same cylindrical cell represent a cylindrically-symmetry + * momentum distribution function. Therefore, here, we temporarily rotate the + * momentum of one of the macroparticles in agreement with this cylindrical symmetry. + * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; + * there is a corresponding assert statement at initialization.) */ + amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; + amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); + u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); +#endif + + CollisionPairFilter( + u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], + u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], + m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, + dt, dV, pair_index, p_mask, p_pair_reaction_weight, + max_N, process_count, scattering_processes, engine); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); + u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); +#endif + + p_pair_indices_1[pair_index] = I1[i1]; + p_pair_indices_2[pair_index] = I2[i2]; + ++i1; if ( i1 == static_cast(I1e) ) { i1 = I1s; } + ++i2; if ( i2 == static_cast(I2e) ) { i2 = I2s; } + ++pair_index; + } +} + +#endif // COLLISION_FILTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H new file mode 100644 index 00000000000..c1be307b811 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H @@ -0,0 +1,85 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef DSMC_H_ +#define DSMC_H_ + +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" +#include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" +#include "Particles/Collision/CollisionBase.H" +#include "Particles/Collision/ScatteringProcess.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleCreation/SmartCopy.H" +#include "Particles/ParticleCreation/SmartUtils.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include +#include + + +/** + * \brief This class performs DSMC (direct simulation Monte Carlo) collisions + * within a cell. Particles are paired up and for each pair a stochastic process + * determines whether a collision occurs. The algorithm is similar to the one + * used for binary Coulomb collisions and the nuclear fusion module. + */ +class DSMC final + : public CollisionBase +{ + // Define shortcuts for frequently-used type names + using ParticleType = WarpXParticleContainer::ParticleType; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleBins = amrex::DenseBins; + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using index_type = ParticleBins::index_type; + +public: + + /** + * \brief Constructor of the DSMC class + * + * @param[in] collision_name the name of the collision + */ + DSMC (std::string collision_name); + + /** Perform the collisions + * + * @param cur_time Current time + * @param dt Time step size + * @param mypc Container of species involved + * + */ + void doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) override; + + /** Perform all binary collisions within a tile + * + * \param[in] lev the mesh-refinement level + * \param[in] mfi iterator for multifab + * \param species_1 first species container + * \param species_2 second species container + * \param copy_species1 SmartCopy for species_1 + * \param copy_species2 SmartCopy for species_2 + * + */ + void doCollisionsWithinTile ( + amrex::Real dt, int lev, amrex::MFIter const& mfi, + WarpXParticleContainer& species_1, + WarpXParticleContainer& species_2, + SmartCopy& copy_species1, + SmartCopy& copy_species2 ); + +private: + amrex::Vector m_scattering_processes; + amrex::Gpu::DeviceVector m_scattering_processes_exe; +}; + +#endif // DSMC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp new file mode 100644 index 00000000000..70a14712a0c --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp @@ -0,0 +1,300 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#include "CollisionFilterFunc.H" +#include "DSMC.H" +#include "SplitAndScatterFunc.H" + + +DSMC::DSMC (const std::string collision_name) + : CollisionBase(collision_name) +{ + using namespace amrex::literals; + amrex::ParmParse pp_collision_name(collision_name); + +#if defined WARPX_DIM_RZ + amrex::Abort("DSMC collisions are only implemented for Cartesian coordinates."); +#endif + + if(m_species_names.size() != 2) + { + amrex::Abort("DSMC collision " + collision_name + " must have exactly two species."); + } + + // query for a list of collision processes + // these could be elastic, excitation, charge_exchange, back, etc. + amrex::Vector scattering_process_names; + pp_collision_name.queryarr("scattering_processes", scattering_process_names); + + // create a vector of ScatteringProcess objects from each scattering + // process name + for (const auto& scattering_process : scattering_process_names) { + std::string kw_cross_section = scattering_process + "_cross_section"; + std::string cross_section_file; + pp_collision_name.query(kw_cross_section.c_str(), cross_section_file); + + // if the scattering process is excitation or ionization get the + // energy associated with that process + amrex::ParticleReal energy = 0._prt; + if (scattering_process.find("excitation") != std::string::npos || + scattering_process.find("ionization") != std::string::npos) { + std::string kw_energy = scattering_process + "_energy"; + utils::parser::getWithParser( + pp_collision_name, kw_energy.c_str(), energy); + } + + ScatteringProcess process(scattering_process, cross_section_file, energy); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, + "Cannot add an unknown scattering process type"); + + m_scattering_processes.push_back(std::move(process)); + } + +#ifdef AMREX_USE_GPU + amrex::Gpu::HostVector h_scattering_processes_exe; + for (auto const& p : m_scattering_processes) { + h_scattering_processes_exe.push_back(p.executor()); + } + m_scattering_processes_exe.resize(h_scattering_processes_exe.size()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scattering_processes_exe.begin(), + h_scattering_processes_exe.end(), m_scattering_processes_exe.begin()); + amrex::Gpu::streamSynchronize(); +#else + for (auto const& p : m_scattering_processes) { + m_scattering_processes_exe.push_back(p.executor()); + } +#endif +} + +void +DSMC::doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) +{ + WARPX_PROFILE("DSMC::doCollisions()"); + + auto& species1 = mypc->GetParticleContainerFromName(m_species_names[0]); + auto& species2 = mypc->GetParticleContainerFromName(m_species_names[1]); + + // SmartCopy objects are created that will facilitate the particle splitting + // operation involved in DSMC collisions between particles with arbitrary + // weights. + SmartCopyFactory copy_factory_species1(species1, species1); + SmartCopyFactory copy_factory_species2(species2, species2); + auto copy_species1 = copy_factory_species1.getSmartCopy(); + auto copy_species2 = copy_factory_species2.getSmartCopy(); + + species1.defineAllParticleTiles(); + species2.defineAllParticleTiles(); + + // Enable tiling + amrex::MFItInfo info; + if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(WarpXParticleContainer::tile_size); } + + // Loop over refinement levels + for (int lev = 0; lev <= species1.finestLevel(); ++lev){ + + amrex::LayoutData* cost = WarpX::getCosts(lev); + + // Loop over all grids/tiles at this level +#ifdef AMREX_USE_OMP + info.SetDynamic(true); +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi = species1.MakeMFIter(lev, info); mfi.isValid(); ++mfi){ + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + } + auto wt = static_cast(amrex::second()); + + doCollisionsWithinTile(dt, lev, mfi, species1, species2, + copy_species1, copy_species2); + + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + wt = static_cast(amrex::second()) - wt; + amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); + } + } + + // Call redistribute to remove particles with negative ids + species1.Redistribute(lev, lev, 0, true, true); + species2.Redistribute(lev, lev, 0, true, true); + } +} + +void +DSMC::doCollisionsWithinTile( + amrex::Real dt, int const lev, amrex::MFIter const& mfi, + WarpXParticleContainer& species_1, + WarpXParticleContainer& species_2, + SmartCopy& copy_species1, + SmartCopy& copy_species2) +{ + using namespace ParticleUtils; + using namespace amrex::literals; + + // get collision processes + auto *scattering_processes = m_scattering_processes_exe.data(); + int const process_count = static_cast(m_scattering_processes_exe.size()); + + // Extract particles in the tile that `mfi` points to + ParticleTileType& ptile_1 = species_1.ParticlesAt(lev, mfi); + ParticleTileType& ptile_2 = species_2.ParticlesAt(lev, mfi); + + // Find the particles that are in each cell of this tile + ParticleBins bins_1 = findParticlesInEachCell( lev, mfi, ptile_1 ); + ParticleBins bins_2 = findParticlesInEachCell( lev, mfi, ptile_2 ); + + // Extract low-level data + int const n_cells = static_cast(bins_1.numBins()); + + // - Species 1 + index_type* AMREX_RESTRICT indices_1 = bins_1.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_1 = bins_1.offsetsPtr(); + amrex::ParticleReal m1 = species_1.getMass(); + const auto ptd_1 = ptile_1.getParticleTileData(); + + // - Species 2 + index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_2 = bins_2.offsetsPtr(); + amrex::ParticleReal m2 = species_2.getMass(); + const auto ptd_2 = ptile_2.getParticleTileData(); + + amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); +#if defined WARPX_DIM_1D_Z + auto dV = geom.CellSize(0); +#elif defined WARPX_DIM_XZ + auto dV = geom.CellSize(0) * geom.CellSize(1); +#elif defined WARPX_DIM_RZ + amrex::Box const& cbx = mfi.tilebox(amrex::IntVect::TheZeroVector()); //Cell-centered box + const auto lo = lbound(cbx); + const auto hi = ubound(cbx); + int nz = hi.y-lo.y+1; + auto dr = geom.CellSize(0); + auto dz = geom.CellSize(1); +#elif defined(WARPX_DIM_3D) + auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); +#endif + + // In the following we set up the "mask" used for creating new particles + // (from splitting events). There is a mask index for every collision + // pair. Below we find the size of the mask based on the greater of the + // number of each species' particle in each cell. + amrex::Gpu::DeviceVector n_pairs_in_each_cell(n_cells); + index_type* AMREX_RESTRICT p_n_pairs_in_each_cell = n_pairs_in_each_cell.dataPtr(); + + // Compute how many pairs in each cell and store in n_pairs_in_each_cell array + // For different species, the number of pairs in a cell is the number of particles of + // the species that has the most particles in that cell + amrex::ParallelFor( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; + const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; + // Particular case: no pair if a species has no particle in that cell + if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) + { + p_n_pairs_in_each_cell[i_cell] = 0; + } + else + { + p_n_pairs_in_each_cell[i_cell] = amrex::max(n_part_in_cell_1,n_part_in_cell_2); + } + } + ); + + // Start indices of the pairs in a cell. Will be used for particle creation + amrex::Gpu::DeviceVector pair_offsets(n_cells); + const index_type n_total_pairs = (n_cells == 0) ? 0: + amrex::Scan::ExclusiveSum(n_cells, + p_n_pairs_in_each_cell, pair_offsets.data()); + index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); + + // Now we create the mask. In the DSMC scheme the mask will serve two + // purposes, 1) record whether a given pair should collide and 2) record + // the scattering process that should occur. + amrex::Gpu::DeviceVector mask(n_total_pairs); + index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); + + // Will be filled with the index of the first particle of a given pair + amrex::Gpu::DeviceVector pair_indices_1(n_total_pairs); + index_type* AMREX_RESTRICT p_pair_indices_1 = pair_indices_1.dataPtr(); + // Will be filled with the index of the second particle of a given pair + amrex::Gpu::DeviceVector pair_indices_2(n_total_pairs); + index_type* AMREX_RESTRICT p_pair_indices_2 = pair_indices_2.dataPtr(); + + // How much weight should be given to the produced particle - based on the + // weight of the collision partner which is not split + amrex::Gpu::DeviceVector pair_reaction_weight(n_total_pairs); + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight = + pair_reaction_weight.dataPtr(); + + // Loop over cells + amrex::ParallelForRNG( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept + { + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + // Same for species 2 + index_type const cell_start_2 = cell_offsets_2[i_cell]; + index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; + // Same but for the pairs + index_type const cell_start_pair = p_pair_offsets[i_cell]; + + // ux from species1 can be accessed like this: + // ux_1[ indices_1[i] ], where i is between + // cell_start_1 (inclusive) and cell_start_2 (exclusive) + + // Do not collide if one species is missing in the cell + if ( cell_stop_1 - cell_start_1 < 1 || + cell_stop_2 - cell_start_2 < 1 ) { return; } + + // shuffle + ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); + ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); +#if defined WARPX_DIM_RZ + int ri = (i_cell - i_cell%nz) / nz; + auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; +#endif + // Call the function in order to perform collisions + // If there are product species, p_mask, p_pair_indices_1/2, and + // p_pair_reaction_weight are filled here + CollisionFilter( + cell_start_1, cell_stop_1, cell_start_2, cell_stop_2, + indices_1, indices_2, + ptd_1, ptd_2, + m1, m2, dt, dV, + cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, + p_pair_reaction_weight, + process_count, scattering_processes, engine ); + } + ); + + const auto num_p_tile1 = ptile_1.numParticles(); + const auto num_p_tile2 = ptile_2.numParticles(); + + // Create the new product particles and define their initial values + // num_added: how many particles of each product species have been created + const int num_added = splitScatteringParticles( + n_total_pairs, + ptile_1, ptile_2, + p_mask, + copy_species1, copy_species2, + m1, m2, + p_pair_indices_1, p_pair_indices_2, + p_pair_reaction_weight); + + if (num_added > 0) { + setNewParticleIDs(ptile_1, num_p_tile1, num_added); + setNewParticleIDs(ptile_2, num_p_tile2, num_added); + } +} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/Make.package b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package new file mode 100644 index 00000000000..b4cfab89c64 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package @@ -0,0 +1,3 @@ +CEXE_sources += DSMC.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/DSMC diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H new file mode 100644 index 00000000000..c1fb7ee7e38 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -0,0 +1,165 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef SPLIT_AND_SCATTER_FUNC_H_ +#define SPLIT_AND_SCATTER_FUNC_H_ + +#include "Particles/Collision/ScatteringProcess.H" + +/** + * \brief Function that performs the particle scattering and injection due + * to binary collisions. + * + * \return num_added the number of particles added to each species. + */ +template ::value, int> foo = 0> +int splitScatteringParticles ( + const index_type& n_total_pairs, + Tile& ptile1, Tile& ptile2, + const Index* AMREX_RESTRICT mask, + CopyFunc&& copy1, CopyFunc&& copy2, + const amrex::ParticleReal m1, const amrex::ParticleReal m2, + const index_type* AMREX_RESTRICT p_pair_indices_1, + const index_type* AMREX_RESTRICT p_pair_indices_2, + const amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight ) noexcept +{ + using namespace amrex; + + if (n_total_pairs == 0) { + return 0; + } + + Gpu::DeviceVector offsets(n_total_pairs); + Index* AMREX_RESTRICT p_offsets = offsets.dataPtr(); + + // The following is used to calculate the appropriate offsets. Note that + // a standard cummulative sum is not appropriate since the mask is also + // used to specify the type of collision and can therefore have values >1 + auto const num_added = amrex::Scan::PrefixSum(n_total_pairs, + [=] AMREX_GPU_DEVICE (Index i) -> Index { return mask[i] ? 1 : 0; }, + [=] AMREX_GPU_DEVICE (Index i, Index s) { p_offsets[i] = s; }, + amrex::Scan::Type::exclusive, amrex::Scan::retSum + ); + + const auto ptile1_index = ptile1.numParticles(); + const auto ptile2_index = ptile2.numParticles(); + ptile1.resize(ptile1_index + num_added); + ptile2.resize(ptile2_index + num_added); + + const auto ptile1_data = ptile1.getParticleTileData(); + const auto ptile2_data = ptile2.getParticleTileData(); + + const Long minus_one_long = -1; + + ParallelForRNG(n_total_pairs, + [=] AMREX_GPU_DEVICE (int i, RandomEngine const& engine) noexcept + { + if (mask[i]) + { + // First, make copies of the colliding particles + copy1(ptile1_data, ptile1_data, p_pair_indices_1[i], p_offsets[i] + ptile1_index, engine); + copy2(ptile2_data, ptile2_data, p_pair_indices_2[i], p_offsets[i] + ptile2_index, engine); + + // Now we adjust the properties of the original and child particles, + // starting with the parent particles + auto& w1 = ptile1_data.m_rdata[PIdx::w][p_pair_indices_1[i]]; + auto& w2 = ptile2_data.m_rdata[PIdx::w][p_pair_indices_2[i]]; + + // Remove p_pair_reaction_weight[i] from the colliding particles' weights. + // If the colliding particle weight decreases to zero, remove particle by + // setting its id to -1. + Gpu::Atomic::AddNoRet(&w1, -p_pair_reaction_weight[i]); + if (w1 <= 0._prt) { + auto& p = ptile1_data.m_aos[p_pair_indices_1[i]]; + p.atomicSetID(minus_one_long); + } + + Gpu::Atomic::AddNoRet(&w2, -p_pair_reaction_weight[i]); + if (w2 <= 0._prt) { + auto& p = ptile2_data.m_aos[p_pair_indices_2[i]]; + p.atomicSetID(minus_one_long); + } + + // Set the child particle properties appropriately + ptile1_data.m_rdata[PIdx::w][p_offsets[i] + ptile1_index] = p_pair_reaction_weight[i]; + ptile2_data.m_rdata[PIdx::w][p_offsets[i] + ptile2_index] = p_pair_reaction_weight[i]; + + auto& ux1 = ptile1_data.m_rdata[PIdx::ux][p_offsets[i] + ptile1_index]; + auto& uy1 = ptile1_data.m_rdata[PIdx::uy][p_offsets[i] + ptile1_index]; + auto& uz1 = ptile1_data.m_rdata[PIdx::uz][p_offsets[i] + ptile1_index]; + auto& ux2 = ptile2_data.m_rdata[PIdx::ux][p_offsets[i] + ptile2_index]; + auto& uy2 = ptile2_data.m_rdata[PIdx::uy][p_offsets[i] + ptile2_index]; + auto& uz2 = ptile2_data.m_rdata[PIdx::uz][p_offsets[i] + ptile2_index]; + + // for simplicity (for now) we assume non-relativistic particles + // and simply calculate the center-of-momentum velocity from the + // rest masses + auto const uCOM_x = (m1 * ux1 + m2 * ux2) / (m1 + m2); + auto const uCOM_y = (m1 * uy1 + m2 * uy2) / (m1 + m2); + auto const uCOM_z = (m1 * uz1 + m2 * uz2) / (m1 + m2); + + // transform to COM frame + ux1 -= uCOM_x; + uy1 -= uCOM_y; + uz1 -= uCOM_z; + ux2 -= uCOM_x; + uy2 -= uCOM_y; + uz2 -= uCOM_z; + + if (mask[i] == int(ScatteringProcessType::ELASTIC)) { + // randomly rotate the velocity vector for the first particle + ParticleUtils::RandomizeVelocity( + ux1, uy1, uz1, std::sqrt(ux1*ux1 + uy1*uy1 + uz1*uz1), engine + ); + // set the second particles velocity so that the total momentum + // is zero + ux2 = -ux1 * m1 / m2; + uy2 = -uy1 * m1 / m2; + uz2 = -uz1 * m1 / m2; + } else if (mask[i] == int(ScatteringProcessType::BACK)) { + // reverse the velocity vectors of both particles + ux1 *= -1.0_prt; + uy1 *= -1.0_prt; + uz1 *= -1.0_prt; + ux2 *= -1.0_prt; + uy2 *= -1.0_prt; + uz2 *= -1.0_prt; + } else if (mask[i] == int(ScatteringProcessType::CHARGE_EXCHANGE)) { + if (m1 == m2) { + auto const temp_ux = ux1; + auto const temp_uy = uy1; + auto const temp_uz = uz1; + ux1 = ux2; + uy1 = uy2; + uz1 = uz2; + ux2 = temp_ux; + uy2 = temp_uy; + uz2 = temp_uz; + } + else { + Abort("Uneven mass charge-exchange not implemented yet."); + } + } + else { + Abort("Unknown scattering process."); + } + // transform back to labframe + ux1 += uCOM_x; + uy1 += uCOM_y; + uz1 += uCOM_z; + ux2 += uCOM_x; + uy2 += uCOM_y; + uz2 += uCOM_z; + } + }); + + Gpu::synchronize(); + return static_cast(num_added); +} +#endif // SPLIT_AND_SCATTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/Make.package b/Source/Particles/Collision/BinaryCollision/Make.package index 92e2f0b5dd1..393e6270345 100644 --- a/Source/Particles/Collision/BinaryCollision/Make.package +++ b/Source/Particles/Collision/BinaryCollision/Make.package @@ -1,4 +1,6 @@ CEXE_sources += BinaryCollisionUtils.cpp CEXE_sources += ParticleCreationFunc.cpp +include $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/DSMC/Make.package + VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H index 8b0e71763bc..bd0c6b06d6f 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H @@ -15,7 +15,6 @@ #include "Utils/WarpXConst.H" #include -#include #include #include @@ -67,89 +66,33 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part const NuclearFusionType& fusion_type, const amrex::RandomEngine& engine) { - // General notations in this function: - // x_sq denotes the square of x - // x_star denotes the value of x in the center of mass frame + amrex::ParticleReal E_coll, v_coll, lab_to_COM_factor; + + BinaryCollisionUtils::get_collision_parameters( + u1x, u1y, u1z, u2x, u2y, u2z, m1, m2, + E_coll, v_coll, lab_to_COM_factor); using namespace amrex::literals; - using namespace amrex::Math; const amrex::ParticleReal w_min = amrex::min(w1, w2); const amrex::ParticleReal w_max = amrex::max(w1, w2); - constexpr auto one_pr = amrex::ParticleReal(1.); - constexpr auto inv_four_pr = amrex::ParticleReal(1./4.); - constexpr amrex::ParticleReal c_sq = PhysConst::c * PhysConst::c; - constexpr amrex::ParticleReal inv_csq = one_pr / ( c_sq ); - - const amrex::ParticleReal m1_sq = m1*m1; - const amrex::ParticleReal m2_sq = m2*m2; - - // Compute Lorentz factor gamma in the lab frame - const amrex::ParticleReal g1 = std::sqrt( one_pr + (u1x*u1x+u1y*u1y+u1z*u1z)*inv_csq ); - const amrex::ParticleReal g2 = std::sqrt( one_pr + (u2x*u2x+u2y*u2y+u2z*u2z)*inv_csq ); - - // Compute momenta - const amrex::ParticleReal p1x = u1x * m1; - const amrex::ParticleReal p1y = u1y * m1; - const amrex::ParticleReal p1z = u1z * m1; - const amrex::ParticleReal p2x = u2x * m2; - const amrex::ParticleReal p2y = u2y * m2; - const amrex::ParticleReal p2z = u2z * m2; - // Square norm of the total (sum between the two particles) momenta in the lab frame - const amrex::ParticleReal p_total_sq = powi<2>(p1x + p2x) + - powi<2>(p1y+p2y) + - powi<2>(p1z+p2z); - - // Total energy in the lab frame - const amrex::ParticleReal E_lab = (m1 * g1 + m2 * g2) * c_sq; - // Total energy squared in the center of mass frame, calculated using the Lorentz invariance - // of the four-momentum norm - const amrex::ParticleReal E_star_sq = E_lab*E_lab - c_sq*p_total_sq; - - // Kinetic energy in the center of mass frame - const amrex::ParticleReal E_star = std::sqrt(E_star_sq); - const amrex::ParticleReal E_kin_star = E_star - (m1 + m2)*c_sq; - // Compute fusion cross section as a function of kinetic energy in the center of mass frame auto fusion_cross_section = amrex::ParticleReal(0.); if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { - fusion_cross_section = ProtonBoronFusionCrossSection(E_kin_star); + fusion_cross_section = ProtonBoronFusionCrossSection(E_coll); } else if ((fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) || (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) || (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium)) { - fusion_cross_section = BoschHaleFusionCrossSection(E_kin_star, fusion_type, m1, m2); + fusion_cross_section = BoschHaleFusionCrossSection(E_coll, fusion_type, m1, m2); } - // Square of the norm of the momentum of one of the particles in the center of mass frame - // Formula obtained by inverting E^2 = p^2*c^2 + m^2*c^4 in the COM frame for each particle - // The expression below is specifically written in a form that avoids returning - // small negative numbers due to machine precision errors, for low-energy particles - const amrex::ParticleReal E_ratio = E_star/((m1 + m2)*c_sq); - const amrex::ParticleReal p_star_sq = m1*m2*c_sq * ( powi<2>(E_ratio) - one_pr ) - + powi<2>(m1 - m2)*c_sq*inv_four_pr * powi<2>( E_ratio - 1._prt/E_ratio ); - - // Lorentz factors in the center of mass frame - const amrex::ParticleReal g1_star = std::sqrt(one_pr + p_star_sq / (m1_sq*c_sq)); - const amrex::ParticleReal g2_star = std::sqrt(one_pr + p_star_sq / (m2_sq*c_sq)); - - // relative velocity in the center of mass frame - const amrex::ParticleReal v_rel = std::sqrt(p_star_sq) * (one_pr/(m1*g1_star) + - one_pr/(m2*g2_star)); - - // Fusion cross section and relative velocity are computed in the center of mass frame. - // On the other hand, the particle densities (weight over volume) in the lab frame are used. To - // take into account this discrepancy, we need to multiply the fusion probability by the ratio - // between the Lorentz factors in the COM frame and the Lorentz factors in the lab frame - // (see Perez et al., Phys.Plasmas.19.083104 (2012)) - const amrex::ParticleReal lab_to_COM_factor = g1_star*g2_star/(g1*g2); - // First estimate of probability to have fusion reaction amrex::ParticleReal probability_estimate = multiplier_ratio * fusion_multiplier * - lab_to_COM_factor * w_max * fusion_cross_section * v_rel * dt / dV; + lab_to_COM_factor * w_max * fusion_cross_section * v_coll * dt / dV; // Effective fusion multiplier amrex::ParticleReal fusion_multiplier_eff = fusion_multiplier; @@ -162,7 +105,7 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part // We aim for a fusion probability of probability_target_value but take into account // the constraint that the fusion_multiplier cannot be smaller than one fusion_multiplier_eff = amrex::max(fusion_multiplier * - probability_target_value / probability_estimate , one_pr); + probability_target_value / probability_estimate , 1._prt); probability_estimate *= fusion_multiplier_eff/fusion_multiplier; } @@ -181,7 +124,7 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part constexpr auto exp_threshold = amrex::ParticleReal(5.e-8); #endif const amrex::ParticleReal probability = (probability_estimate < exp_threshold) ? - probability_estimate: one_pr - std::exp(-probability_estimate); + probability_estimate: 1._prt - std::exp(-probability_estimate); // Get a random number const amrex::ParticleReal random_number = amrex::Random(engine); @@ -198,6 +141,4 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part } } - - #endif // SINGLE_NUCLEAR_FUSION_EVENT_H_ diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp index 0d8ffc5b8ae..33629cd530f 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp @@ -19,45 +19,45 @@ ParticleCreationFunc::ParticleCreationFunc (const std::string collision_name, MultiParticleContainer const * const mypc) - { - const amrex::ParmParse pp_collision_name(collision_name); +{ + const amrex::ParmParse pp_collision_name(collision_name); - m_collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); + m_collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); - if (m_collision_type == CollisionType::ProtonBoronToAlphasFusion) - { - // Proton-Boron fusion only produces alpha particles - m_num_product_species = 1; - // Proton-Boron fusion produces 3 alpha particles per fusion reaction - m_num_products_host.push_back(3); + if (m_collision_type == CollisionType::ProtonBoronToAlphasFusion) + { + // Proton-Boron fusion only produces alpha particles + m_num_product_species = 1; + // Proton-Boron fusion produces 3 alpha particles per fusion reaction + m_num_products_host.push_back(3); #ifndef AMREX_USE_GPU - // On CPU, the device vector can be filled immediately - m_num_products_device.push_back(3); + // On CPU, the device vector can be filled immediately + m_num_products_device.push_back(3); #endif - } - else if ((m_collision_type == CollisionType::DeuteriumTritiumToNeutronHeliumFusion) - || (m_collision_type == CollisionType::DeuteriumDeuteriumToProtonTritiumFusion) - || (m_collision_type == CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion)) - { - m_num_product_species = 2; - m_num_products_host.push_back(1); - m_num_products_host.push_back(1); + } + else if ((m_collision_type == CollisionType::DeuteriumTritiumToNeutronHeliumFusion) + || (m_collision_type == CollisionType::DeuteriumDeuteriumToProtonTritiumFusion) + || (m_collision_type == CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion)) + { + m_num_product_species = 2; + m_num_products_host.push_back(1); + m_num_products_host.push_back(1); #ifndef AMREX_USE_GPU - // On CPU, the device vector can be filled immediately - m_num_products_device.push_back(1); - m_num_products_device.push_back(1); + // On CPU, the device vector can be filled immediately + m_num_products_device.push_back(1); + m_num_products_device.push_back(1); #endif - } - else - { - WARPX_ABORT_WITH_MESSAGE("Unknown collision type in ParticleCreationFunc"); - } + } + else + { + WARPX_ABORT_WITH_MESSAGE("Unknown collision type in ParticleCreationFunc"); + } #ifdef AMREX_USE_GPU - m_num_products_device.resize(m_num_product_species); - amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_num_products_host.begin(), - m_num_products_host.end(), - m_num_products_device.begin()); - amrex::Gpu::streamSynchronize(); + m_num_products_device.resize(m_num_product_species); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_num_products_host.begin(), + m_num_products_host.end(), + m_num_products_device.begin()); + amrex::Gpu::streamSynchronize(); #endif - } +} diff --git a/Source/Particles/Collision/CMakeLists.txt b/Source/Particles/Collision/CMakeLists.txt index f3183032179..62eb3b3ad9c 100644 --- a/Source/Particles/Collision/CMakeLists.txt +++ b/Source/Particles/Collision/CMakeLists.txt @@ -4,6 +4,7 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE CollisionHandler.cpp CollisionBase.cpp + ScatteringProcess.cpp ) endforeach() diff --git a/Source/Particles/Collision/CollisionHandler.cpp b/Source/Particles/Collision/CollisionHandler.cpp index 213e5edd577..5abee34c86a 100644 --- a/Source/Particles/Collision/CollisionHandler.cpp +++ b/Source/Particles/Collision/CollisionHandler.cpp @@ -10,6 +10,7 @@ #include "Particles/Collision/BackgroundStopping/BackgroundStopping.H" #include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" #include "Particles/Collision/BinaryCollision/BinaryCollision.H" +#include "Particles/Collision/BinaryCollision/DSMC/DSMC.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" #include "Utils/TextMsg.H" @@ -52,6 +53,9 @@ CollisionHandler::CollisionHandler(MultiParticleContainer const * const mypc) else if (type == "background_stopping") { allcollisions[i] = std::make_unique(collision_names[i]); } + else if (type == "dsmc") { + allcollisions[i] = std::make_unique(collision_names[i]); + } else if (type == "nuclearfusion") { allcollisions[i] = std::make_unique>( diff --git a/Source/Particles/Collision/Make.package b/Source/Particles/Collision/Make.package index e9aad3691a3..9ec153879f4 100644 --- a/Source/Particles/Collision/Make.package +++ b/Source/Particles/Collision/Make.package @@ -1,5 +1,6 @@ CEXE_sources += CollisionHandler.cpp CEXE_sources += CollisionBase.cpp +CEXE_sources += ScatteringProcess.cpp include $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/Make.package include $(WARPX_HOME)/Source/Particles/Collision/BackgroundMCC/Make.package diff --git a/Source/Particles/Collision/BackgroundMCC/MCCProcess.H b/Source/Particles/Collision/ScatteringProcess.H similarity index 81% rename from Source/Particles/Collision/BackgroundMCC/MCCProcess.H rename to Source/Particles/Collision/ScatteringProcess.H index 9b844959e52..d05ea32e2fe 100644 --- a/Source/Particles/Collision/BackgroundMCC/MCCProcess.H +++ b/Source/Particles/Collision/ScatteringProcess.H @@ -1,18 +1,20 @@ -/* Copyright 2021 Modern Electron +/* Copyright 2021-2023 The WarpX Community * * This file is part of WarpX. * + * Authors: Modern Electron, Roelof Groenewald (TAE Technologies) + * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ -#define WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ +#ifndef WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ +#define WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ #include #include #include #include -enum class MCCProcessType { +enum class ScatteringProcessType { INVALID, ELASTIC, BACK, @@ -21,29 +23,29 @@ enum class MCCProcessType { IONIZATION, }; -class MCCProcess +class ScatteringProcess { public: - MCCProcess ( + ScatteringProcess ( const std::string& scattering_process, const std::string& cross_section_file, amrex::ParticleReal energy ); template - MCCProcess ( + ScatteringProcess ( const std::string& scattering_process, const InputVector&& energies, const InputVector&& sigmas, amrex::ParticleReal energy ); - ~MCCProcess() = default; + ~ScatteringProcess() = default; - MCCProcess (MCCProcess const&) = delete; - MCCProcess& operator= (MCCProcess const&) = delete; - MCCProcess (MCCProcess &&) = default; - MCCProcess& operator= (MCCProcess &&) = default; + ScatteringProcess (ScatteringProcess const&) = delete; + ScatteringProcess& operator= (ScatteringProcess const&) = delete; + ScatteringProcess (ScatteringProcess &&) = default; + ScatteringProcess& operator= (ScatteringProcess &&) = default; /** Read the given cross-section data file to memory. * @@ -99,7 +101,7 @@ public: amrex::ParticleReal* m_sigmas_data = nullptr; amrex::ParticleReal m_energy_lo, m_energy_hi, m_sigma_lo, m_sigma_hi, m_dE; amrex::ParticleReal m_energy_penalty; - MCCProcessType m_type; + ScatteringProcessType m_type; }; [[nodiscard]] @@ -121,12 +123,12 @@ public: [[nodiscard]] amrex::ParticleReal getMaxEnergyInput () const { return m_exe_h.m_energy_hi; } [[nodiscard]] amrex::ParticleReal getEnergyInputStep () const { return m_exe_h.m_dE; } - [[nodiscard]] MCCProcessType type () const { return m_exe_h.m_type; } + [[nodiscard]] ScatteringProcessType type () const { return m_exe_h.m_type; } private: static - MCCProcessType parseProcessType(const std::string& process); + ScatteringProcessType parseProcessType(const std::string& process); void init (const std::string& scattering_process, amrex::ParticleReal energy); @@ -142,4 +144,4 @@ private: int m_grid_size; }; -#endif // WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ +#endif // WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ diff --git a/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp b/Source/Particles/Collision/ScatteringProcess.cpp similarity index 83% rename from Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp rename to Source/Particles/Collision/ScatteringProcess.cpp index 458b3c136a3..c15d11eea07 100644 --- a/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp +++ b/Source/Particles/Collision/ScatteringProcess.cpp @@ -1,15 +1,17 @@ -/* Copyright 2021 Modern Electron +/* Copyright 2021-2023 The WarpX Community * * This file is part of WarpX. * + * Authors: Modern Electron, Roelof Groenewald (TAE Technologies) + * * License: BSD-3-Clause-LBNL */ -#include "MCCProcess.H" +#include "ScatteringProcess.H" #include "Utils/TextMsg.H" #include "WarpX.H" -MCCProcess::MCCProcess ( +ScatteringProcess::ScatteringProcess ( const std::string& scattering_process, const std::string& cross_section_file, const amrex::ParticleReal energy ) @@ -21,7 +23,7 @@ MCCProcess::MCCProcess ( } template -MCCProcess::MCCProcess ( +ScatteringProcess::ScatteringProcess ( const std::string& scattering_process, const InputVector&& energies, const InputVector&& sigmas, @@ -34,7 +36,7 @@ MCCProcess::MCCProcess ( } void -MCCProcess::init (const std::string& scattering_process, const amrex::ParticleReal energy) +ScatteringProcess::init (const std::string& scattering_process, const amrex::ParticleReal energy) { using namespace amrex::literals; m_exe_h.m_sigmas_data = m_sigmas_h.data(); @@ -72,26 +74,26 @@ MCCProcess::init (const std::string& scattering_process, const amrex::ParticleRe #endif } -MCCProcessType -MCCProcess::parseProcessType(const std::string& scattering_process) +ScatteringProcessType +ScatteringProcess::parseProcessType(const std::string& scattering_process) { if (scattering_process == "elastic") { - return MCCProcessType::ELASTIC; + return ScatteringProcessType::ELASTIC; } else if (scattering_process == "back") { - return MCCProcessType::BACK; + return ScatteringProcessType::BACK; } else if (scattering_process == "charge_exchange") { - return MCCProcessType::CHARGE_EXCHANGE; + return ScatteringProcessType::CHARGE_EXCHANGE; } else if (scattering_process == "ionization") { - return MCCProcessType::IONIZATION; + return ScatteringProcessType::IONIZATION; } else if (scattering_process.find("excitation") != std::string::npos) { - return MCCProcessType::EXCITATION; + return ScatteringProcessType::EXCITATION; } else { - return MCCProcessType::INVALID; + return ScatteringProcessType::INVALID; } } void -MCCProcess::readCrossSectionFile ( +ScatteringProcess::readCrossSectionFile ( const std::string cross_section_file, amrex::Vector& energies, amrex::Gpu::HostVector& sigmas ) @@ -109,7 +111,7 @@ MCCProcess::readCrossSectionFile ( } void -MCCProcess::sanityCheckEnergyGrid ( +ScatteringProcess::sanityCheckEnergyGrid ( const amrex::Vector& energies, amrex::ParticleReal dE ) From d318a76f5e31feae43ff74001ac1aef2ae2ea5bf Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 22 Dec 2023 17:02:11 +0100 Subject: [PATCH 079/176] Fix: BeamRelevant 1D, 2D (#4558) The `BeamRelevant` diagnostics in 1D and 2D predicted wrong average positions for the symmetry dimensions. --- Source/Diagnostics/ReducedDiags/BeamRelevant.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index d2a55a4671a..fbe67ba6fc5 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -214,14 +214,21 @@ void BeamRelevant::ComputeDiags (int step) const ParticleReal p_pos0 = p.pos(0); const ParticleReal p_w = p.rdata(PIdx::w); -#if (defined WARPX_DIM_RZ) +#if defined(WARPX_DIM_3D) + const ParticleReal p_pos1 = p.pos(1); + const ParticleReal p_x_mean = p_pos0*p_w; + const ParticleReal p_y_mean = p_pos1*p_w; +#elif defined(WARPX_DIM_RZ) const ParticleReal p_theta = p.rdata(PIdx::theta); const ParticleReal p_x_mean = p_pos0*std::cos(p_theta)*p_w; const ParticleReal p_y_mean = p_pos0*std::sin(p_theta)*p_w; -#else - const ParticleReal p_pos1 = p.pos(1); +#elif defined(WARPX_DIM_XZ) const ParticleReal p_x_mean = p_pos0*p_w; - const ParticleReal p_y_mean = p_pos1*p_w; + const ParticleReal p_y_mean = 0; +#elif defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(p_pos0); + const ParticleReal p_x_mean = 0; + const ParticleReal p_y_mean = 0; #endif const ParticleReal p_z_mean = p.pos(index_z)*p_w; From 220908726376a96f1fe9913e75d106342bbdaa5f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 22 Dec 2023 16:04:36 -0800 Subject: [PATCH 080/176] Update documentation for checksum (#4561) --- Docs/source/developers/checksum.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Docs/source/developers/checksum.rst b/Docs/source/developers/checksum.rst index 1099d68dccf..4412c3e3cc4 100644 --- a/Docs/source/developers/checksum.rst +++ b/Docs/source/developers/checksum.rst @@ -32,7 +32,7 @@ You can execute ``checksumAPI.py`` as a Python script for that, and pass the plo .. code-block:: bash - ./checksumAPI.py --evaluate --plotfile --test-name + ./checksumAPI.py --evaluate --output-file --format <'openpmd' or 'plotfile'> --test-name See additional options @@ -41,17 +41,17 @@ See additional options * ``--rtol`` relative tolerance for the comparison * ``--atol`` absolute tolerance for the comparison (a sum of both is used by ``numpy.isclose()``) -Reset a benchmark with new values that you know are correct ------------------------------------------------------------ +Create/Reset a benchmark with new values that you know are correct +------------------------------------------------------------------ -Reset a benchmark from a plotfile generated locally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Create/Reset a benchmark from a plotfile generated locally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This is using ``checksumAPI.py`` as a Python script. .. code-block:: bash - ./checksumAPI.py --reset-benchmark --plotfile --test-name + ./checksumAPI.py --reset-benchmark --output-file --format <'openpmd' or 'plotfile'> --test-name See additional options From a32b56138d0abb1397dbb9dbf3ba6c6a67bb15c1 Mon Sep 17 00:00:00 2001 From: David Grote Date: Fri, 22 Dec 2023 17:46:07 -0800 Subject: [PATCH 081/176] Implicit electromagnetic solver using Picard iterations (#4071) * Added evolve_scheme parameter * Add implicit advance using Picard iteration * Added WarpXEvolveImplicitPicard.cpp to CMakeLists * Fix push for 1D and 2D * Add Langmuir_multi_implicit_picard CI test * Add n_picard_iterations * Further changes - now the solver is working (mostly) * Fix some floats * Fix default value of relative_time * Major update, including charge conserving scheme * Bug fix in CurrentDeposition.H * Fix FieldGather for 2D for By * Clean up print statements * Fix B field gather in 1D * Only add particle attributes if initializing level zero * Set nodel_sync=true, fixes instabilities * Only calculate interation convergence when needed * Fix KernelTimer in CurrentDeposition.H * Use WarpX::sync_nodal_points * Update doc for evolve_scheme * Update ImplicitPushXP to match PushPX * Add RZ version * Various fixes * Add some ignore_unused statements * Fixes to match changes from merge * Add reference to paper in doc * Updated paper reference in doc * Add SemiImplicitPicard scheme * Fixes needed after merge * Use parser to query picard iteration parameters * Remove unneeded const on scalar arguments * More fixes for clang-tidy * Remove unneeded print statements * More clang-tidy cleanup * Bug fix, forgotten declaration * Hopefully one last clang-tidy fix * Update RZ CI tests modified by fixes to fields along the axis * Add 1d CI test * Clean up for clang-tidy * Update benchmarks for CI test * Small fix for QED and CUDA * Updates to the documentation * Cleanup parameter input * Move implicit evolve into the main evolve routine * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add more checks of input parameters * Rename WarpXEvolveImplicitPicard.cpp to WarpXOneStepImplicitPicard.cpp * Clean of one step file * Remove commented code * Clean up from merge * Remove commented code * Add picard_iteration_tolerance flag * Fix clang-tidy issue * clang-tidy fixes * Update Docs/source/usage/parameters.rst Fix Delta Co-authored-by: Remi Lehe * Update Docs/source/usage/parameters.rst Co-authored-by: Remi Lehe * Update Docs/source/usage/parameters.rst Co-authored-by: Remi Lehe * Update Docs/source/usage/parameters.rst Co-authored-by: Remi Lehe * With Redistribute, only nearest neighbor exchanges are needed * Update documentation * Changed name to SaveParticlesAtImplicitStepStart * Update comments in UpdatePosition.H * Abort if Vay deposition is used * Change setup to require energy-conserving and setting galerkin_scheme * Update documentation for Evolve routine * Undid change to setting aux_is_nodal * Add comments to PushType * Update comment for ImplicitPushXP * More updates to comment for ImplicitPushXP * Fix setting of external fields in ImplicitPushXP * Add comment to doDepositionShapeNImplicit * Updates to charge conserving gather * Renamed implicit field gather routine with Esirkepov stencil * Update naming and comments in doGatherShapeNImplicit --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Remi Lehe --- Docs/source/usage/parameters.rst | 37 + Examples/Tests/Implicit/analysis_1d.py | 39 + Examples/Tests/Implicit/inputs_1d | 81 ++ .../benchmarks_json/ImplicitPicard_1d.json | 29 + Regression/WarpX-tests.ini | 17 + Source/Evolve/CMakeLists.txt | 1 + Source/Evolve/Make.package | 1 + Source/Evolve/WarpXEvolve.cpp | 146 +-- Source/Evolve/WarpXOneStepImplicitPicard.cpp | 423 ++++++++ Source/Evolve/WarpXPushType.H | 18 + .../Particles/Deposition/CurrentDeposition.H | 960 ++++++++++++++---- Source/Particles/Gather/FieldGather.H | 521 ++++++++++ Source/Particles/LaserParticleContainer.H | 3 +- Source/Particles/LaserParticleContainer.cpp | 7 +- Source/Particles/MultiParticleContainer.H | 7 +- Source/Particles/MultiParticleContainer.cpp | 5 +- Source/Particles/PhotonParticleContainer.H | 7 +- Source/Particles/PhotonParticleContainer.cpp | 5 +- Source/Particles/PhysicalParticleContainer.H | 19 +- .../Particles/PhysicalParticleContainer.cpp | 341 ++++++- Source/Particles/Pusher/PushSelector.H | 66 +- Source/Particles/Pusher/UpdatePosition.H | 44 +- .../RigidInjectedParticleContainer.H | 4 +- .../RigidInjectedParticleContainer.cpp | 5 +- Source/Particles/ShapeFactors.H | 10 +- Source/Particles/WarpXParticleContainer.H | 7 +- Source/Particles/WarpXParticleContainer.cpp | 128 ++- Source/Utils/WarpXAlgorithmSelection.H | 11 + Source/Utils/WarpXAlgorithmSelection.cpp | 11 +- Source/WarpX.H | 48 +- Source/WarpX.cpp | 66 ++ 31 files changed, 2702 insertions(+), 365 deletions(-) create mode 100755 Examples/Tests/Implicit/analysis_1d.py create mode 100644 Examples/Tests/Implicit/inputs_1d create mode 100644 Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json create mode 100644 Source/Evolve/WarpXOneStepImplicitPicard.cpp create mode 100644 Source/Evolve/WarpXPushType.H diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 3ff9c4e62ee..88d8b4dbe6f 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -81,6 +81,43 @@ Overall simulation parameters one should not expect to obtain the same random numbers, even if a fixed ``warpx.random_seed`` is provided. +* ``algo.evolve_scheme`` (`string`, default: `explicit`) + Specifies the evolve scheme used by WarpX. + + * ``explicit``: Use an explicit solver, such as the standard FDTD or PSATD + + * ``implicit_picard``: Use an implicit solver with exact energy conservation that uses a Picard iteration to solve the system. + Note that this method is for demonstration only. It is inefficient and does not work well when + :math:`\omega_{pe} \Delta t` is close to or greater than one. + The method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. + The version implemented is an updated version that is relativistically correct, including the relativistic gamma factor for the particles. + For exact energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, + and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in + which case charge will also be conserved). + + * ``semi_implicit_picard``: Use an energy conserving semi-implicit solver that uses a Picard iteration to solve the system. + Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. It is inefficient and does not work well or at all when :math:`\omega_{pe} \Delta t` is close to or greater than one. + The method is described in `Chen et al., A semi-implicit, energy- and charge-conserving particle-in-cell algorithm for the relativistic Vlasov-Maxwell equations `__. + For energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, + and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in + which case charge will also be conserved). + +* ``algo.max_picard_iterations`` (`integer`, default: 10) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the maximum number of Picard + itearations that are done each time step. + +* ``algo.picard_iteration_tolerance`` (`float`, default: 1.e-7) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the convergence tolerance of + the iterations, the maximum of the relative change of the L2 norm of the field from one iteration to the next. + If this is set to zero, the maximum number of iterations will always be done with the change only calculated on the last + iteration (for a slight optimization). + +* ``algo.require_picard_convergence`` (`bool`, default: 1) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets whether the iteration each step + is required to converge. + If it is required, an abort is raised if it does not converge and the code then exits. + If not, then a warning is issued and the calculation continues. + * ``warpx.do_electrostatic`` (`string`) optional (default `none`) Specifies the electrostatic mode. When turned on, instead of updating the fields at each iteration with the full Maxwell equations, the fields diff --git a/Examples/Tests/Implicit/analysis_1d.py b/Examples/Tests/Implicit/analysis_1d.py new file mode 100755 index 00000000000..3d4fc375bc9 --- /dev/null +++ b/Examples/Tests/Implicit/analysis_1d.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +# Copyright 2023 David Grote +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs_1d`. This simulates a 1D periodic plasma using the implicit solver. +import os +import sys + +import numpy as np + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +field_energy = np.loadtxt('diags/reducedfiles/field_energy.txt', skiprows=1) +particle_energy = np.loadtxt('diags/reducedfiles/particle_energy.txt', skiprows=1) + +total_energy = field_energy[:,2] + particle_energy[:,2] + +delta_E = (total_energy - total_energy[0])/total_energy[0] +max_delta_E = np.abs(delta_E).max() + +tolerance_rel = 1.e-14 + +print(f"max change in energy: {max_delta_E}") +print(f"tolerance: {tolerance_rel}") + +assert( max_delta_E < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/Implicit/inputs_1d b/Examples/Tests/Implicit/inputs_1d new file mode 100644 index 00000000000..50d28a2db75 --- /dev/null +++ b/Examples/Tests/Implicit/inputs_1d @@ -0,0 +1,81 @@ +################################# +############ CONSTANTS ############# +################################# + +my_constants.n0 = 1.e30 # plasma densirty, m^-3 +my_constants.nz = 40 # number of grid cells +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.nppcz = 100 # number of particles/cell in z +my_constants.dt = 0.1/wpe # time step size, s + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_step = 100 +amr.n_cell = nz +amr.max_level = 0 + +geometry.dims = 1 +geometry.prob_lo = 0.0 +geometry.prob_hi = 10.*de0 +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic + +################################# +############ NUMERICS ########### +################################# + +warpx.const_dt = dt +algo.evolve_scheme = implicit_picard +algo.max_picard_iterations = 31 +algo.picard_iteration_tolerance = 0. +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ PLASMA ############# +################################# + +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 100 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = w ux uy uz +diag1.protons.variables = w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json new file mode 100644 index 00000000000..cd4120002be --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 3730.0029376363264, + "By": 1593.5906541698305, + "Bz": 0.0, + "Ex": 797541065253.7858, + "Ey": 981292393404.0359, + "Ez": 3528134993266.091, + "divE": 2.0829069134855788e+21, + "jx": 7.639291641209293e+17, + "jy": 1.4113587963237038e+18, + "jz": 1.3506587033985085e+18, + "rho": 18442449008.581665 + }, + "protons": { + "particle_momentum_x": 5.231105747759245e-19, + "particle_momentum_y": 5.367982834807453e-19, + "particle_momentum_z": 5.253213507906386e-19, + "particle_position_x": 0.00010628272743703996, + "particle_weight": 5.314093261582036e+22 + }, + "electrons": { + "particle_momentum_x": 1.196379551301037e-20, + "particle_momentum_y": 1.2271443795645239e-20, + "particle_momentum_z": 1.2277752539495415e-20, + "particle_position_x": 0.00010649569055433632, + "particle_weight": 5.314093261582036e+22 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 3fba702fbdc..da05ed4d5bc 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4462,3 +4462,20 @@ compileTest = 0 doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/nodal_electrostatic/analysis_3d.py + +[ImplicitPicard_1d] +buildDir = . +inputFile = Examples/Tests/Implicit/inputs_1d +runtime_params = warpx.abort_on_warning_threshold=high +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_1d.py diff --git a/Source/Evolve/CMakeLists.txt b/Source/Evolve/CMakeLists.txt index 3ef0dae5e8e..a84d6e1e42b 100644 --- a/Source/Evolve/CMakeLists.txt +++ b/Source/Evolve/CMakeLists.txt @@ -4,5 +4,6 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE WarpXEvolve.cpp WarpXComputeDt.cpp + WarpXOneStepImplicitPicard.cpp ) endforeach() diff --git a/Source/Evolve/Make.package b/Source/Evolve/Make.package index 275b8bfde5a..a9a877475b9 100644 --- a/Source/Evolve/Make.package +++ b/Source/Evolve/Make.package @@ -1,4 +1,5 @@ CEXE_sources += WarpXEvolve.cpp CEXE_sources += WarpXComputeDt.cpp +CEXE_sources += WarpXOneStepImplicitPicard.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Evolve diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 119dfab5d99..b062501ed1b 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -118,48 +118,50 @@ WarpX::Evolve (int numsteps) } } - // At the beginning, we have B^{n} and E^{n}. - // Particles have p^{n} and x^{n}. - // is_synchronized is true. - if (is_synchronized) { - if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { - // Not called at each iteration, so exchange all guard cells - FillBoundaryE(guard_cells.ng_alloc_EB); - FillBoundaryB(guard_cells.ng_alloc_EB); - } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - // on first step, push p by -0.5*dt - for (int lev = 0; lev <= finest_level; ++lev) - { - mypc->PushP(lev, -0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); - } - is_synchronized = false; - - } else { - if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { - // Beyond one step, we have E^{n} and B^{n}. - // Particles have p^{n-1/2} and x^{n}. - - // E and B are up-to-date inside the domain only - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - // E and B: enough guard cells to update Aux or call Field Gather in fp and cp - // Need to update Aux on lower levels, to interpolate to higher levels. - if (fft_do_time_averaging) + if (evolve_scheme == EvolveScheme::Explicit) { + // At the beginning, we have B^{n} and E^{n}. + // Particles have p^{n} and x^{n}. + // is_synchronized is true. + if (is_synchronized) { + if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + // Not called at each iteration, so exchange all guard cells + FillBoundaryE(guard_cells.ng_alloc_EB); + FillBoundaryB(guard_cells.ng_alloc_EB); + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + // on first step, push p by -0.5*dt + for (int lev = 0; lev <= finest_level; ++lev) { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); + mypc->PushP(lev, -0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); } - // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - FillBoundaryAux(guard_cells.ng_UpdateAux); + is_synchronized = false; + + } else { + if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + // Beyond one step, we have E^{n} and B^{n}. + // Particles have p^{n-1/2} and x^{n}. + + // E and B are up-to-date inside the domain only + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + // E and B: enough guard cells to update Aux or call Field Gather in fp and cp + // Need to update Aux on lower levels, to interpolate to higher levels. + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + FillBoundaryAux(guard_cells.ng_UpdateAux); + } } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); } // If needed, deposit the initial ion charge and current densities that @@ -183,11 +185,16 @@ WarpX::Evolve (int numsteps) // gather fields, push particles, deposit sources, update fields ExecutePythonCallback("particleinjection"); - // Electrostatic or hybrid-PIC case: only gather fields and push - // particles, deposition and calculation of fields done further below - if ( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || + + if (evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard) { + OneStep_ImplicitPicard(cur_time); + } + else if ( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) { + // Electrostatic or hybrid-PIC case: only gather fields and push + // particles, deposition and calculation of fields done further below const bool skip_deposition = true; PushParticlesandDeposit(cur_time, skip_deposition); } @@ -221,31 +228,33 @@ WarpX::Evolve (int numsteps) // value of step in code (first step is 0) mypc->doResampling(istep[0]+1, verbose); - if (num_mirrors>0){ - applyMirrors(cur_time); - // E : guard cells are NOT up-to-date - // B : guard cells are NOT up-to-date - } - - if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { - // At the end of last step, push p by 0.5*dt to synchronize - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - if (fft_do_time_averaging) - { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); + if (evolve_scheme == EvolveScheme::Explicit) { + if (num_mirrors>0){ + applyMirrors(cur_time); + // E : guard cells are NOT up-to-date + // B : guard cells are NOT up-to-date } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - for (int lev = 0; lev <= finest_level; ++lev) { - mypc->PushP(lev, 0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1], - *Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1], - *Bfield_aux[lev][2]); + + if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { + // At the end of last step, push p by 0.5*dt to synchronize + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + for (int lev = 0; lev <= finest_level; ++lev) { + mypc->PushP(lev, 0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1], + *Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1], + *Bfield_aux[lev][2]); + } + is_synchronized = true; } - is_synchronized = true; } for (int lev = 0; lev <= max_level; ++lev) { @@ -963,17 +972,18 @@ WarpX::doQEDEvents (int lev) #endif void -WarpX::PushParticlesandDeposit (amrex::Real cur_time, bool skip_current) +WarpX::PushParticlesandDeposit (amrex::Real cur_time, bool skip_current, PushType push_type) { // Evolve particles to p^{n+1/2} and x^{n+1} // Deposit current, j^{n+1/2} for (int lev = 0; lev <= finest_level; ++lev) { - PushParticlesandDeposit(lev, cur_time, DtType::Full, skip_current); + PushParticlesandDeposit(lev, cur_time, DtType::Full, skip_current, push_type); } } void -WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_current) +WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_current, + PushType push_type) { amrex::MultiFab* current_x = nullptr; amrex::MultiFab* current_y = nullptr; @@ -1007,7 +1017,7 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, rho_fp[lev].get(), charge_buf[lev].get(), Efield_cax[lev][0].get(), Efield_cax[lev][1].get(), Efield_cax[lev][2].get(), Bfield_cax[lev][0].get(), Bfield_cax[lev][1].get(), Bfield_cax[lev][2].get(), - cur_time, dt[lev], a_dt_type, skip_current); + cur_time, dt[lev], a_dt_type, skip_current, push_type); if (! skip_current) { #ifdef WARPX_DIM_RZ // This is called after all particles have deposited their current and charge. diff --git a/Source/Evolve/WarpXOneStepImplicitPicard.cpp b/Source/Evolve/WarpXOneStepImplicitPicard.cpp new file mode 100644 index 00000000000..56ed26db1e2 --- /dev/null +++ b/Source/Evolve/WarpXOneStepImplicitPicard.cpp @@ -0,0 +1,423 @@ +/* Copyright 2022 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "WarpX.H" + +#include "BoundaryConditions/PML.H" +#include "Diagnostics/MultiDiagnostics.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" +#ifdef WARPX_USE_PSATD +# ifdef WARPX_DIM_RZ +# include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" +# else +# include "FieldSolver/SpectralSolver/SpectralSolver.H" +# endif +#endif +#include "Parallelization/GuardCellManager.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleBoundaryBuffer.H" +#include "Python/callbacks.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" +#include "Utils/WarpXUtil.H" +#include "Utils/WarpXConst.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +void +WarpX::EvolveImplicitPicardInit (int lev) +{ + + if (lev == 0) { + // Add space to save the positions and velocities at the start of the time steps + for (auto const& pc : *mypc) { +#if (AMREX_SPACEDIM >= 2) + pc->AddRealComp("x_n"); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + pc->AddRealComp("y_n"); +#endif + pc->AddRealComp("z_n"); + pc->AddRealComp("ux_n"); + pc->AddRealComp("uy_n"); + pc->AddRealComp("uz_n"); + } + } + + // Initialize MultiFabs to hold the E and B fields at the start of the time steps + // Only one refinement level is supported + const int nlevs_max = maxLevel() + 1; + Efield_n.resize(nlevs_max); + Efield_save.resize(nlevs_max); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + Bfield_n.resize(nlevs_max); + Bfield_save.resize(nlevs_max); + } + + // The Efield_n and Bfield_n will hold the fields at the start of the time step. + // This is needed since in each iteration the fields are advanced from the values + // at the start of the step. + // The Efield_save and Bfield_save will hold the fields from the previous iteration, + // to check the change in the fields after the iterations to check for convergence. + // The Efiel_fp and Bfield_fp will hole the n+theta during the iterations and then + // advance to the n+1 time level after the iterations complete. + AllocInitMultiFabFromModel(Efield_n[lev][0], *Efield_fp[0][0], lev, "Efield_n[0]"); + AllocInitMultiFabFromModel(Efield_n[lev][1], *Efield_fp[0][1], lev, "Efield_n[1]"); + AllocInitMultiFabFromModel(Efield_n[lev][2], *Efield_fp[0][2], lev, "Efield_n[2]"); + AllocInitMultiFabFromModel(Efield_save[lev][0], *Efield_fp[0][0], lev, "Efield_save[0]"); + AllocInitMultiFabFromModel(Efield_save[lev][1], *Efield_fp[0][1], lev, "Efield_save[1]"); + AllocInitMultiFabFromModel(Efield_save[lev][2], *Efield_fp[0][2], lev, "Efield_save[2]"); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + AllocInitMultiFabFromModel(Bfield_n[lev][0], *Bfield_fp[0][0], lev, "Bfield_n[0]"); + AllocInitMultiFabFromModel(Bfield_n[lev][1], *Bfield_fp[0][1], lev, "Bfield_n[1]"); + AllocInitMultiFabFromModel(Bfield_n[lev][2], *Bfield_fp[0][2], lev, "Bfield_n[2]"); + AllocInitMultiFabFromModel(Bfield_save[lev][0], *Bfield_fp[0][0], lev, "Bfield_save[0]"); + AllocInitMultiFabFromModel(Bfield_save[lev][1], *Bfield_fp[0][1], lev, "Bfield_save[1]"); + AllocInitMultiFabFromModel(Bfield_save[lev][2], *Bfield_fp[0][2], lev, "Bfield_save[2]"); + } + +} + +void +WarpX::OneStep_ImplicitPicard(amrex::Real cur_time) +{ + using namespace amrex::literals; + + // We have E^{n}. + // Particles have p^{n} and x^{n}. + // With full implicit, B^{n} + // With semi-implicit, B^{n-1/2} + + // Save the values at the start of the time step, + // copying particle data to x_n etc. + for (auto const& pc : *mypc) { + SaveParticlesAtImplicitStepStart (*pc, 0); + } + + // Save the fields at the start of the step + amrex::MultiFab::Copy(*Efield_n[0][0], *Efield_fp[0][0], 0, 0, ncomps, Efield_fp[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_n[0][1], *Efield_fp[0][1], 0, 0, ncomps, Efield_fp[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_n[0][2], *Efield_fp[0][2], 0, 0, ncomps, Efield_fp[0][2]->nGrowVect()); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + amrex::MultiFab::Copy(*Bfield_n[0][0], *Bfield_fp[0][0], 0, 0, ncomps, Bfield_fp[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_n[0][1], *Bfield_fp[0][1], 0, 0, ncomps, Bfield_fp[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_n[0][2], *Bfield_fp[0][2], 0, 0, ncomps, Bfield_fp[0][2]->nGrowVect()); + } else if (evolve_scheme == EvolveScheme::SemiImplicitPicard) { + // This updates Bfield_fp so it holds the new B at n+1/2 + EvolveB(dt[0], DtType::Full); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); + } + + // Start the iterations + amrex::Real deltaE = 1._rt; + amrex::Real deltaB = 1._rt; + int iteration_count = 0; + while (iteration_count < max_picard_iterations && + (deltaE > picard_iteration_tolerance || deltaB > picard_iteration_tolerance)) { + iteration_count++; + + // Advance the particle positions by 1/2 dt, + // particle velocities by dt, then take average of old and new v, + // deposit currents, giving J at n+1/2 + // This uses Efield_fp and Bfield_fp, the field at n+1/2 from the previous iteration. + bool skip_current = false; + PushType push_type = PushType::Implicit; + PushParticlesandDeposit(cur_time, skip_current, push_type); + + SyncCurrentAndRho(); + + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Save the E at n+1/2 from the previous iteration so that the change + // in this iteration can be calculated + amrex::MultiFab::Copy(*Efield_save[0][0], *Efield_fp[0][0], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Efield_save[0][1], *Efield_fp[0][1], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Efield_save[0][2], *Efield_fp[0][2], 0, 0, ncomps, 0); + } + + // Copy Efield_n into Efield_fp since EvolveE updates Efield_fp in place + amrex::MultiFab::Copy(*Efield_fp[0][0], *Efield_n[0][0], 0, 0, ncomps, Efield_n[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][1], *Efield_n[0][1], 0, 0, ncomps, Efield_n[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][2], *Efield_n[0][2], 0, 0, ncomps, Efield_n[0][2]->nGrowVect()); + + // Updates Efield_fp so it holds the new E at n+1/2 + EvolveE(0.5_rt*dt[0]); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyEfieldBoundary(0, PatchType::fine); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Save the B at n+1/2 from the previous iteration so that the change + // in this iteration can be calculated + amrex::MultiFab::Copy(*Bfield_save[0][0], *Bfield_fp[0][0], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Bfield_save[0][1], *Bfield_fp[0][1], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Bfield_save[0][2], *Bfield_fp[0][2], 0, 0, ncomps, 0); + } + + // Copy Bfield_n into Bfield_fp since EvolveB updates Bfield_fp in place + amrex::MultiFab::Copy(*Bfield_fp[0][0], *Bfield_n[0][0], 0, 0, ncomps, Bfield_n[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][1], *Bfield_n[0][1], 0, 0, ncomps, Bfield_n[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][2], *Bfield_n[0][2], 0, 0, ncomps, Bfield_n[0][2]->nGrowVect()); + + // This updates Bfield_fp so it holds the new B at n+1/2 + EvolveB(0.5_rt*dt[0], DtType::Full); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); + } + + // The B field update needs + if (num_mirrors>0){ + applyMirrors(cur_time); + // E : guard cells are NOT up-to-date from the mirrors + // B : guard cells are NOT up-to-date from the mirrors + } + + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Calculate the change in E and B from this iteration + // deltaE = abs(Enew - Eold)/max(abs(Enew)) + Efield_save[0][0]->minus(*Efield_fp[0][0], 0, ncomps, 0); + Efield_save[0][1]->minus(*Efield_fp[0][1], 0, ncomps, 0); + Efield_save[0][2]->minus(*Efield_fp[0][2], 0, ncomps, 0); + amrex::Real maxE0 = std::max(1._rt, Efield_fp[0][0]->norm0(0, 0)); + amrex::Real maxE1 = std::max(1._rt, Efield_fp[0][1]->norm0(0, 0)); + amrex::Real maxE2 = std::max(1._rt, Efield_fp[0][2]->norm0(0, 0)); + amrex::Real deltaE0 = Efield_save[0][0]->norm0(0, 0)/maxE0; + amrex::Real deltaE1 = Efield_save[0][1]->norm0(0, 0)/maxE1; + amrex::Real deltaE2 = Efield_save[0][2]->norm0(0, 0)/maxE2; + deltaE = std::max(std::max(deltaE0, deltaE1), deltaE2); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + Bfield_save[0][0]->minus(*Bfield_fp[0][0], 0, ncomps, 0); + Bfield_save[0][1]->minus(*Bfield_fp[0][1], 0, ncomps, 0); + Bfield_save[0][2]->minus(*Bfield_fp[0][2], 0, ncomps, 0); + amrex::Real maxB0 = std::max(1._rt, Bfield_fp[0][0]->norm0(0, 0)); + amrex::Real maxB1 = std::max(1._rt, Bfield_fp[0][1]->norm0(0, 0)); + amrex::Real maxB2 = std::max(1._rt, Bfield_fp[0][2]->norm0(0, 0)); + amrex::Real deltaB0 = Bfield_save[0][0]->norm0(0, 0)/maxB0; + amrex::Real deltaB1 = Bfield_save[0][1]->norm0(0, 0)/maxB1; + amrex::Real deltaB2 = Bfield_save[0][2]->norm0(0, 0)/maxB2; + deltaB = std::max(std::max(deltaB0, deltaB1), deltaB2); + } else { + deltaB = 0.; + } + amrex::Print() << "Max delta " << iteration_count << " " << deltaE << " " << deltaB << "\n"; + } + + // Now, the particle positions and velocities and the Efield_fp and Bfield_fp hold + // the new values at n+1/2 + } + + amrex::Print() << "Picard iterations = " << iteration_count << ", Eerror = " << deltaE << ", Berror = " << deltaB << "\n"; + if (picard_iteration_tolerance > 0. && iteration_count == max_picard_iterations) { + std::stringstream convergenceMsg; + convergenceMsg << "The Picard implicit solver failed to converge after " << iteration_count << " iterations, with Eerror = " << deltaE << ", Berror = " << deltaB << " with a tolerance of " << picard_iteration_tolerance; + if (require_picard_convergence) { + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } else { + ablastr::warn_manager::WMRecordWarning("PicardSolver", convergenceMsg.str()); + } + } + + // Advance particles to step n+1 + for (auto const& pc : *mypc) { + FinishImplicitParticleUpdate(*pc, 0); + } + + // Advance fields to step n+1 + // WarpX::sync_nodal_points is used to avoid instability + FinishImplicitFieldUpdate(Efield_fp, Efield_n); + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + FinishImplicitFieldUpdate(Bfield_fp, Bfield_n); + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + } + +} + +void +WarpX::SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev) +{ + +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc.getParticleComps(); + + for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + x_n[ip] = xp; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + y_n[ip] = yp; +#endif + z_n[ip] = zp; + + ux_n[ip] = ux[ip]; + uy_n[ip] = uy[ip]; + uz_n[ip] = uz[ip]; + + }); + + } + } +} + +void +WarpX::FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev) +{ + using namespace amrex::literals; + +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc.getParticleComps(); + + for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + const auto setPosition = SetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + xp = 2._rt*xp - x_n[ip]; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + yp = 2._rt*yp - y_n[ip]; +#endif + zp = 2._rt*zp - z_n[ip]; + + ux[ip] = 2._rt*ux[ip] - ux_n[ip]; + uy[ip] = 2._rt*uy[ip] - uy_n[ip]; + uz[ip] = 2._rt*uz[ip] - uz_n[ip]; + + setPosition(ip, xp, yp, zp); + }); + + } + } +} + +void +WarpX::FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Field_fp, + amrex::Vector, 3 > >& Field_n) +{ + using namespace amrex::literals; + + for (int lev = 0; lev <= finest_level; ++lev) { + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*Field_fp[lev][0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + + amrex::Array4 const& Fx = Field_fp[lev][0]->array(mfi); + amrex::Array4 const& Fy = Field_fp[lev][1]->array(mfi); + amrex::Array4 const& Fz = Field_fp[lev][2]->array(mfi); + + amrex::Array4 const& Fx_n = Field_n[lev][0]->array(mfi); + amrex::Array4 const& Fy_n = Field_n[lev][1]->array(mfi); + amrex::Array4 const& Fz_n = Field_n[lev][2]->array(mfi); + + amrex::Box tbx = mfi.tilebox(Field_fp[lev][0]->ixType().toIntVect()); + amrex::Box tby = mfi.tilebox(Field_fp[lev][1]->ixType().toIntVect()); + amrex::Box tbz = mfi.tilebox(Field_fp[lev][2]->ixType().toIntVect()); + + amrex::ParallelFor( + tbx, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fx(i,j,k,n) = 2._rt*Fx(i,j,k,n) - Fx_n(i,j,k,n); + }, + tby, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fy(i,j,k,n) = 2._rt*Fy(i,j,k,n) - Fy_n(i,j,k,n); + }, + tbz, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fz(i,j,k,n) = 2._rt*Fz(i,j,k,n) - Fz_n(i,j,k,n); + }); + } + } +} diff --git a/Source/Evolve/WarpXPushType.H b/Source/Evolve/WarpXPushType.H new file mode 100644 index 00000000000..dbca64a398c --- /dev/null +++ b/Source/Evolve/WarpXPushType.H @@ -0,0 +1,18 @@ +/* Copyright 2022 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_PUSHTYPE_H_ +#define WARPX_PUSHTYPE_H_ + +// Specify which scheme to use for the particle advance +enum struct PushType : int +{ + Explicit = 0, // Use the standard leap-frog scheme + Implicit // Use the Crank-Nicolson scheme. + // See for example Eqs. 15-18 in Chen, JCP 407 (2020) 109228 +}; + +#endif // WARPX_PUSHTYPE_H_ diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index ac377a6799d..efe6efcc788 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -26,6 +26,246 @@ #include #include +/** + * \brief Kernel for the direct current deposition for thread thread_num + * \tparam depos_order deposition order + * \param xp, yp, zp The particle positions. + * \param wq The charge of the macroparticle + * \param vx,vy,vz The particle velocities + * \param jx_arr,jy_arr,jz_arr Array4 of current density, either full array or tile. + * \param jx_type,jy_type,jz_type The grid types along each direction, either NODE or CELL + * \param relative_time Time at which to deposit J, relative to the time of the + * current positions of the particles. When different than 0, + * the particle position will be temporarily modified to match + * the time of the deposition. + * \param dzi, dxi, dyi The inverse cell sizes + * \param zmin, xmin, ymin The lower bounds of the domain + * \param invvol The inverse volume of a grid cell + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void doDepositionShapeNKernel(const amrex::ParticleReal xp, + const amrex::ParticleReal yp, + const amrex::ParticleReal zp, + const amrex::ParticleReal wq, + const amrex::ParticleReal vx, + const amrex::ParticleReal vy, + const amrex::ParticleReal vz, + amrex::Array4 const& jx_arr, + amrex::Array4 const& jy_arr, + amrex::Array4 const& jz_arr, + amrex::IntVect const& jx_type, + amrex::IntVect const& jy_type, + amrex::IntVect const& jz_type, + const amrex::Real relative_time, + AMREX_D_DECL(const amrex::Real dzi, + const amrex::Real dxi, + const amrex::Real dyi), + AMREX_D_DECL(const amrex::Real zmin, + const amrex::Real xmin, + const amrex::Real ymin), + const amrex::Real invvol, + const amrex::Dim3 lo, + const int n_rz_azimuthal_modes) +{ + using namespace amrex::literals; +#if !defined(WARPX_DIM_RZ) + amrex::ignore_unused(n_rz_azimuthal_modes); +#endif +#if defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(xp, yp); +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ignore_unused(yp); +#endif + + constexpr int zdir = WARPX_ZINDEX; + constexpr int NODE = amrex::IndexType::NODE; + constexpr int CELL = amrex::IndexType::CELL; + + // wqx, wqy wqz are particle current in each direction +#if defined(WARPX_DIM_RZ) + // In RZ, wqx is actually wqr, and wqy is wqtheta + // Convert to cylindrical at the mid point + const amrex::Real xpmid = xp + relative_time*vx; + const amrex::Real ypmid = yp + relative_time*vy; + const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); + amrex::Real costheta; + amrex::Real sintheta; + if (rpmid > 0._rt) { + costheta = xpmid/rpmid; + sintheta = ypmid/rpmid; + } else { + costheta = 1._rt; + sintheta = 0._rt; + } + const Complex xy0 = Complex{costheta, sintheta}; + const amrex::Real wqx = wq*invvol*(+vx*costheta + vy*sintheta); + const amrex::Real wqy = wq*invvol*(-vx*sintheta + vy*costheta); +#else + const amrex::Real wqx = wq*invvol*vx; + const amrex::Real wqy = wq*invvol*vy; +#endif + const amrex::Real wqz = wq*invvol*vz; + + // --- Compute shape factors + Compute_shape_factor< depos_order > const compute_shape_factor; +#if (AMREX_SPACEDIM >= 2) + // x direction + // Get particle position after 1/2 push back in position +#if defined(WARPX_DIM_RZ) + // Keep these double to avoid bug in single precision + const double xmid = (rpmid - xmin)*dxi; +#else + const double xmid = ((xp - xmin) + relative_time*vx)*dxi; +#endif + // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current + // sx_j[xyz] shape factor along x for the centering of each current + // There are only two possible centerings, node or cell centered, so at most only two shape factor + // arrays will be needed. + // Keep these double to avoid bug in single precision + double sx_node[depos_order + 1] = {0.}; + double sx_cell[depos_order + 1] = {0.}; + int j_node = 0; + int j_cell = 0; + if (jx_type[0] == NODE || jy_type[0] == NODE || jz_type[0] == NODE) { + j_node = compute_shape_factor(sx_node, xmid); + } + if (jx_type[0] == CELL || jy_type[0] == CELL || jz_type[0] == CELL) { + j_cell = compute_shape_factor(sx_cell, xmid - 0.5); + } + + amrex::Real sx_jx[depos_order + 1] = {0._rt}; + amrex::Real sx_jy[depos_order + 1] = {0._rt}; + amrex::Real sx_jz[depos_order + 1] = {0._rt}; + for (int ix=0; ix<=depos_order; ix++) + { + sx_jx[ix] = ((jx_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + sx_jy[ix] = ((jy_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + sx_jz[ix] = ((jz_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + } + + int const j_jx = ((jx_type[0] == NODE) ? j_node : j_cell); + int const j_jy = ((jy_type[0] == NODE) ? j_node : j_cell); + int const j_jz = ((jz_type[0] == NODE) ? j_node : j_cell); +#endif //AMREX_SPACEDIM >= 2 + +#if defined(WARPX_DIM_3D) + // y direction + // Keep these double to avoid bug in single precision + const double ymid = ((yp - ymin) + relative_time*vy)*dyi; + double sy_node[depos_order + 1] = {0.}; + double sy_cell[depos_order + 1] = {0.}; + int k_node = 0; + int k_cell = 0; + if (jx_type[1] == NODE || jy_type[1] == NODE || jz_type[1] == NODE) { + k_node = compute_shape_factor(sy_node, ymid); + } + if (jx_type[1] == CELL || jy_type[1] == CELL || jz_type[1] == CELL) { + k_cell = compute_shape_factor(sy_cell, ymid - 0.5); + } + amrex::Real sy_jx[depos_order + 1] = {0._rt}; + amrex::Real sy_jy[depos_order + 1] = {0._rt}; + amrex::Real sy_jz[depos_order + 1] = {0._rt}; + for (int iy=0; iy<=depos_order; iy++) + { + sy_jx[iy] = ((jx_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + sy_jy[iy] = ((jy_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + sy_jz[iy] = ((jz_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + } + int const k_jx = ((jx_type[1] == NODE) ? k_node : k_cell); + int const k_jy = ((jy_type[1] == NODE) ? k_node : k_cell); + int const k_jz = ((jz_type[1] == NODE) ? k_node : k_cell); +#endif + + // z direction + // Keep these double to avoid bug in single precision + const double zmid = ((zp - zmin) + relative_time*vz)*dzi; + double sz_node[depos_order + 1] = {0.}; + double sz_cell[depos_order + 1] = {0.}; + int l_node = 0; + int l_cell = 0; + if (jx_type[zdir] == NODE || jy_type[zdir] == NODE || jz_type[zdir] == NODE) { + l_node = compute_shape_factor(sz_node, zmid); + } + if (jx_type[zdir] == CELL || jy_type[zdir] == CELL || jz_type[zdir] == CELL) { + l_cell = compute_shape_factor(sz_cell, zmid - 0.5); + } + amrex::Real sz_jx[depos_order + 1] = {0._rt}; + amrex::Real sz_jy[depos_order + 1] = {0._rt}; + amrex::Real sz_jz[depos_order + 1] = {0._rt}; + for (int iz=0; iz<=depos_order; iz++) + { + sz_jx[iz] = ((jx_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + sz_jy[iz] = ((jy_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + sz_jz[iz] = ((jz_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + } + int const l_jx = ((jx_type[zdir] == NODE) ? l_node : l_cell); + int const l_jy = ((jy_type[zdir] == NODE) ? l_node : l_cell); + int const l_jz = ((jz_type[zdir] == NODE) ? l_node : l_cell); + + // Deposit current into jx_arr, jy_arr and jz_arr +#if defined(WARPX_DIM_1D_Z) + for (int iz=0; iz<=depos_order; iz++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+l_jx+iz, 0, 0, 0), + sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+l_jy+iz, 0, 0, 0), + sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+l_jz+iz, 0, 0, 0), + sz_jz[iz]*wqz); + } +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + for (int iz=0; iz<=depos_order; iz++){ + for (int ix=0; ix<=depos_order; ix++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 0), + sx_jx[ix]*sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 0), + sx_jy[ix]*sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 0), + sx_jz[ix]*sz_jz[iz]*wqz); +#if defined(WARPX_DIM_RZ) + Complex xy = xy0; // Note that xy is equal to e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 on the weighting comes from the normalization of the modes + amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode-1), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode ), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.imag()); + amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode-1), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode ), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.imag()); + amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode-1), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode ), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.imag()); + xy = xy*xy0; + } +#endif + } + } +#elif defined(WARPX_DIM_3D) + for (int iz=0; iz<=depos_order; iz++){ + for (int iy=0; iy<=depos_order; iy++){ + for (int ix=0; ix<=depos_order; ix++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+j_jx+ix, lo.y+k_jx+iy, lo.z+l_jx+iz), + sx_jx[ix]*sy_jx[iy]*sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+j_jy+ix, lo.y+k_jy+iy, lo.z+l_jy+iz), + sx_jy[ix]*sy_jy[iy]*sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+j_jz+ix, lo.y+k_jz+iy, lo.z+l_jz+iz), + sx_jz[ix]*sy_jz[iy]*sz_jz[iz]*wqz); + } + } + } +#endif +} + /** * \brief Current Deposition for thread thread_num * \tparam depos_order deposition order @@ -113,10 +353,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::IntVect const jy_type = jy_fab.box().type(); amrex::IntVect const jz_type = jz_fab.box().type(); - constexpr int zdir = WARPX_ZINDEX; - constexpr int NODE = amrex::IndexType::NODE; - constexpr int CELL = amrex::IndexType::CELL; - // Loop over particles and deposit into jx_fab, jy_fab and jz_fab #if defined(WARPX_USE_GPUCLOCK) amrex::Real* cost_real = nullptr; @@ -126,208 +362,181 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, } #endif amrex::ParallelFor( - np_to_deposit, - [=] AMREX_GPU_DEVICE (long ip) { + np_to_deposit, + [=] AMREX_GPU_DEVICE (long ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), cost_real); #endif + amrex::ParticleReal xp, yp, zp; + GetPosition(ip, xp, yp, zp); + // --- Get particle quantities const amrex::Real gaminv = 1.0_rt/std::sqrt(1.0_rt + uxp[ip]*uxp[ip]*clightsq + uyp[ip]*uyp[ip]*clightsq + uzp[ip]*uzp[ip]*clightsq); + const amrex::Real vx = uxp[ip]*gaminv; + const amrex::Real vy = uyp[ip]*gaminv; + const amrex::Real vz = uzp[ip]*gaminv; + amrex::Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; } - amrex::ParticleReal xp, yp, zp; - GetPosition(ip, xp, yp, zp); + doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, + jx_type, jy_type, jz_type, + relative_time, + AMREX_D_DECL(dzi, dxi, dyi), + AMREX_D_DECL(zmin, xmin, ymin), + invvol, lo, n_rz_azimuthal_modes); - const amrex::Real vx = uxp[ip]*gaminv; - const amrex::Real vy = uyp[ip]*gaminv; - const amrex::Real vz = uzp[ip]*gaminv; - // wqx, wqy wqz are particle current in each direction -#if defined(WARPX_DIM_RZ) - // In RZ, wqx is actually wqr, and wqy is wqtheta - // Convert to cylindrical at the mid point - const amrex::Real xpmid = xp + relative_time*vx; - const amrex::Real ypmid = yp + relative_time*vy; - const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); - amrex::Real costheta; - amrex::Real sintheta; - if (rpmid > 0._rt) { - costheta = xpmid/rpmid; - sintheta = ypmid/rpmid; - } else { - costheta = 1._rt; - sintheta = 0._rt; - } - const Complex xy0 = Complex{costheta, sintheta}; - const amrex::Real wqx = wq*invvol*(+vx*costheta + vy*sintheta); - const amrex::Real wqy = wq*invvol*(-vx*sintheta + vy*costheta); -#else - const amrex::Real wqx = wq*invvol*vx; - const amrex::Real wqy = wq*invvol*vy; + } + ); +#if defined(WARPX_USE_GPUCLOCK) + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + amrex::Gpu::streamSynchronize(); + *cost += *cost_real; + amrex::The_Managed_Arena()->free(cost_real); + } #endif - const amrex::Real wqz = wq*invvol*vz; +} - // --- Compute shape factors - Compute_shape_factor< depos_order > const compute_shape_factor; -#if (AMREX_SPACEDIM >= 2) - // x direction - // Get particle position after 1/2 push back in position -#if defined(WARPX_DIM_RZ) - // Keep these double to avoid bug in single precision - const double xmid = (rpmid - xmin)*dxi; -#else - const double xmid = ((xp - xmin) + relative_time*vx)*dxi; +/** + * \brief Direct current deposition for thread thread_num for the implicit scheme + * The only difference from doDepositionShapeN is in how the particle gamma + * is calculated. + * \tparam depos_order deposition order + * \param GetPosition A functor for returning the particle position. + * \param wp Pointer to array of particle weights. + * \param uxp_n,uyp_n,uzp_n Pointer to arrays of particle momentum at time n. + * \param uxp,uyp,uzp Pointer to arrays of particle momentum at time n+1/2. + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. + * \param np_to_deposit Number of particles for which current is deposited. + * \param dx 3D cell size + * \param xyzmin Physical lower bounds of domain. + * \param lo Index lower bounds of domain. + * \param q species charge. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. + * \param load_balance_costs_update_algo Selected method for updating load balance costs. + */ +template +void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, + const amrex::ParticleReal * const wp, + const amrex::ParticleReal * const uxp_n, + const amrex::ParticleReal * const uyp_n, + const amrex::ParticleReal * const uzp_n, + const amrex::ParticleReal * const uxp, + const amrex::ParticleReal * const uyp, + const amrex::ParticleReal * const uzp, + const int * const ion_lev, + amrex::FArrayBox& jx_fab, + amrex::FArrayBox& jy_fab, + amrex::FArrayBox& jz_fab, + const long np_to_deposit, + const std::array& dx, + const std::array& xyzmin, + const amrex::Dim3 lo, + const amrex::Real q, + const int n_rz_azimuthal_modes, + amrex::Real* cost, + const long load_balance_costs_update_algo) +{ + using namespace amrex::literals; +#if !defined(WARPX_DIM_RZ) + amrex::ignore_unused(n_rz_azimuthal_modes); #endif - // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current - // sx_j[xyz] shape factor along x for the centering of each current - // There are only two possible centerings, node or cell centered, so at most only two shape factor - // arrays will be needed. - // Keep these double to avoid bug in single precision - double sx_node[depos_order + 1] = {0.}; - double sx_cell[depos_order + 1] = {0.}; - int j_node = 0; - int j_cell = 0; - if (jx_type[0] == NODE || jy_type[0] == NODE || jz_type[0] == NODE) { - j_node = compute_shape_factor(sx_node, xmid); - } - if (jx_type[0] == CELL || jy_type[0] == CELL || jz_type[0] == CELL) { - j_cell = compute_shape_factor(sx_cell, xmid - 0.5); - } - amrex::Real sx_jx[depos_order + 1] = {0._rt}; - amrex::Real sx_jy[depos_order + 1] = {0._rt}; - amrex::Real sx_jz[depos_order + 1] = {0._rt}; - for (int ix=0; ix<=depos_order; ix++) - { - sx_jx[ix] = ((jx_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - sx_jy[ix] = ((jy_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - sx_jz[ix] = ((jz_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - } +#if !defined(AMREX_USE_GPU) + amrex::ignore_unused(cost, load_balance_costs_update_algo); +#endif - int const j_jx = ((jx_type[0] == NODE) ? j_node : j_cell); - int const j_jy = ((jy_type[0] == NODE) ? j_node : j_cell); - int const j_jz = ((jz_type[0] == NODE) ? j_node : j_cell); -#endif //AMREX_SPACEDIM >= 2 + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) + const bool do_ionization = ion_lev; + const amrex::Real dzi = 1.0_rt/dx[2]; +#if defined(WARPX_DIM_1D_Z) + const amrex::Real invvol = dzi; +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real dxi = 1.0_rt/dx[0]; + const amrex::Real invvol = dxi*dzi; +#elif defined(WARPX_DIM_3D) + const amrex::Real dxi = 1.0_rt/dx[0]; + const amrex::Real dyi = 1.0_rt/dx[1]; + const amrex::Real invvol = dxi*dyi*dzi; +#endif +#if (AMREX_SPACEDIM >= 2) + const amrex::Real xmin = xyzmin[0]; +#endif #if defined(WARPX_DIM_3D) - // y direction - // Keep these double to avoid bug in single precision - const double ymid = ((yp - ymin) + relative_time*vy)*dyi; - double sy_node[depos_order + 1] = {0.}; - double sy_cell[depos_order + 1] = {0.}; - int k_node = 0; - int k_cell = 0; - if (jx_type[1] == NODE || jy_type[1] == NODE || jz_type[1] == NODE) { - k_node = compute_shape_factor(sy_node, ymid); - } - if (jx_type[1] == CELL || jy_type[1] == CELL || jz_type[1] == CELL) { - k_cell = compute_shape_factor(sy_cell, ymid - 0.5); - } - amrex::Real sy_jx[depos_order + 1] = {0._rt}; - amrex::Real sy_jy[depos_order + 1] = {0._rt}; - amrex::Real sy_jz[depos_order + 1] = {0._rt}; - for (int iy=0; iy<=depos_order; iy++) - { - sy_jx[iy] = ((jx_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - sy_jy[iy] = ((jy_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - sy_jz[iy] = ((jz_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - } - int const k_jx = ((jx_type[1] == NODE) ? k_node : k_cell); - int const k_jy = ((jy_type[1] == NODE) ? k_node : k_cell); - int const k_jz = ((jz_type[1] == NODE) ? k_node : k_cell); + const amrex::Real ymin = xyzmin[1]; #endif + const amrex::Real zmin = xyzmin[2]; - // z direction - // Keep these double to avoid bug in single precision - const double zmid = ((zp - zmin) + relative_time*vz)*dzi; - double sz_node[depos_order + 1] = {0.}; - double sz_cell[depos_order + 1] = {0.}; - int l_node = 0; - int l_cell = 0; - if (jx_type[zdir] == NODE || jy_type[zdir] == NODE || jz_type[zdir] == NODE) { - l_node = compute_shape_factor(sz_node, zmid); - } - if (jx_type[zdir] == CELL || jy_type[zdir] == CELL || jz_type[zdir] == CELL) { - l_cell = compute_shape_factor(sz_cell, zmid - 0.5); - } - amrex::Real sz_jx[depos_order + 1] = {0._rt}; - amrex::Real sz_jy[depos_order + 1] = {0._rt}; - amrex::Real sz_jz[depos_order + 1] = {0._rt}; - for (int iz=0; iz<=depos_order; iz++) - { - sz_jx[iz] = ((jx_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - sz_jy[iz] = ((jy_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - sz_jz[iz] = ((jz_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - } - int const l_jx = ((jx_type[zdir] == NODE) ? l_node : l_cell); - int const l_jy = ((jy_type[zdir] == NODE) ? l_node : l_cell); - int const l_jz = ((jz_type[zdir] == NODE) ? l_node : l_cell); + amrex::Array4 const& jx_arr = jx_fab.array(); + amrex::Array4 const& jy_arr = jy_fab.array(); + amrex::Array4 const& jz_arr = jz_fab.array(); + amrex::IntVect const jx_type = jx_fab.box().type(); + amrex::IntVect const jy_type = jy_fab.box().type(); + amrex::IntVect const jz_type = jz_fab.box().type(); - // Deposit current into jx_arr, jy_arr and jz_arr -#if defined(WARPX_DIM_1D_Z) - for (int iz=0; iz<=depos_order; iz++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+l_jx+iz, 0, 0, 0), - sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+l_jy+iz, 0, 0, 0), - sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+l_jz+iz, 0, 0, 0), - sz_jz[iz]*wqz); - } + // Loop over particles and deposit into jx_fab, jy_fab and jz_fab +#if defined(WARPX_USE_GPUCLOCK) + amrex::Real* cost_real = nullptr; + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); + *cost_real = 0._rt; + } #endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - for (int iz=0; iz<=depos_order; iz++){ - for (int ix=0; ix<=depos_order; ix++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 0), - sx_jx[ix]*sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 0), - sx_jy[ix]*sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 0), - sx_jz[ix]*sz_jz[iz]*wqz); -#if defined(WARPX_DIM_RZ) - Complex xy = xy0; // Note that xy is equal to e^{i m theta} - for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { - // The factor 2 on the weighting comes from the normalization of the modes - amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode-1), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode ), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.imag()); - amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode-1), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode ), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.imag()); - amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode-1), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode ), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.imag()); - xy = xy*xy0; - } + amrex::ParallelFor( + np_to_deposit, + [=] AMREX_GPU_DEVICE (long ip) { +#if defined(WARPX_USE_GPUCLOCK) + const auto KernelTimer = ablastr::parallelization::KernelTimer( + cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), + cost_real); #endif - } - } -#elif defined(WARPX_DIM_3D) - for (int iz=0; iz<=depos_order; iz++){ - for (int iy=0; iy<=depos_order; iy++){ - for (int ix=0; ix<=depos_order; ix++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+j_jx+ix, lo.y+k_jx+iy, lo.z+l_jx+iz), - sx_jx[ix]*sy_jx[iy]*sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+j_jy+ix, lo.y+k_jy+iy, lo.z+l_jy+iz), - sx_jy[ix]*sy_jy[iy]*sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+j_jz+ix, lo.y+k_jz+iy, lo.z+l_jz+iz), - sx_jz[ix]*sy_jz[iy]*sz_jz[iz]*wqz); - } - } + + amrex::ParticleReal xp, yp, zp; + GetPosition(ip, xp, yp, zp); + + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The uxp,uyp,uzp are the velocities at time level n+1/2 + const amrex::ParticleReal uxp_np1 = 2._prt*uxp[ip] - uxp_n[ip]; + const amrex::ParticleReal uyp_np1 = 2._prt*uyp[ip] - uyp_n[ip]; + const amrex::ParticleReal uzp_np1 = 2._prt*uzp[ip] - uzp_n[ip]; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (uxp_n[ip]*uxp_n[ip] + uyp_n[ip]*uyp_n[ip] + uzp_n[ip]*uzp_n[ip])*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (uxp_np1*uxp_np1 + uyp_np1*uyp_np1 + uzp_np1*uzp_np1)*inv_c2); + const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); + + const amrex::Real vx = uxp[ip]*gaminv; + const amrex::Real vy = uyp[ip]*gaminv; + const amrex::Real vz = uzp[ip]*gaminv; + + amrex::Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; } -#endif + + const amrex::Real relative_time = 0._rt; + doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, + jx_type, jy_type, jz_type, + relative_time, + AMREX_D_DECL(dzi, dxi, dyi), + AMREX_D_DECL(zmin, xmin, ymin), + invvol, lo, n_rz_azimuthal_modes); + } ); #if defined(WARPX_USE_GPUCLOCK) @@ -927,6 +1136,403 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif } +/** + * \brief Esirkepov Current Deposition for thread thread_num for implicit scheme + * The difference from doEsirkepovDepositionShapeN is in how the old and new + * particles positions are determined and in how the particle gamma is calculated. + * + * \tparam depos_order deposition order + * \param GetPosition A functor for returning the particle position. + * \param wp Pointer to array of particle weights. + * \param uxp,uyp,uzp Pointer to arrays of particle momentum. + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. + * \param np_to_deposit Number of particles for which current is deposited. + * \param dt Time step for particle level + * \param dx 3D cell size + * \param xyzmin Physical lower bounds of domain. + * \param lo Index lower bounds of domain. + * \param q species charge. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. + * \param load_balance_costs_update_algo Selected method for updating load balance costs. + */ +template +void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, + const amrex::ParticleReal * const yp_n, + const amrex::ParticleReal * const zp_n, + const GetParticlePosition& GetPosition, + const amrex::ParticleReal * const wp, + [[maybe_unused]]const amrex::ParticleReal * const uxp_n, + [[maybe_unused]]const amrex::ParticleReal * const uyp_n, + [[maybe_unused]]const amrex::ParticleReal * const uzp_n, + [[maybe_unused]]const amrex::ParticleReal * const uxp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uyp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uzp_nph, + const int * const ion_lev, + const amrex::Array4& Jx_arr, + const amrex::Array4& Jy_arr, + const amrex::Array4& Jz_arr, + const long np_to_deposit, + const amrex::Real dt, + const std::array& dx, + const std::array xyzmin, + const amrex::Dim3 lo, + const amrex::Real q, + const int n_rz_azimuthal_modes, + amrex::Real * const cost, + const long load_balance_costs_update_algo) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(AMREX_USE_GPU) + amrex::ignore_unused(cost, load_balance_costs_update_algo); +#endif + + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) + bool const do_ionization = ion_lev; +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if defined(WARPX_DIM_3D) + Real const invdtdx = 1.0_rt / (dt*dx[1]*dx[2]); + Real const invdtdy = 1.0_rt / (dt*dx[0]*dx[2]); + Real const invdtdz = 1.0_rt / (dt*dx[0]*dx[1]); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + Real const invdtdx = 1.0_rt / (dt*dx[2]); + Real const invdtdz = 1.0_rt / (dt*dx[0]); + Real const invvol = 1.0_rt / (dx[0]*dx[2]); +#elif defined(WARPX_DIM_1D_Z) + Real const invdtdz = 1.0_rt / (dt*dx[0]); + Real const invvol = 1.0_rt / (dx[2]); +#endif + +#if defined(WARPX_DIM_RZ) + Complex const I = Complex{0._rt, 1._rt}; +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + + // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr +#if defined(WARPX_USE_GPUCLOCK) + amrex::Real* cost_real = nullptr; + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); + *cost_real = 0._rt; + } +#endif + amrex::ParallelFor( + np_to_deposit, + [=] AMREX_GPU_DEVICE (long const ip) { +#if defined(WARPX_USE_GPUCLOCK) + const auto KernelTimer = ablastr::parallelization::KernelTimer( + cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), + cost_real); +#endif + +#if !defined(WARPX_DIM_3D) + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The uxp,uyp,uzp are the velocities at time level n+1/2 + const amrex::ParticleReal uxp_np1 = 2._prt*uxp_nph[ip] - uxp_n[ip]; + const amrex::ParticleReal uyp_np1 = 2._prt*uyp_nph[ip] - uyp_n[ip]; + const amrex::ParticleReal uzp_np1 = 2._prt*uzp_nph[ip] - uzp_n[ip]; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (uxp_n[ip]*uxp_n[ip] + uyp_n[ip]*uyp_n[ip] + uzp_n[ip]*uzp_n[ip])*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (uxp_np1*uxp_np1 + uyp_np1*uyp_np1 + uzp_np1*uzp_np1)*inv_c2); + const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); +#endif + + // wqx, wqy wqz are particle current in each direction + Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; + } + + ParticleReal xp_nph, yp_nph, zp_nph; + GetPosition(ip, xp_nph, yp_nph, zp_nph); + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; +#else + ignore_unused(xp_n); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; +#else + ignore_unused(yp_n); +#endif + ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; + +#if !defined(WARPX_DIM_1D_Z) + amrex::Real const wqx = wq*invdtdx; +#endif +#if defined(WARPX_DIM_3D) + amrex::Real const wqy = wq*invdtdy; +#endif + amrex::Real const wqz = wq*invdtdz; + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n[ip]; + amrex::Real const yp_old = yp_n[ip]; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_new, sintheta_new; + if (rp_new > 0._rt) { + costheta_new = xp_new/rp_new; + sintheta_new = yp_new/rp_new; + } else { + costheta_new = 1._rt; + sintheta_new = 0._rt; + } + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + amrex::Real costheta_old, sintheta_old; + if (rp_old > 0._rt) { + costheta_old = xp_old/rp_old; + sintheta_old = yp_old/rp_old; + } else { + costheta_old = 1._rt; + sintheta_old = 0._rt; + } + const Complex xy_new0 = Complex{costheta_new, sintheta_new}; + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + const Complex xy_old0 = Complex{costheta_old, sintheta_old}; + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; +#else +#if !defined(WARPX_DIM_1D_Z) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n[ip] - xmin)*dxi; +#endif +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n[ip] - ymin)*dyi; +#endif + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n[ip] - zmin)*dzi; + +#if defined(WARPX_DIM_RZ) + amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; +#elif defined(WARPX_DIM_XZ) + amrex::Real const vy = uyp_nph[ip]*gaminv; +#elif defined(WARPX_DIM_1D_Z) + amrex::Real const vx = uxp_nph[ip]*gaminv; + amrex::Real const vy = uyp_nph[ip]*gaminv; +#endif + + // Shape factor arrays + // Note that there are extra values above and below + // to possibly hold the factor for the old particle + // which can be at a different grid location. + // Keep these double to avoid bug in single precision +#if !defined(WARPX_DIM_1D_Z) + double sx_new[depos_order + 3] = {0.}; + double sx_old[depos_order + 3] = {0.}; +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double sy_new[depos_order + 3] = {0.}; + double sy_old[depos_order + 3] = {0.}; +#endif + // Keep these double to avoid bug in single precision + double sz_new[depos_order + 3] = {0.}; + double sz_old[depos_order + 3] = {0.}; + + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + Compute_shape_factor< depos_order > compute_shape_factor; + Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + +#if !defined(WARPX_DIM_1D_Z) + const int i_new = compute_shape_factor(sx_new+1, x_new); + const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); +#endif +#if defined(WARPX_DIM_3D) + const int j_new = compute_shape_factor(sy_new+1, y_new); + const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); +#endif + const int k_new = compute_shape_factor(sz_new+1, z_new); + const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); + + // computes min/max positions of current contributions +#if !defined(WARPX_DIM_1D_Z) + int dil = 1, diu = 1; + if (i_old < i_new) { dil = 0; } + if (i_old > i_new) { diu = 0; } +#endif +#if defined(WARPX_DIM_3D) + int djl = 1, dju = 1; + if (j_old < j_new) { djl = 0; } + if (j_old > j_new) { dju = 0; } +#endif + int dkl = 1, dku = 1; + if (k_old < k_new) { dkl = 0; } + if (k_old > k_new) { dku = 0; } + +#if defined(WARPX_DIM_3D) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int j=djl; j<=depos_order+2-dju; j++) { + amrex::Real sdxi = 0._rt; + for (int i=dil; i<=depos_order+1-diu; i++) { + sdxi += wqx*(sx_old[i] - sx_new[i])*( + one_third*(sy_new[j]*sz_new[k] + sy_old[j]*sz_old[k]) + +one_sixth*(sy_new[j]*sz_old[k] + sy_old[j]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdxi); + } + } + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + amrex::Real sdyj = 0._rt; + for (int j=djl; j<=depos_order+1-dju; j++) { + sdyj += wqy*(sy_old[j] - sy_new[j])*( + one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) + +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdyj); + } + } + } + for (int j=djl; j<=depos_order+2-dju; j++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + amrex::Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k])*( + one_third*(sx_new[i]*sy_new[j] + sx_old[i]*sy_old[j]) + +one_sixth*(sx_new[i]*sy_old[j] + sx_old[i]*sy_new[j])); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdzk); + } + } + } + +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real sdxi = 0._rt; + for (int i=dil; i<=depos_order+1-diu; i++) { + sdxi += wqx*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdxi); +#if defined(WARPX_DIM_RZ) + Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + const Complex djr_cmplx = 2._rt *sdxi*xy_mid; + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djr_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djr_cmplx.imag()); + xy_mid = xy_mid*xy_mid0; + } +#endif + } + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + Real const sdyj = wq*vy*invvol*( + one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) + +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdyj); +#if defined(WARPX_DIM_RZ) + Complex xy_new = xy_new0; + Complex xy_mid = xy_mid0; + Complex xy_old = xy_old0; + // Throughout the following loop, xy_ takes the value e^{i m theta_} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + // The minus sign comes from the different convention with respect to Davidson et al. + const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(amrex::Real)imode + *(Complex(sx_new[i]*sz_new[k], 0._rt)*(xy_new - xy_mid) + + Complex(sx_old[i]*sz_old[k], 0._rt)*(xy_mid - xy_old)); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djt_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djt_cmplx.imag()); + xy_new = xy_new*xy_new0; + xy_mid = xy_mid*xy_mid0; + xy_old = xy_old*xy_old0; + } +#endif + } + } + for (int i=dil; i<=depos_order+2-diu; i++) { + Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdzk); +#if defined(WARPX_DIM_RZ) + Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + const Complex djz_cmplx = 2._rt * sdzk * xy_mid; + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djz_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djz_cmplx.imag()); + xy_mid = xy_mid*xy_mid0; + } +#endif + } + } +#elif defined(WARPX_DIM_1D_Z) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real const sdxi = wq*vx*invvol*0.5_rt*(sz_old[k] + sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+k_new-1+k, 0, 0, 0), sdxi); + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real const sdyj = wq*vy*invvol*0.5_rt*(sz_old[k] + sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+k_new-1+k, 0, 0, 0), sdyj); + } + amrex::Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+k_new-1+k, 0, 0, 0), sdzk); + } +#endif + } + ); +#if defined(WARPX_USE_GPUCLOCK) + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + amrex::Gpu::streamSynchronize(); + *cost += *cost_real; + amrex::The_Managed_Arena()->free(cost_real); + } +#endif +} + /** * \brief Vay current deposition * ( Vay et al, 2013) diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index 1b5cede3c6f..dd6b7276681 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -451,6 +451,435 @@ void doGatherShapeN (const amrex::ParticleReal xp, #endif } +/** + * \brief Energy conserving field gather for thread thread_num for the implicit scheme + * This uses the same stencil for the gather that is used for Esirkepov current deposition. + * + * \tparam depos_order Particle shape order + * \param xp_n,yp_n,zp_n Particle position coordinates at start of step + * \param xp_nph,yp_nph,zp_nph Particle position coordinates at half step + * \param Exp,Eyp,Ezp Electric field on particles. + * \param Bxp,Byp,Bzp Magnetic field on particles. + * \param Ex_arr,Ey_arr,Ez_arr Array4 of the electric field, either full array or tile. + * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. + * \param Ex_type,Ey_type,Ez_type IndexType of the electric field + * \param Bx_type,By_type,Bz_type IndexType of the magnetic field + * \param dx 3D cell spacing + * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + */ +template +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void doGatherShapeNEsirkepovStencilImplicit ( + [[maybe_unused]] const amrex::ParticleReal xp_n, + [[maybe_unused]] const amrex::ParticleReal yp_n, + const amrex::ParticleReal zp_n, + [[maybe_unused]] const amrex::ParticleReal xp_nph, + [[maybe_unused]] const amrex::ParticleReal yp_nph, + const amrex::ParticleReal zp_nph, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp, + amrex::Array4 const& Ex_arr, + amrex::Array4 const& Ey_arr, + amrex::Array4 const& Ez_arr, + amrex::Array4 const& Bx_arr, + amrex::Array4 const& By_arr, + amrex::Array4 const& Bz_arr, + [[maybe_unused]] const amrex::IndexType Ex_type, + [[maybe_unused]] const amrex::IndexType Ey_type, + [[maybe_unused]] const amrex::IndexType Ez_type, + [[maybe_unused]] const amrex::IndexType Bx_type, + [[maybe_unused]] const amrex::IndexType By_type, + [[maybe_unused]] const amrex::IndexType Bz_type, + const amrex::GpuArray& dx, + const amrex::GpuArray& xyzmin, + const amrex::Dim3& lo, + const int n_rz_azimuthal_modes) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal xp_np1 = 2._prt*xp_nph - xp_n; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal yp_np1 = 2._prt*yp_nph - yp_n; +#endif + ParticleReal zp_np1 = 2._prt*zp_nph - zp_n; + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n; + amrex::Real const yp_old = yp_n; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; +#else +#if !defined(WARPX_DIM_1D_Z) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n - xmin)*dxi; +#endif +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n - ymin)*dyi; +#endif + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n - zmin)*dzi; + + // Shape factor arrays + // Note that there are extra values above and below + // to possibly hold the factor for the old particle + // which can be at a different grid location. + // Keep these double to avoid bug in single precision +#if !defined(WARPX_DIM_1D_Z) + double sx_E_new[depos_order + 3] = {0.}; + double sx_E_old[depos_order + 3] = {0.}; +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double sy_E_new[depos_order + 3] = {0.}; + double sy_E_old[depos_order + 3] = {0.}; +#endif + // Keep these double to avoid bug in single precision + double sz_E_new[depos_order + 3] = {0.}; + double sz_E_old[depos_order + 3] = {0.}; + +#if defined(WARPX_DIM_3D) + double sx_B_new[depos_order + 3] = {0.}; + double sx_B_old[depos_order + 3] = {0.}; + double sy_B_new[depos_order + 3] = {0.}; + double sy_B_old[depos_order + 3] = {0.}; + double sz_B_new[depos_order + 3] = {0.}; + double sz_B_old[depos_order + 3] = {0.}; +#endif + +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + // Special shape functions are needed for By which is cell + // centered in both x and z. One lower order shape function is used. + double sx_By_new[depos_order + 2] = {0.}; + double sx_By_old[depos_order + 2] = {0.}; + double sz_By_new[depos_order + 2] = {0.}; + double sz_By_old[depos_order + 2] = {0.}; +#endif + + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + Compute_shape_factor< depos_order > compute_shape_factor; + Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + +#if !defined(WARPX_DIM_1D_Z) + const int i_E_new = compute_shape_factor(sx_E_new+1, x_new); + const int i_E_old = compute_shifted_shape_factor(sx_E_old, x_old, i_E_new); +#endif +#if defined(WARPX_DIM_3D) + const int j_E_new = compute_shape_factor(sy_E_new+1, y_new); + const int j_E_old = compute_shifted_shape_factor(sy_E_old, y_old, j_E_new); +#endif + const int k_E_new = compute_shape_factor(sz_E_new+1, z_new); + const int k_E_old = compute_shifted_shape_factor(sz_E_old, z_old, k_E_new); + +#if defined(WARPX_DIM_3D) + const int i_B_new = compute_shape_factor(sx_B_new+1, x_new - 0.5_rt); + const int i_B_old = compute_shifted_shape_factor(sx_B_old, x_old - 0.5_rt, i_B_new); + const int j_B_new = compute_shape_factor(sy_B_new+1, y_new - 0.5_rt); + const int j_B_old = compute_shifted_shape_factor(sy_B_old, y_old - 0.5_rt, j_B_new); + const int k_B_new = compute_shape_factor(sz_B_new+1, z_new - 0.5_rt); + const int k_B_old = compute_shifted_shape_factor(sz_B_old, z_old - 0.5_rt, k_B_new); +#endif + + // computes min/max positions of current contributions +#if !defined(WARPX_DIM_1D_Z) + int dil_E = 1, diu_E = 1; + if (i_E_old < i_E_new) { dil_E = 0; } + if (i_E_old > i_E_new) { diu_E = 0; } +#endif +#if defined(WARPX_DIM_3D) + int djl_E = 1, dju_E = 1; + if (j_E_old < j_E_new) { djl_E = 0; } + if (j_E_old > j_E_new) { dju_E = 0; } +#endif + int dkl_E = 1, dku_E = 1; + if (k_E_old < k_E_new) { dkl_E = 0; } + if (k_E_old > k_E_new) { dku_E = 0; } + +#if defined(WARPX_DIM_3D) + int dil_B = 1, diu_B = 1; + if (i_B_old < i_B_new) { dil_B = 0; } + if (i_B_old > i_B_new) { diu_B = 0; } + int djl_B = 1, dju_B = 1; + if (j_B_old < j_B_new) { djl_B = 0; } + if (j_B_old > j_B_new) { dju_B = 0; } + int dkl_B = 1, dku_B = 1; + if (k_B_old < k_B_new) { dkl_B = 0; } + if (k_B_old > k_B_new) { dku_B = 0; } +#endif + +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + Compute_shape_factor< depos_order-1 > compute_shape_factor_By; + Compute_shifted_shape_factor< depos_order-1 > compute_shifted_shape_factor_By; + const int i_By_new = compute_shape_factor_By(sx_By_new+1, x_new - 0.5_rt); + const int i_By_old = compute_shifted_shape_factor_By(sx_By_old, x_old - 0.5_rt, i_By_new); + const int k_By_new = compute_shape_factor_By(sz_By_new+1, z_new - 0.5_rt); + const int k_By_old = compute_shifted_shape_factor_By(sz_By_old, z_old - 0.5_rt, k_By_new); + int dil_By = 1, diu_By = 1; + if (i_By_old < i_By_new) { dil_By = 0; } + if (i_By_old > i_By_new) { diu_By = 0; } + int dkl_By = 1, dku_By = 1; + if (k_By_old < k_By_new) { dkl_By = 0; } + if (k_By_old > k_By_new) { dku_By = 0; } +#endif + +#if defined(WARPX_DIM_3D) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int j=djl_E; j<=depos_order+2-dju_E; j++) { + amrex::Real sdzjk = one_third*(sy_E_new[j]*sz_E_new[k] + sy_E_old[j]*sz_E_old[k]) + +one_sixth*(sy_E_new[j]*sz_E_old[k] + sy_E_old[j]*sz_E_new[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Exp += Ex_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdxiov*sdzjk; + } + } + } + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdyik = one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k]); + amrex::Real sdyj = 0._rt; + for (int j=djl_E; j<=depos_order+1-dju_E; j++) { + sdyj += (sy_E_old[j] - sy_E_new[j]); + auto sdyjov = static_cast((y_new - y_old) == 0. ? 1. : sdyj/(y_new - y_old)); + Eyp += Ey_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdyjov*sdyik; + } + } + } + for (int j=djl_E; j<=depos_order+2-dju_E; j++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdzij = one_third*(sx_E_new[i]*sy_E_new[j] + sx_E_old[i]*sy_E_old[j]) + +one_sixth*(sx_E_new[i]*sy_E_old[j] + sx_E_old[i]*sy_E_new[j]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Ezp += Ez_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdzkov*sdzij; + } + } + } + for (int k=dkl_B; k<=depos_order+2-dku_B; k++) { + for (int j=djl_B; j<=depos_order+2-dju_B; j++) { + amrex::Real sdzjk = one_third*(sy_B_new[j]*sz_B_new[k] + sy_B_old[j]*sz_B_old[k]) + +one_sixth*(sy_B_new[j]*sz_B_old[k] + sy_B_old[j]*sz_B_new[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_B; i<=depos_order+1-diu_B; i++) { + sdxi += (sx_B_old[i] - sx_B_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Bxp += Bx_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_B_new-1+k)*sdxiov*sdzjk; + } + } + } + for (int k=dkl_B; k<=depos_order+2-dku_B; k++) { + for (int i=dil_B; i<=depos_order+2-diu_B; i++) { + amrex::Real sdyik = one_third*(sx_B_new[i]*sz_B_new[k] + sx_B_old[i]*sz_B_old[k]) + +one_sixth*(sx_B_new[i]*sz_B_old[k] + sx_B_old[i]*sz_B_new[k]); + amrex::Real sdyj = 0._rt; + for (int j=djl_B; j<=depos_order+1-dju_B; j++) { + sdyj += (sy_B_old[j] - sy_B_new[j]); + auto sdyjov = static_cast((y_new - y_old) == 0. ? 1. : sdyj/(y_new - y_old)); + Byp += By_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_B_new-1+k)*sdyjov*sdyik; + } + } + } + for (int j=djl_B; j<=depos_order+2-dju_B; j++) { + for (int i=dil_B; i<=depos_order+2-diu_B; i++) { + amrex::Real sdzij = one_third*(sx_B_new[i]*sy_B_new[j] + sx_B_old[i]*sy_B_old[j]) + +one_sixth*(sx_B_new[i]*sy_B_old[j] + sx_B_old[i]*sy_B_new[j]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_B; k<=depos_order+1-dku_B; k++) { + sdzk += (sz_B_old[k] - sz_B_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Bzp += Bz_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_E_new-1+k)*sdzkov*sdzij; + } + } + } + +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real sdzk = 0.5_rt*(sz_E_new[k] + sz_E_old[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Exp += Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdxiov*sdzk; + Bzp += Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdxiov*sdzk; + } + } + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + Real const sdyj = ( + one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k])); + Eyp += Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdyj; + } + } + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdxi = 0.5_rt*(sx_E_new[i] + sx_E_old[i]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Ezp += Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdzkov*sdxi; + Bxp += Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdzkov*sdxi; + } + } + for (int k=dkl_By; k<=depos_order+1-dku_By; k++) { + for (int i=dil_By; i<=depos_order+1-diu_By; i++) { + Real const sdyj = ( + one_third*(sx_By_new[i]*sz_By_new[k] + sx_By_old[i]*sz_By_old[k]) + +one_sixth*(sx_By_new[i]*sz_By_old[k] + sx_By_old[i]*sz_By_new[k])); + Byp += By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 0)*sdyj; + } + } + +#ifdef WARPX_DIM_RZ + Complex xy_mid = xy_mid0; + + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + + // Gather field on particle Exp from field on grid ex_arr + // Gather field on particle Bzp from field on grid bz_arr + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real sdzk = 0.5_rt*(sz_E_new[k] + sz_E_old[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + const amrex::Real dEx = (+ Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + const amrex::Real dBz = (+ Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Exp += dEx*sdxiov*sdzk; + Bzp += dBz*sdxiov*sdzk; + } + } + // Gather field on particle Eyp from field on grid ey_arr + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + Real const sdyj = ( + one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k])); + const amrex::Real dEy = (+ Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Eyp += dEy*sdyj; + } + } + // Gather field on particle Ezp from field on grid ez_arr + // Gather field on particle Bxp from field on grid bx_arr + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdxi = 0.5_rt*(sx_E_new[i] + sx_E_old[i]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + const amrex::Real dEz = (+ Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + const amrex::Real dBx = (+ Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Ezp += dEz*sdzkov*sdxi; + Bxp += dBx*sdzkov*sdxi; + } + } + // Gather field on particle Byp from field on grid by_arr + for (int k=dkl_By; k<=depos_order+1-dku_By; k++) { + for (int i=dil_By; i<=depos_order+1-diu_By; i++) { + Real const sdyj = ( + one_third*(sx_By_new[i]*sz_By_new[k] + sx_By_old[i]*sz_By_old[k]) + +one_sixth*(sx_By_new[i]*sz_By_old[k] + sx_By_old[i]*sz_By_new[k])); + const amrex::Real dBy = (+ By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 2*imode-1)*xy_mid.real() + - By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 2*imode)*xy_mid.imag()); + Byp += dBy*sdyj; + } + } + xy_mid = xy_mid*xy_mid0; + } + + // Convert Exp and Eyp (which are actually Er and Etheta) to Ex and Ey + const amrex::Real Exp_save = Exp; + Exp = costheta_mid*Exp - sintheta_mid*Eyp; + Eyp = costheta_mid*Eyp + sintheta_mid*Exp_save; + const amrex::Real Bxp_save = Bxp; + Bxp = costheta_mid*Bxp - sintheta_mid*Byp; + Byp = costheta_mid*Byp + sintheta_mid*Bxp_save; + +#endif + +#elif defined(WARPX_DIM_1D_Z) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real const sdzk = 0.5_rt*(sz_E_old[k] + sz_E_new[k]); + Exp += Ex_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + Eyp += Ey_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + Bzp += Bz_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + } + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Bxp += Bx_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + Byp += By_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + Ezp += Ez_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + } +#endif +} + /** * \brief Field gather for particles * @@ -606,4 +1035,96 @@ void doGatherShapeN (const amrex::ParticleReal xp, } } + +/** + * \brief Field gather for a single particle + * + * \param xp_n,yp_n,zp_n Particle position coordinates at start of step + * \param xp_nph,yp_nph,zp_nph Particle position coordinates at half time level (n + half) + * \param Exp,Eyp,Ezp Electric field on particles. + * \param Bxp,Byp,Bzp Magnetic field on particles. + * \param ex_arr,ey_arr,ez_arr Array4 of the electric field, either full array or tile. + * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. + * \param ex_type,ey_type,ez_type IndexType of the electric field + * \param bx_type,by_type,bz_type IndexType of the magnetic field + * \param dx_arr 3D cell spacing + * \param xyzmin_arr Physical lower bounds of domain in x, y, z. + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + * \param nox order of the particle shape function + * \param galerkin_interpolation whether to use lower order in v + */ +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void doGatherShapeNImplicit ( + const amrex::ParticleReal xp_n, + const amrex::ParticleReal yp_n, + const amrex::ParticleReal zp_n, + const amrex::ParticleReal xp_nph, + const amrex::ParticleReal yp_nph, + const amrex::ParticleReal zp_nph, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp, + amrex::Array4 const& ex_arr, + amrex::Array4 const& ey_arr, + amrex::Array4 const& ez_arr, + amrex::Array4 const& bx_arr, + amrex::Array4 const& by_arr, + amrex::Array4 const& bz_arr, + const amrex::IndexType ex_type, + const amrex::IndexType ey_type, + const amrex::IndexType ez_type, + const amrex::IndexType bx_type, + const amrex::IndexType by_type, + const amrex::IndexType bz_type, + const amrex::GpuArray& dx_arr, + const amrex::GpuArray& xyzmin_arr, + const amrex::Dim3& lo, + const int n_rz_azimuthal_modes, + const int nox, + const bool galerkin_interpolation) +{ + if (galerkin_interpolation) { + if (nox == 1) { + doGatherShapeNEsirkepovStencilImplicit<1>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherShapeNEsirkepovStencilImplicit<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherShapeNEsirkepovStencilImplicit<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } else { + if (nox == 1) { + doGatherShapeN<1,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherShapeN<2,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherShapeN<3,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } +} + #endif // FIELDGATHER_H_ diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index b722daa40fc..da32acea4e0 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -10,6 +10,7 @@ #define WARPX_LaserParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Laser/LaserProfiles.H" #include "WarpXParticleContainer.H" @@ -74,7 +75,7 @@ public: const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false) final; + bool skip_deposition=false, PushType push_type=PushType::Explicit) final; void PushP (int lev, amrex::Real dt, const amrex::MultiFab& , diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index 5c2950759d3..88da5b0afad 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -9,6 +9,7 @@ #include "LaserParticleContainer.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Laser/LaserProfiles.H" #include "Particles/LaserParticleContainer.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -561,7 +562,7 @@ LaserParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, - Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition) + Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition, PushType push_type) { WARPX_PROFILE("LaserParticleContainer::Evolve()"); WARPX_PROFILE_VAR_NS("LaserParticleContainer::Evolve::ParticlePush", blp_pp); @@ -664,14 +665,14 @@ LaserParticleContainer::Evolve (int lev, // Deposit inside domains DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, 0, np_current, thread_num, - lev, lev, dt, relative_time); + lev, lev, dt, relative_time, push_type); if (has_buffer) { // Deposit in buffers DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, - lev, lev-1, dt, relative_time); + lev, lev-1, dt, relative_time, push_type); } } diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index c32ff07464d..285b0a9777c 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -14,6 +14,7 @@ #include "MultiParticleContainer_fwd.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Collision/CollisionHandler.H" #ifdef WARPX_QED # include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper_fwd.H" @@ -107,7 +108,8 @@ public: amrex::MultiFab* rho, amrex::MultiFab* crho, const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false); + amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, + PushType push_type=PushType::Explicit); /// /// This pushes the particle positions by one half time step for all the species in the @@ -331,6 +333,9 @@ public: [[nodiscard]] int getSpeciesID (std::string product_str) const; + amrex::Vector>::iterator begin() {return allcontainers.begin();} + amrex::Vector>::iterator end() {return allcontainers.end();} + protected: #ifdef WARPX_QED diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 213a5344ee8..a2e26559300 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -455,7 +455,8 @@ MultiParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { if (! skip_deposition) { jx.setVal(0.0); @@ -469,7 +470,7 @@ MultiParticleContainer::Evolve (int lev, } for (auto& pc : allcontainers) { pc->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, cjx, cjy, cjz, - rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt, a_dt_type, skip_deposition); + rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt, a_dt_type, skip_deposition, push_type); } } diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index c9d0eae3c4b..34afac53482 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -9,6 +9,7 @@ #define WARPX_PhotonParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Gather/ScaleFields.H" #include "PhysicalParticleContainer.H" @@ -69,7 +70,8 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; void PushPX(WarpXParIter& pti, amrex::FArrayBox const * exfab, @@ -124,7 +126,8 @@ public: int const /*lev*/, int const /*depos_lev*/, amrex::Real const /*dt*/, - amrex::Real const /*relative_time*/) override {} + amrex::Real const /*relative_time*/, + PushType /*push_type*/) override {} }; #endif // #ifndef WARPX_PhotonParticleContainer_H_ diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index 732d986a897..aa9f04be224 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -240,7 +240,8 @@ PhotonParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { // This does gather, push and deposit. // Push and deposit have been re-written for photons @@ -252,6 +253,6 @@ PhotonParticleContainer::Evolve (int lev, rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, - t, dt, a_dt_type, skip_deposition); + t, dt, a_dt_type, skip_deposition, push_type); } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 04c9682486c..ecc6f9536e1 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -11,6 +11,7 @@ #define WARPX_PhysicalParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Initialization/PlasmaInjector.H" #include "Particles/ElementaryProcess/Ionization.H" #ifdef WARPX_QED @@ -105,6 +106,7 @@ public: * \param dt time step by which particles are advanced * \param a_dt_type type of time step (used for sub-cycling) * \param skip_deposition Skip the charge and current deposition. + * \param push_type Type of particle push, explicit or implicit. Defaults to explicit * * Evolve iterates over particle iterator (each box) and performs filtering, * field gather, particle push and current deposition for all particles @@ -134,7 +136,8 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false ) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; virtual void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, @@ -150,6 +153,20 @@ public: amrex::Real dt, ScaleFields scaleFields, DtType a_dt_type=DtType::Full); + void ImplicitPushXP (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + amrex::IntVect ngEB, int /*e_is_nodal*/, + long offset, + long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type=DtType::Full); + void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 729abd6f092..cabd7320f58 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2047,7 +2047,8 @@ PhysicalParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition) + Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { WARPX_PROFILE("PhysicalParticleContainer::Evolve()"); @@ -2176,10 +2177,17 @@ PhysicalParticleContainer::Evolve (int lev, WARPX_PROFILE_VAR_START(blp_fg); const auto np_to_push = np_gather; const auto gather_lev = lev; - PushPX(pti, exfab, eyfab, ezfab, - bxfab, byfab, bzfab, - Ex.nGrowVect(), e_is_nodal, - 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + if (push_type == PushType::Explicit) { + PushPX(pti, exfab, eyfab, ezfab, + bxfab, byfab, bzfab, + Ex.nGrowVect(), e_is_nodal, + 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + } else if (push_type == PushType::Implicit) { + ImplicitPushXP(pti, exfab, eyfab, ezfab, + bxfab, byfab, bzfab, + Ex.nGrowVect(), e_is_nodal, + 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + } if (np_gather < np) { @@ -2210,11 +2218,19 @@ PhysicalParticleContainer::Evolve (int lev, // Field gather and push for particles in gather buffers e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); - PushPX(pti, cexfab, ceyfab, cezfab, - cbxfab, cbyfab, cbzfab, - cEx->nGrowVect(), e_is_nodal, - nfine_gather, np-nfine_gather, - lev, lev-1, dt, ScaleFields(false), a_dt_type); + if (push_type == PushType::Explicit) { + PushPX(pti, cexfab, ceyfab, cezfab, + cbxfab, cbyfab, cbzfab, + cEx->nGrowVect(), e_is_nodal, + nfine_gather, np-nfine_gather, + lev, lev-1, dt, ScaleFields(false), a_dt_type); + } else if (push_type == PushType::Implicit) { + ImplicitPushXP(pti, cexfab, ceyfab, cezfab, + cbxfab, cbyfab, cbzfab, + cEx->nGrowVect(), e_is_nodal, + nfine_gather, np-nfine_gather, + lev, lev-1, dt, ScaleFields(false), a_dt_type); + } } WARPX_PROFILE_VAR_STOP(blp_fg); @@ -2222,8 +2238,8 @@ PhysicalParticleContainer::Evolve (int lev, // Current Deposition if (!skip_deposition) { - // Deposit at t_{n+1/2} - const amrex::Real relative_time = -0.5_rt * dt; + // Deposit at t_{n+1/2} with explicit push + const amrex::Real relative_time = (push_type == PushType::Explicit ? -0.5_rt * dt : 0.0_rt); const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; @@ -2231,14 +2247,14 @@ PhysicalParticleContainer::Evolve (int lev, // Deposit inside domains DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, 0, np_current, thread_num, - lev, lev, dt, relative_time); + lev, lev, dt, relative_time, push_type); if (has_buffer) { // Deposit in buffers DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, - lev, lev-1, dt, relative_time); + lev, lev-1, dt, relative_time, push_type); } } // end of "if electrostatic_solver_id == ElectrostaticSolverAlgo::None" } // end of "if do_not_push" @@ -2924,26 +2940,40 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, if (!do_sync) #endif { - doParticlePush<0>(getPosition, setPosition, copyAttribs, ip, - ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, - m, q, pusher_algo, do_crr, do_copy, + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, #ifdef WARPX_QED - t_chi_max, + t_chi_max, #endif - dt); + dt); + + UpdatePosition(xp, yp, zp, ux[ip], uy[ip], uz[ip], dt); + setPosition(ip, xp, yp, zp); } #ifdef WARPX_QED else { if constexpr (qed_control == has_qed) { - doParticlePush<1>(getPosition, setPosition, copyAttribs, ip, - ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, - m, q, pusher_algo, do_crr, do_copy, - t_chi_max, - dt); + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, + t_chi_max, + dt); + + UpdatePosition(xp, yp, zp, ux[ip], uy[ip], uz[ip], dt); + setPosition(ip, xp, yp, zp); } } #endif @@ -2965,6 +2995,261 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, }); } +/* \brief Perform the implicit particle push operation in one fused kernel + * The main difference from PushPX is the order of operations: + * - push position by 1/2 dt + * - gather fields + * - push velocity by dt + * - average old and new velocity to get time centered value + * The routines ends with both position and velocity at the half time level. + */ +void +PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + amrex::IntVect ngEB, int /*e_is_nodal*/, + long offset, + long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE((gather_lev==(lev-1)) || + (gather_lev==(lev )), + "Gather buffers only work for lev-1"); + // If no particles, do not do anything + if (np_to_push == 0) { return; } + + // Get cell size on gather_lev + const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + + // Get box from which field is gathered. + // If not gathering from the finest level, the box is coarsened. + Box box; + if (lev == gather_lev) { + box = pti.tilebox(); + } else { + const IntVect& ref_ratio = WarpX::RefRatio(gather_lev); + box = amrex::coarsen(pti.tilebox(),ref_ratio); + } + + // Add guard cells to the box. + box.grow(ngEB); + + auto setPosition = SetParticlePosition(pti, offset); + + const auto getExternalEB = GetExternalEBField(pti, offset); + + const amrex::ParticleReal Ex_external_particle = m_E_external_particle[0]; + const amrex::ParticleReal Ey_external_particle = m_E_external_particle[1]; + const amrex::ParticleReal Ez_external_particle = m_E_external_particle[2]; + const amrex::ParticleReal Bx_external_particle = m_B_external_particle[0]; + const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; + const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; + + // Lower corner of tile box physical domain (take into account Galilean shift) + const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + + const Dim3 lo = lbound(box); + + bool galerkin_interpolation = WarpX::galerkin_interpolation; + int nox = WarpX::nox; + int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + + amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + + amrex::Array4 const& ex_arr = exfab->array(); + amrex::Array4 const& ey_arr = eyfab->array(); + amrex::Array4 const& ez_arr = ezfab->array(); + amrex::Array4 const& bx_arr = bxfab->array(); + amrex::Array4 const& by_arr = byfab->array(); + amrex::Array4 const& bz_arr = bzfab->array(); + + amrex::IndexType const ex_type = exfab->box().ixType(); + amrex::IndexType const ey_type = eyfab->box().ixType(); + amrex::IndexType const ez_type = ezfab->box().ixType(); + amrex::IndexType const bx_type = bxfab->box().ixType(); + amrex::IndexType const by_type = byfab->box().ixType(); + amrex::IndexType const bz_type = bzfab->box().ixType(); + + auto& attribs = pti.GetAttribs(); + ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr() + offset; + ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr() + offset; + ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr() + offset; + +#if (AMREX_SPACEDIM >= 2) + ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + int do_copy = (m_do_back_transformed_particles && (a_dt_type!=DtType::SecondHalf) ); + CopyParticleAttribs copyAttribs; + if (do_copy) { + copyAttribs = CopyParticleAttribs(pti, tmp_particle_data, offset); + } + + int* AMREX_RESTRICT ion_lev = nullptr; + if (do_field_ionization) { + ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr() + offset; + } + + // Loop over the particles and update their momentum + const amrex::ParticleReal q = this->charge; + const amrex::ParticleReal m = this-> mass; + + const auto pusher_algo = WarpX::particle_pusher_algo; + const auto do_crr = do_classical_radiation_reaction; +#ifdef WARPX_QED + const auto do_sync = m_do_qed_quantum_sync; + amrex::Real t_chi_max = 0.0; + if (do_sync) { t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); } + + QuantumSynchrotronEvolveOpticalDepth evolve_opt; + amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR = nullptr; + const bool local_has_quantum_sync = has_quantum_sync(); + if (local_has_quantum_sync) { + evolve_opt = m_shr_p_qs_engine->build_evolve_functor(); + p_optical_depth_QSR = pti.GetAttribs(particle_comps["opticalDepthQSR"]).dataPtr() + offset; + } +#endif + + const auto t_do_not_gather = do_not_gather; + + enum exteb_flags : int { no_exteb, has_exteb }; + enum qed_flags : int { no_qed, has_qed }; + + int exteb_runtime_flag = getExternalEB.isNoOp() ? no_exteb : has_exteb; +#ifdef WARPX_QED + int qed_runtime_flag = (local_has_quantum_sync || do_sync) ? has_qed : no_qed; +#else + int qed_runtime_flag = no_qed; +#endif + + // Using this version of ParallelFor with compile time options + // improves performance when qed or external EB are not used by reducing + // register pressure. + amrex::ParallelFor(TypeList, + CompileTimeOptions>{}, + {exteb_runtime_flag, qed_runtime_flag}, + np_to_push, [=] AMREX_GPU_DEVICE (long ip, auto exteb_control, + auto qed_control) + { + // Position advance starts from the position at the start of the step + // but uses the most recent velocity. +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal xp = x_n[ip]; + amrex::ParticleReal xp_n = x_n[ip]; +#else + amrex::ParticleReal xp = 0._rt; + amrex::ParticleReal xp_n = 0._rt; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal yp = y_n[ip]; + amrex::ParticleReal yp_n = y_n[ip]; +#else + amrex::ParticleReal yp = 0._rt; + amrex::ParticleReal yp_n = 0._rt; +#endif + amrex::ParticleReal zp = z_n[ip]; + amrex::ParticleReal zp_n = z_n[ip]; + + UpdatePositionImplicit(xp, yp, zp, ux_n[ip], uy_n[ip], uz_n[ip], ux[ip], uy[ip], uz[ip], 0.5_rt*dt); + setPosition(ip, xp, yp, zp); + + amrex::ParticleReal Exp = Ex_external_particle; + amrex::ParticleReal Eyp = Ey_external_particle; + amrex::ParticleReal Ezp = Ez_external_particle; + amrex::ParticleReal Bxp = Bx_external_particle; + amrex::ParticleReal Byp = By_external_particle; + amrex::ParticleReal Bzp = Bz_external_particle; + + if(!t_do_not_gather){ + // first gather E and B to the particle positions + doGatherShapeNImplicit(xp_n, yp_n, zp_n, xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + nox, galerkin_interpolation); + } + + // Externally applied E and B-field in Cartesian co-ordinates + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; + if constexpr (exteb_control == has_exteb) { + getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + } + + scaleFields(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + // The momentum push starts with the velocity at the start of the step + ux[ip] = ux_n[ip]; + uy[ip] = uy_n[ip]; + uz[ip] = uz_n[ip]; + +#ifdef WARPX_QED + if (!do_sync) +#endif + { + doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, +#ifdef WARPX_QED + t_chi_max, +#endif + dt); + } +#ifdef WARPX_QED + else { + if constexpr (qed_control == has_qed) { + doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 0, + m, q, pusher_algo, do_crr, + t_chi_max, + dt); + } + } +#endif + +#ifdef WARPX_QED + [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; + [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; + [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc + if constexpr (qed_control == has_qed) { + if (local_has_quantum_sync) { + evolve_opt(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp,Bxp, Byp, Bzp, + dt, p_optical_depth_QSR[ip]); + } + } +#else + amrex::ignore_unused(qed_control); +#endif + + // Take average to get the time centered value + ux[ip] = 0.5_rt*(ux[ip] + ux_n[ip]); + uy[ip] = 0.5_rt*(uy[ip] + uy_n[ip]); + uz[ip] = 0.5_rt*(uz[ip] + uz_n[ip]); + + }); +} + void PhysicalParticleContainer::InitIonizationModule () { diff --git a/Source/Particles/Pusher/PushSelector.H b/Source/Particles/Pusher/PushSelector.H index fd75f97f7c3..2a6ac8a7d6c 100644 --- a/Source/Particles/Pusher/PushSelector.H +++ b/Source/Particles/Pusher/PushSelector.H @@ -12,7 +12,6 @@ #include "Particles/Pusher/UpdateMomentumBorisWithRadiationReaction.H" #include "Particles/Pusher/UpdateMomentumHigueraCary.H" #include "Particles/Pusher/UpdateMomentumVay.H" -#include "Particles/Pusher/UpdatePosition.H" #include "Particles/WarpXParticleContainer.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -21,13 +20,9 @@ #include /** - * \brief Push position and momentum for a single particle + * \brief Push momentum for a single particle * * \tparam do_sync Whether to include quantum synchrotron radiation (QSR) - * \param GetPosition A functor for returning the particle position. - * \param SetPosition A functor for setting the particle position. - * \param copyAttribs A functor for storing the old u and x - * \param i The index of the particle to work on * \param ux, uy, uz Particle momentum * \param Ex, Ey, Ez Electric field on particles. * \param Bx, By, Bz Magnetic field on particles. @@ -36,41 +31,34 @@ * \param a_q Charge of this species. * \param pusher_algo 0: Boris, 1: Vay, 2: HigueraCary * \param do_crr Whether to do the classical radiation reaction - * \param do_copy Whether to copy the old x and u for the BTD * \param t_chi_max Cutoff chi for QSR * \param dt Time step size */ template AMREX_GPU_DEVICE AMREX_FORCE_INLINE -void doParticlePush(const GetParticlePosition& GetPosition, - const SetParticlePosition& SetPosition, - const CopyParticleAttribs& copyAttribs, - const long i, - amrex::ParticleReal& ux, - amrex::ParticleReal& uy, - amrex::ParticleReal& uz, - const amrex::ParticleReal Ex, - const amrex::ParticleReal Ey, - const amrex::ParticleReal Ez, - const amrex::ParticleReal Bx, - const amrex::ParticleReal By, - const amrex::ParticleReal Bz, - const int ion_lev, - const amrex::ParticleReal m, - const amrex::ParticleReal a_q, - const int pusher_algo, - const int do_crr, - const int do_copy, +void doParticleMomentumPush(amrex::ParticleReal& ux, + amrex::ParticleReal& uy, + amrex::ParticleReal& uz, + const amrex::ParticleReal Ex, + const amrex::ParticleReal Ey, + const amrex::ParticleReal Ez, + const amrex::ParticleReal Bx, + const amrex::ParticleReal By, + const amrex::ParticleReal Bz, + const int ion_lev, + const amrex::ParticleReal m, + const amrex::ParticleReal a_q, + const int pusher_algo, + const int do_crr, #ifdef WARPX_QED - const amrex::Real t_chi_max, + const amrex::Real t_chi_max, #endif - const amrex::Real dt) + const amrex::Real dt) { amrex::ParticleReal qp = a_q; if (ion_lev) { qp *= ion_lev; } - if (do_copy) { copyAttribs(i); } if (do_crr) { #ifdef WARPX_QED amrex::ignore_unused(t_chi_max); @@ -88,10 +76,6 @@ void doParticlePush(const GetParticlePosition& GetPosition, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); } - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else #endif { @@ -99,35 +83,19 @@ void doParticlePush(const GetParticlePosition& GetPosition, UpdateMomentumBorisWithRadiationReaction(ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } } else if (pusher_algo == ParticlePusherAlgo::Boris) { UpdateMomentumBoris( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else if (pusher_algo == ParticlePusherAlgo::Vay) { UpdateMomentumVay( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else if (pusher_algo == ParticlePusherAlgo::HigueraCary) { UpdateMomentumHigueraCary( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } //else { // amrex::Abort("Unknown particle pusher"); // } diff --git a/Source/Particles/Pusher/UpdatePosition.H b/Source/Particles/Pusher/UpdatePosition.H index 4bce5c10211..4743c79c00f 100644 --- a/Source/Particles/Pusher/UpdatePosition.H +++ b/Source/Particles/Pusher/UpdatePosition.H @@ -16,7 +16,10 @@ /** \brief Push the particle's positions over one timestep, - * given the value of its momenta `ux`, `uy`, `uz` */ + * given the value of its momenta `ux`, `uy`, `uz`. + * This uses the standard leapfrog algorithm + * x^{n+1} - x^{n} = dt*u^{n+1/2}/gamma^{n+1/2} + */ AMREX_GPU_HOST_DEVICE AMREX_INLINE void UpdatePosition(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, const amrex::ParticleReal ux, const amrex::ParticleReal uy, const amrex::ParticleReal uz, @@ -42,4 +45,43 @@ void UpdatePosition(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::Parti z += uz * inv_gamma * dt; } +/** \brief Push the particle's positions over one timestep, + * given the value of its momenta `ux`, `uy`, `uz`. + * The implicit version is the Crank-Nicolson scheme, + * x^{n+1} - x^{n} = dt*(u^{n+1} + u^{n})/(gamma^{n+1} + gamma^{n}) + * See Eqs. 15 and 17 in Chen, JCP 407 (2020) 109228. + */ +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void UpdatePositionImplicit(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, + const amrex::ParticleReal ux_n, const amrex::ParticleReal uy_n, const amrex::ParticleReal uz_n, + const amrex::ParticleReal ux, const amrex::ParticleReal uy, const amrex::ParticleReal uz, + const amrex::Real dt ) +{ + using namespace amrex::literals; + + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The ux,uy,uz are the velocities at time level n+1/2 + const amrex::ParticleReal ux_np1 = 2._prt*ux - ux_n; + const amrex::ParticleReal uy_np1 = 2._prt*uy - uy_n; + const amrex::ParticleReal uz_np1 = 2._prt*uz - uz_n; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (ux_n*ux_n + uy_n*uy_n + uz_n*uz_n)*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (ux_np1*ux_np1 + uy_np1*uy_np1 + uz_np1*uz_np1)*inv_c2); + const amrex::ParticleReal inv_gamma = 2.0_prt/(gamma_n + gamma_np1); + + // Update positions over one time step +#if (AMREX_SPACEDIM >= 2) + x += ux * inv_gamma * dt; +#else + amrex::ignore_unused(x); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) // RZ pushes particles in 3D + y += uy * inv_gamma * dt; +#else + amrex::ignore_unused(y); +#endif + z += uz * inv_gamma * dt; +} + #endif // WARPX_PARTICLES_PUSHER_UPDATEPOSITION_H_ diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index 0d5bf0d0f0e..bc20420ea6e 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -9,6 +9,7 @@ #define WARPX_RigidInjectedParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Gather/ScaleFields.H" #include "PhysicalParticleContainer.H" @@ -84,7 +85,8 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false ) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 162645a9e57..d50851ffb1b 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -299,7 +299,8 @@ RigidInjectedParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { // Update location of injection plane in the boosted frame @@ -324,7 +325,7 @@ RigidInjectedParticleContainer::Evolve (int lev, rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, - t, dt, a_dt_type, skip_deposition); + t, dt, a_dt_type, skip_deposition, push_type); } void diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index 60e54b5d59b..a0b4ed63a30 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -89,8 +89,14 @@ struct Compute_shifted_shape_factor const T x_old, const int i_new) const { - if constexpr (depos_order == 1){ - const auto i = static_cast(x_old); + if constexpr (depos_order == 0){ + const auto i = static_cast(std::floor(x_old + T(0.5))); + const int i_shift = i - i_new; + sx[1+i_shift] = T(1.0); + return i; + } + else if constexpr (depos_order == 1){ + const auto i = static_cast(std::floor(x_old)); const int i_shift = i - i_new; const T xint = x_old - T(i); sx[1+i_shift] = T(1.0) - xint; diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 024cd8d9157..1e2f3ff3862 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -13,6 +13,7 @@ #include "WarpXParticleContainer_fwd.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Initialization/PlasmaInjector.H" #include "Particles/ParticleBoundaries.H" #include "SpeciesPhysicalProperties.H" @@ -151,7 +152,8 @@ public: amrex::MultiFab* rho, amrex::MultiFab* crho, const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false) = 0; + amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, + PushType push_type=PushType::Explicit) = 0; virtual void PostRestart () = 0; @@ -250,7 +252,8 @@ public: int lev, int depos_lev, amrex::Real dt, - amrex::Real relative_time); + amrex::Real relative_time, + PushType push_type); // If particles start outside of the domain, ContinuousInjection // makes sure that they are initialized when they enter the domain, and diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 447721a3e68..29c8cd3fddb 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -323,7 +323,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, amrex::MultiFab * const jx, amrex::MultiFab * const jy, amrex::MultiFab * const jz, long const offset, long const np_to_deposit, int const thread_num, const int lev, int const depos_lev, - amrex::Real const dt, amrex::Real const relative_time) + amrex::Real const dt, amrex::Real const relative_time, PushType push_type) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || (depos_lev==(lev )), @@ -502,6 +502,9 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, WARPX_PROFILE_VAR_STOP(blp_get_max_tilesize); // Now pick current deposition algorithm + if (push_type == PushType::Implicit) { + amrex::Abort("Cannot do shared memory deposition with implicit algorithm"); + } if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { WARPX_ABORT_WITH_MESSAGE("Cannot do shared memory deposition with Esirkepov algorithm"); } @@ -538,29 +541,80 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, // If not doing shared memory deposition, call normal kernels else { if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { - if (WarpX::nox == 1){ - doEsirkepovDepositionShapeN<1>( + if (push_type == PushType::Explicit) { + if (WarpX::nox == 1){ + doEsirkepovDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 2){ - doEsirkepovDepositionShapeN<2>( + } else if (WarpX::nox == 2){ + doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 3){ - doEsirkepovDepositionShapeN<3>( + } else if (WarpX::nox == 3){ + doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); + } + } else if (push_type == PushType::Implicit) { +#if (AMREX_SPACEDIM >= 2) + auto& xp_n = pti.GetAttribs(particle_comps["x_n"]); + const ParticleReal* xp_n_data = xp_n.dataPtr() + offset; +#else + const ParticleReal* xp_n_data = nullptr; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + auto& yp_n = pti.GetAttribs(particle_comps["y_n"]); + const ParticleReal* yp_n_data = yp_n.dataPtr() + offset; +#else + const ParticleReal* yp_n_data = nullptr; +#endif + auto& zp_n = pti.GetAttribs(particle_comps["z_n"]); + const ParticleReal* zp_n_data = zp_n.dataPtr() + offset; + auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); + auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); + auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + if (WarpX::nox == 1){ + doChargeConservingDepositionShapeNImplicit<1>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doChargeConservingDepositionShapeNImplicit<2>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doChargeConservingDepositionShapeNImplicit<3>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } } } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { + if (push_type == PushType::Implicit) { + WARPX_ABORT_WITH_MESSAGE("The Vay algorithm cannot be used with implicit algorithm."); + } if (WarpX::nox == 1){ doVayDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, @@ -584,27 +638,61 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, WarpX::load_balance_costs_update_algo); } } else { // Direct deposition - if (WarpX::nox == 1){ - doDepositionShapeN<1>( + if (push_type == PushType::Explicit) { + if (WarpX::nox == 1){ + doDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 2){ - doDepositionShapeN<2>( + } else if (WarpX::nox == 2){ + doDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 3){ - doDepositionShapeN<3>( + } else if (WarpX::nox == 3){ + doDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); + } + } else if (push_type == PushType::Implicit) { + auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); + auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); + auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + if (WarpX::nox == 1){ + doDepositionShapeNImplicit<1>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doDepositionShapeNImplicit<2>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doDepositionShapeNImplicit<3>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } } } } @@ -653,7 +741,7 @@ WarpXParticleContainer::DepositCurrent ( DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, J[lev][0].get(), J[lev][1].get(), J[lev][2].get(), - 0, np, thread_num, lev, lev, dt, relative_time); + 0, np, thread_num, lev, lev, dt, relative_time, PushType::Explicit); } #ifdef AMREX_USE_OMP } diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 7ad33409900..87db0ae9b9b 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -23,6 +23,17 @@ struct MediumForEM { }; }; +/** + * \brief struct to select the overall evolve scheme + */ +struct EvolveScheme { + enum { + Explicit = 0, + ImplicitPicard = 1, + SemiImplicitPicard = 2 + }; +}; + /** * \brief struct to select algorithm for macroscopic Maxwell solver LaxWendroff (semi-implicit) represents sigma*E = sigma*0.5*(E^(n) + E^(n+1)) diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 2123a65627b..75c488134e0 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -23,6 +23,13 @@ // Define dictionary with correspondence between user-input strings, // and corresponding integer for use inside the code +const std::map evolve_scheme_to_int = { + {"explicit", EvolveScheme::Explicit }, + {"implicit_picard", EvolveScheme::ImplicitPicard }, + {"semi_implicit_picard", EvolveScheme::SemiImplicitPicard }, + {"default", EvolveScheme::Explicit } +}; + const std::map grid_to_int = { {"collocated", GridType::Collocated}, {"staggered", GridType::Staggered}, @@ -147,7 +154,9 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ // Pick the right dictionary std::map algo_to_int; - if (0 == std::strcmp(pp_search_key, "maxwell_solver")) { + if (0 == std::strcmp(pp_search_key, "evolve_scheme")) { + algo_to_int = evolve_scheme_to_int; + } else if (0 == std::strcmp(pp_search_key, "maxwell_solver")) { algo_to_int = electromagnetic_solver_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "grid_type")) { algo_to_int = grid_to_int; diff --git a/Source/WarpX.H b/Source/WarpX.H index c644aa67982..7a7c5ee89c1 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -37,6 +37,7 @@ #endif #include "AcceleratorLattice/AcceleratorLattice.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "FieldSolver/ElectrostaticSolver.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" #include "Filter/BilinearFilter.H" @@ -120,6 +121,12 @@ public: void Evolve (int numsteps = -1); + void EvolveImplicitPicardInit (int lev); + void SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev); + void FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev); + void FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Efield_fp, + amrex::Vector, 3 > >& Efield_n); + MultiParticleContainer& GetPartContainer () { return *mypc; } MultiFluidContainer& GetFluidContainer () { return *myfl; } MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; } @@ -157,6 +164,14 @@ public: static short particle_pusher_algo; //! Integer that corresponds to the type of Maxwell solver (Yee, CKC, PSATD, ECT) static short electromagnetic_solver_id; + //! Integer that corresponds to the evolve scheme (explicit, implicit_picard, semi_implicit_picard) + static short evolve_scheme; + //! The maximum number of Picard iterations to do each time step + static int max_picard_iterations; + //! The tolerance for the Picard iteration convergence + static amrex::Real picard_iteration_tolerance; + //! Flags whether the Picard iterations are required to converge + static bool require_picard_convergence; /** Records a number corresponding to the load balance cost update strategy * being used (0, 1, 2 corresponding to timers, heuristic, or gpuclock). */ @@ -420,6 +435,25 @@ public: const std::string& name, std::optional initial_value); + /** + * \brief + * Allocate the MultiFab so that is like the specified MultiFab (same ba and dm) + * and optionally initialize it. This also adds the MultiFab + * to the map of MultiFabs (used to ease the access to MultiFabs from the Python + * interface + * + * \param mf[out] The MultiFab unique pointer to be allocated + * \param mf_model[in] The MultiFab to model + * \param name[in] The name of the MultiFab to use in the map + * \param initial_value[in] The optional initial value + */ + static void AllocInitMultiFabFromModel ( + std::unique_ptr& mf, + amrex::MultiFab& mf_model, + int level, + const std::string& name, + std::optional initial_value = {}); + // Maps of all of the MultiFabs and iMultiFabs used (this can include MFs from other classes) // This is a convenience for the Python interface, allowing all MultiFabs // to be easily referenced from Python. @@ -739,8 +773,10 @@ public: void doQEDEvents (int lev); #endif - void PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false); - void PushParticlesandDeposit (amrex::Real cur_time, bool skip_current=false); + void PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false, + PushType push_type=PushType::Explicit); + void PushParticlesandDeposit (amrex::Real cur_time, bool skip_current=false, + PushType push_type=PushType::Explicit); // This function does aux(lev) = fp(lev) + I(aux(lev-1)-cp(lev)). // Caller must make sure fp and cp have ghost cells filled. @@ -1226,6 +1262,8 @@ private: void OneStep_nosub (amrex::Real cur_time); void OneStep_sub1 (amrex::Real cur_time); + void OneStep_ImplicitPicard(amrex::Real cur_time); + /** * \brief Perform one PIC iteration, with the multiple J deposition per time step */ @@ -1433,6 +1471,12 @@ private: amrex::Vector, 3 > > Efield_avg_fp; amrex::Vector, 3 > > Bfield_avg_fp; + // Implicit, fields at start of step and from the previous iteration + amrex::Vector, 3 > > Efield_n; + amrex::Vector, 3 > > Bfield_n; + amrex::Vector, 3 > > Efield_save; + amrex::Vector, 3 > > Bfield_save; + // Memory buffers for computing magnetostatic fields // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order amrex::Vector, 3 > > vector_potential_fp_nodal; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index e862391383e..f3008e6157d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -111,6 +111,10 @@ short WarpX::charge_deposition_algo; short WarpX::field_gathering_algo; short WarpX::particle_pusher_algo; short WarpX::electromagnetic_solver_id; +short WarpX::evolve_scheme; +int WarpX::max_picard_iterations = 10; +Real WarpX::picard_iteration_tolerance = 1.e-7; +bool WarpX::require_picard_convergence = true; short WarpX::psatd_solution_type; short WarpX::J_in_time; short WarpX::rho_in_time; @@ -1142,6 +1146,7 @@ WarpX::ReadParameters () current_deposition_algo = static_cast(GetAlgorithmInteger(pp_algo, "current_deposition")); charge_deposition_algo = static_cast(GetAlgorithmInteger(pp_algo, "charge_deposition")); particle_pusher_algo = static_cast(GetAlgorithmInteger(pp_algo, "particle_pusher")); + evolve_scheme = static_cast(GetAlgorithmInteger(pp_algo, "evolve_scheme")); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo != CurrentDepositionAlgo::Esirkepov || @@ -1222,6 +1227,44 @@ WarpX::ReadParameters () macroscopic_solver_algo = GetAlgorithmInteger(pp_algo,"macroscopic_sigma_method"); } + if (evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard) { + utils::parser::queryWithParser(pp_algo, "max_picard_iterations", max_picard_iterations); + utils::parser::queryWithParser(pp_algo, "picard_iteration_tolerance", picard_iteration_tolerance); + utils::parser::queryWithParser(pp_algo, "require_picard_convergence", require_picard_convergence); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + current_deposition_algo == CurrentDepositionAlgo::Esirkepov || + current_deposition_algo == CurrentDepositionAlgo::Direct, + "Only Esirkepov or Direct current deposition supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || + electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC, + "Only the Yee EM solver is supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + particle_pusher_algo == ParticlePusherAlgo::Boris || + particle_pusher_algo == ParticlePusherAlgo::HigueraCary, + "Only the Boris and Higuera particle pushers are supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + field_gathering_algo != GatheringAlgo::MomentumConserving, + "With implicit and semi-implicit schemes, the momentum conserving field gather is not supported as it would not conserve energy"); + + if (current_deposition_algo == CurrentDepositionAlgo::Direct) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !galerkin_interpolation, + "With implicit and semi-implicit schemes and direct deposition, the Galerkin field gathering must be turned off in order to conserve energy"); + } + + if (current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + galerkin_interpolation, + "With implicit and semi-implicit schemes and Esirkepov deposition, the Galerkin field gathering must be turned on in order to conserve energy"); + } + } + // Load balancing parameters std::vector load_balance_intervals_string_vec = {"0"}; pp_algo.queryarr("load_balance_intervals", load_balance_intervals_string_vec); @@ -2087,6 +2130,11 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d AllocLevelMFs(lev, ba, dm, guard_cells.ng_alloc_EB, guard_cells.ng_alloc_J, guard_cells.ng_alloc_Rho, guard_cells.ng_alloc_F, guard_cells.ng_alloc_G, aux_is_nodal); + if (evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard) { + EvolveImplicitPicardInit(lev); + } + m_accelerator_lattice[lev] = std::make_unique(); m_accelerator_lattice[lev]->InitElementFinder(lev, ba, dm); @@ -3266,3 +3314,21 @@ WarpX::AliasInitMultiFab ( } multifab_map[name_with_suffix] = mf.get(); } + +void +WarpX::AllocInitMultiFabFromModel ( + std::unique_ptr& mf, + amrex::MultiFab& mf_model, + const int level, + const std::string& name, + std::optional initial_value) +{ + const auto name_with_suffix = TagWithLevelSuffix(name, level); + const auto tag = amrex::MFInfo().SetTag(name_with_suffix); + mf = std::make_unique(mf_model.boxArray(), mf_model.DistributionMap(), + mf_model.nComp(), mf_model.nGrowVect(), tag); + if (initial_value) { + mf->setVal(*initial_value); + } + multifab_map[name_with_suffix] = mf.get(); +} From f275c7adcd733b15806e48e618e94629a2ba4f7e Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Fri, 22 Dec 2023 20:04:31 -0800 Subject: [PATCH 082/176] Fix default initialization of runtime attributes. (#4508) * Refactor DefaultInitializeRuntimeAttributes to free function template * refactor filterCopyTransform * update ParticleCreationFunc * do filterCreateTransformFromFAB * fix ionization initial level * add start and stop indices * trying to make it run on GPU based on the allocator * run on GPU or not depending on allocator type * fix typo * fix non-QED compilation * fix non-QED builds correctly * fix narrowing * fix segfault * Call user runtime attributes after initializing position and momenta * don't re-initialize QED comps if they've already been set * Add automated test * Update json for new test * Fix bug in AddNParticles * Add comments --------- Co-authored-by: Remi Lehe --- .../Tests/ionization/analysis_ionization.py | 8 + Examples/Tests/ionization/inputs_2d_rt | 2 + .../benchmarks_json/ionization_lab.json | 1 + .../BackgroundMCC/BackgroundMCCCollision.cpp | 2 +- .../BinaryCollision/BinaryCollision.H | 10 +- .../BinaryCollision/ParticleCreationFunc.H | 23 ++ Source/Particles/LaserParticleContainer.H | 3 +- Source/Particles/MultiParticleContainer.cpp | 8 +- .../ParticleCreation/DefaultInitialization.H | 199 ++++++++++++++++++ .../ParticleCreation/FilterCopyTransform.H | 84 ++++++-- .../FilterCreateTransformFromFAB.H | 58 ++++- Source/Particles/PhysicalParticleContainer.H | 31 ++- .../Particles/PhysicalParticleContainer.cpp | 114 ++-------- Source/Particles/WarpXParticleContainer.H | 19 +- Source/Particles/WarpXParticleContainer.cpp | 4 +- 15 files changed, 421 insertions(+), 145 deletions(-) diff --git a/Examples/Tests/ionization/analysis_ionization.py b/Examples/Tests/ionization/analysis_ionization.py index e5d61fc0c0a..95732b03e36 100755 --- a/Examples/Tests/ionization/analysis_ionization.py +++ b/Examples/Tests/ionization/analysis_ionization.py @@ -93,5 +93,13 @@ assert( error_rel < tolerance_rel ) +# Check that the user runtime component (if it exists) worked as expected +try: + orig_z = ad['electrons', 'particle_orig_z'].v + assert np.all( (orig_z > 0) & (orig_z < 1.5e-5) ) + print('particle_orig_z has reasonable values') +except yt.utilities.exceptions.YTFieldNotFound: + pass # Some of the tested script to not have the quantity orig_z + test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/ionization/inputs_2d_rt b/Examples/Tests/ionization/inputs_2d_rt index 130eb1cc46a..f7035c567ac 100644 --- a/Examples/Tests/ionization/inputs_2d_rt +++ b/Examples/Tests/ionization/inputs_2d_rt @@ -36,6 +36,8 @@ ions.physical_element = N electrons.mass = m_e electrons.charge = -q_e electrons.injection_style = none +electrons.addRealAttributes = orig_z +electrons.attribute.orig_z(x,y,z,ux,uy,uz,t) = z lasers.names = laser1 laser1.profile = Gaussian diff --git a/Regression/Checksum/benchmarks_json/ionization_lab.json b/Regression/Checksum/benchmarks_json/ionization_lab.json index 3fafb968f29..f1be9ec3017 100644 --- a/Regression/Checksum/benchmarks_json/ionization_lab.json +++ b/Regression/Checksum/benchmarks_json/ionization_lab.json @@ -3,6 +3,7 @@ "particle_momentum_x": 4.407898469197755e-18, "particle_momentum_y": 0.0, "particle_momentum_z": 2.642991174223682e-18, + "particle_orig_z": 0.43016526372226926, "particle_position_x": 0.1095015206652257, "particle_position_y": 0.6413864600981052, "particle_weight": 3.443203125e-10 diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index ece9b073339..672c6456ec7 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -501,7 +501,7 @@ void BackgroundMCCCollision::doBackgroundIonization m_mass1, sqrt_kb_m, m_background_temperature_func, t ); - const auto num_added = filterCopyTransformParticles<1>( + const auto num_added = filterCopyTransformParticles<1>(species1, species2, elec_tile, ion_tile, elec_tile, np_elec, np_ion, Filter, CopyElec, CopyIon, Transform ); diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 19e98b8a15a..5c90dab25e6 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -174,7 +174,7 @@ public: amrex::LayoutData* cost = WarpX::getCosts(lev); - // Loop over all grids/tiles at this level + // Loop over all grids/tiles at this level #ifdef AMREX_USE_OMP info.SetDynamic(true); #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -368,7 +368,9 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_1, tile_products_data, + soa_1, soa_1, + product_species_vector, + tile_products_data, particle_ptr_1, particle_ptr_1, m1, m1, products_mass, p_mask, products_np, copy_species1, copy_species2, @@ -530,7 +532,9 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_2, tile_products_data, + soa_1, soa_2, + product_species_vector, + tile_products_data, particle_ptr_1, particle_ptr_2, m1, m2, products_mass, p_mask, products_np, copy_species1, copy_species2, diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index 9ad1b7a6cbf..dc830b477df 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -100,6 +100,7 @@ public: amrex::Vector operator() ( const index_type& n_total_pairs, const SoaData_type& soa_1, const SoaData_type& soa_2, + const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, ParticleType* particle_ptr_1, ParticleType* particle_ptr_2, const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, @@ -251,6 +252,27 @@ public: } }); + // Initialize the user runtime components + for (int i = 0; i < m_num_product_species; i++) + { + int start_index = int(products_np[i]); + int stop_index = int(products_np[i] + num_added_vec[i]); + ParticleCreation::DefaultInitializeRuntimeAttributes(*tile_products[i], + 0, 0, + pc_products[i]->getUserRealAttribs(), pc_products[i]->getUserIntAttribs(), + pc_products[i]->getParticleComps(), pc_products[i]->getParticleiComps(), + pc_products[i]->getUserRealAttribParser(), + pc_products[i]->getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the SmartCopy functors + pc_products[i]->get_breit_wheeler_engine_ptr(), + pc_products[i]->get_quantum_sync_engine_ptr(), +#endif + pc_products[i]->getIonizationInitialLevel(), + start_index, stop_index); + } + amrex::Gpu::synchronize(); return num_added_vec; @@ -289,6 +311,7 @@ public: amrex::Vector operator() ( const index_type& /*n_total_pairs*/, const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, + amrex::Vector& /*pc_products*/, ParticleTileType** /*tile_products*/, ParticleType* /*particle_ptr_1*/, ParticleType* /*particle_ptr_2*/, const amrex::ParticleReal& /*m1*/, const amrex::ParticleReal& /*m2*/, diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index da32acea4e0..e6fa308431c 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -59,8 +59,7 @@ public: amrex::ParticleTile, NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& /*pinned_tile*/, const int /*n_external_attr_real*/, - const int /*n_external_attr_int*/, - const amrex::RandomEngine& /*engine*/) final {} + const int /*n_external_attr_int*/) final {} void ReadHeader (std::istream& is) final; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index a2e26559300..a31f426a0e4 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -901,7 +901,7 @@ MultiParticleContainer::doFieldIonization (int lev, Bx[pti], By[pti], Bz[pti]); const auto np_dst = dst_tile.numParticles(); - const auto num_added = filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, + const auto num_added = filterCopyTransformParticles<1>(*pc_product, dst_tile, src_tile, np_dst, Filter, Copy, Transform); setNewParticleIDs(dst_tile, np_dst, num_added); @@ -1388,7 +1388,7 @@ MultiParticleContainer::doQEDSchwinger () const auto Transform = SchwingerTransformFunc{m_qed_schwinger_y_size, PIdx::w}; - const auto num_added = filterCreateTransformFromFAB<1>( dst_ele_tile, + const auto num_added = filterCreateTransformFromFAB<1>( *pc_product_ele, *pc_product_pos, dst_ele_tile, dst_pos_tile, box, fieldsEB, np_ele_dst, np_pos_dst,Filter, CreateEle, CreatePos, Transform); @@ -1541,7 +1541,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, const auto np_dst_ele = dst_ele_tile.numParticles(); const auto np_dst_pos = dst_pos_tile.numParticles(); - const auto num_added = filterCopyTransformParticles<1>( + const auto num_added = filterCopyTransformParticles<1>(*pc_product_ele, *pc_product_pos, dst_ele_tile, dst_pos_tile, src_tile, np_dst_ele, np_dst_pos, Filter, CopyEle, CopyPos, Transform); @@ -1621,7 +1621,7 @@ void MultiParticleContainer::doQedQuantumSync (int lev, const auto np_dst = dst_tile.numParticles(); const auto num_added = - filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, + filterCopyTransformParticles<1>(*pc_product_phot, dst_tile, src_tile, np_dst, Filter, CopyPhot, Transform); setNewParticleIDs(dst_tile, np_dst, num_added); diff --git a/Source/Particles/ParticleCreation/DefaultInitialization.H b/Source/Particles/ParticleCreation/DefaultInitialization.H index 09ff5cce6f0..870fc82bd0f 100644 --- a/Source/Particles/ParticleCreation/DefaultInitialization.H +++ b/Source/Particles/ParticleCreation/DefaultInitialization.H @@ -8,6 +8,12 @@ #ifndef DEFAULTINITIALIZATION_H_ #define DEFAULTINITIALIZATION_H_ +#include +#ifdef WARPX_QED +# include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H" +# include "Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H" +#endif + #include #include @@ -81,4 +87,197 @@ int initializeIntValue (const InitializationPolicy policy) noexcept } } +namespace ParticleCreation { + + /** + * \brief Default initialize runtime attributes in a tile. This routine does not initialize the + * first n_external_attr_real real attributes and the first n_external_attr_int integer + * attributes, which have been in principle externally set elsewhere. + * + * @tparam[in] The type of the particle tile to operate on (e.g. could use different allocators) + * @param[inout] ptile the tile in which attributes are initialized + * @param[in] n_external_attr_real The number of real attributes that have been externally set. + * These are NOT initialized by this function. + * @param[in] n_external_attr_int The number of integer attributes that have been externally set. + * These are NOT initialized by this function. + * @param[in] user_real_attribs The names of the real components for this particle tile + * @param[in] user_int_attribs The names of the int components for this particle tile + * @param[in] particle_comps map between particle component index and component name for real comps + * @param[in] particle_icomps map between particle component index and component name for int comps + * @param[in] user_real_attrib_parser the parser functions used to initialize the user real components + * @param[in] user_int_attrib_parser the parser functions used to initialize the user int components + * @param[in] do_qed_comps whether to initialize the qed components (these are usually handled by + * SmartCopy, but NOT when adding particles in AddNParticles) + * @param[in] p_bw_engine the engine to use for setting the breit-wheeler component for QED + * @param[in] p_qs_engine the engine to use for setting the quantum synchrotron component for QED + * @param[in] ionization_initial_level the ionization level particles created should start at + * @param[in] start the index to start initializing particles + * @param[in] stop the index to stop initializing particles + */ +template +void DefaultInitializeRuntimeAttributes (PTile& ptile, + const int n_external_attr_real, + const int n_external_attr_int, + const std::vector& user_real_attribs, + const std::vector& user_int_attribs, + const std::map& particle_comps, + const std::map& particle_icomps, + const std::vector& user_real_attrib_parser, + const std::vector& user_int_attrib_parser, +#ifdef WARPX_QED + const bool do_qed_comps, + BreitWheelerEngine* p_bw_engine, + QuantumSynchrotronEngine* p_qs_engine, +#endif + const int ionization_initial_level, + int start, int stop) +{ + using namespace amrex::literals; + + // Preparing data needed for user defined attributes + const auto n_user_real_attribs = static_cast(user_real_attribs.size()); + const auto n_user_int_attribs = static_cast(user_int_attribs.size()); + const auto get_position = GetParticlePosition(ptile); + const auto soa = ptile.getParticleTileData(); + const amrex::ParticleReal* AMREX_RESTRICT ux = soa.m_rdata[PIdx::ux]; + const amrex::ParticleReal* AMREX_RESTRICT uy = soa.m_rdata[PIdx::uy]; + const amrex::ParticleReal* AMREX_RESTRICT uz = soa.m_rdata[PIdx::uz]; + constexpr int lev = 0; + const amrex::Real t = WarpX::GetInstance().gett_new(lev); + + // Initialize the last NumRuntimeRealComps() - n_external_attr_real runtime real attributes + for (int j = PIdx::nattribs + n_external_attr_real; j < ptile.NumRealComps() ; ++j) + { + auto attr_ptr = ptile.GetStructOfArrays().GetRealData(j).data(); +#ifdef WARPX_QED + // Current runtime comp is quantum synchrotron optical depth + if (particle_comps.find("opticalDepthQSR") != particle_comps.end() && + particle_comps.at("opticalDepthQSR") == j) + { + if (!do_qed_comps) { continue; } + const QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt = + p_qs_engine->build_optical_depth_functor(); + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelForRNG(stop - start, + [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept { + int ip = i + start; + attr_ptr[ip] = quantum_sync_get_opt(engine); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = quantum_sync_get_opt(amrex::RandomEngine{}); + } + } + } + + // Current runtime comp is Breit-Wheeler optical depth + if (particle_comps.find("opticalDepthBW") != particle_comps.end() && + particle_comps.at("opticalDepthBW") == j) + { + if (!do_qed_comps) { continue; } + const BreitWheelerGetOpticalDepth breit_wheeler_get_opt = + p_bw_engine->build_optical_depth_functor();; + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelForRNG(stop - start, + [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept { + int ip = i + start; + attr_ptr[ip] = breit_wheeler_get_opt(engine); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = breit_wheeler_get_opt(amrex::RandomEngine{}); + } + } + } +#endif + + for (int ia = 0; ia < n_user_real_attribs; ++ia) + { + // Current runtime comp is ia-th user defined attribute + if (particle_comps.find(user_real_attribs[ia]) != particle_comps.end() && + particle_comps.at(user_real_attribs[ia]) == j) + { + const amrex::ParserExecutor<7> user_real_attrib_parserexec = + user_real_attrib_parser[ia]->compile<7>(); + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = user_real_attrib_parserexec(xp, yp, zp, + ux[ip], uy[ip], uz[ip], t); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = user_real_attrib_parserexec(xp, yp, zp, + ux[ip], uy[ip], uz[ip], t); + } + } + } + } + } + + // Initialize the last NumRuntimeIntComps() - n_external_attr_int runtime int attributes + for (int j = n_external_attr_int; j < ptile.NumIntComps() ; ++j) + { + auto attr_ptr = ptile.GetStructOfArrays().GetIntData(j).data(); + + // Current runtime comp is ionization level + if (particle_icomps.find("ionizationLevel") != particle_icomps.end() && + particle_icomps.at("ionizationLevel") == j) + { + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + attr_ptr[ip] = ionization_initial_level; + }); + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = ionization_initial_level; + } + } + } + + for (int ia = 0; ia < n_user_int_attribs; ++ia) + { + // Current runtime comp is ia-th user defined attribute + if (particle_icomps.find(user_int_attribs[ia]) != particle_icomps.end() && + particle_icomps.at(user_int_attribs[ia]) == j) + { + const amrex::ParserExecutor<7> user_int_attrib_parserexec = + user_int_attrib_parser[ia]->compile<7>(); + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = static_cast( + user_int_attrib_parserexec(xp, yp, zp, ux[ip], uy[ip], uz[ip], t)); + }); + } else { + for (int ip = start; ip < stop; ++ip) { + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = static_cast( + user_int_attrib_parserexec(xp, yp, zp, ux[ip], uy[ip], uz[ip], t)); + } + } + } + } + } +} + +} + #endif diff --git a/Source/Particles/ParticleCreation/FilterCopyTransform.H b/Source/Particles/ParticleCreation/FilterCopyTransform.H index 26d70fc6718..095ebbf6a85 100644 --- a/Source/Particles/ParticleCreation/FilterCopyTransform.H +++ b/Source/Particles/ParticleCreation/FilterCopyTransform.H @@ -8,6 +8,8 @@ #ifndef FILTER_COPY_TRANSFORM_H_ #define FILTER_COPY_TRANSFORM_H_ +#include "Particles/ParticleCreation/DefaultInitialization.H" + #include #include @@ -47,10 +49,11 @@ * * \return num_added the number of particles that were written to dst. */ -template ::value, int> foo = 0> -Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Index dst_index, +Index filterCopyTransformParticles (DstPC& pc, DstTile& dst, SrcTile& src, + Index* mask, Index dst_index, CopyFunc&& copy, TransFunc&& transform) noexcept { using namespace amrex; @@ -61,7 +64,9 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind Gpu::DeviceVector offsets(np); auto total = amrex::Scan::ExclusiveSum(np, mask, offsets.data()); const Index num_added = N * total; - dst.resize(std::max(dst_index + num_added, dst.numParticles())); + auto old_np = dst.size(); + auto new_np = std::max(dst_index + num_added, dst.numParticles()); + dst.resize(new_np); auto *const p_offsets = offsets.dataPtr(); @@ -80,6 +85,21 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst, + 0, 0, + pc.getUserRealAttribs(), pc.getUserIntAttribs(), + pc.getParticleComps(), pc.getParticleiComps(), + pc.getUserRealAttribParser(), + pc.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc.get_breit_wheeler_engine_ptr(), + pc.get_quantum_sync_engine_ptr(), +#endif + pc.getIonizationInitialLevel(), + old_np, new_np); + Gpu::synchronize(); return num_added; } @@ -121,9 +141,9 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind * * \return num_added the number of particles that were written to dst. */ -template -Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, +Index filterCopyTransformParticles (DstPC& pc, DstTile& dst, SrcTile& src, Index dst_index, PredFunc&& filter, CopyFunc&& copy, TransFunc&& transform) noexcept { using namespace amrex; @@ -142,9 +162,9 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, p_mask[i] = filter(src_data, i, engine); }); - return filterCopyTransformParticles(dst, src, mask.dataPtr(), dst_index, - std::forward(copy), - std::forward(transform)); + return filterCopyTransformParticles(pc, dst, src, mask.dataPtr(), dst_index, + std::forward(copy), + std::forward(transform)); } /** @@ -188,10 +208,10 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, * * \return num_added the number of particles that were written to dst. */ -template ::value, int> foo = 0> -Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, Index* mask, +Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, SrcTile& src, Index* mask, Index dst1_index, Index dst2_index, CopyFunc1&& copy1, CopyFunc2&& copy2, TransFunc&& transform) noexcept @@ -204,8 +224,13 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, Gpu::DeviceVector offsets(np); auto total = amrex::Scan::ExclusiveSum(np, mask, offsets.data()); const Index num_added = N * total; - dst1.resize(std::max(dst1_index + num_added, dst1.numParticles())); - dst2.resize(std::max(dst2_index + num_added, dst2.numParticles())); + auto old_np1 = dst1.size(); + auto new_np1 = std::max(dst1_index + num_added, dst1.numParticles()); + dst1.resize(new_np1); + + auto old_np2 = dst2.size(); + auto new_np2 = std::max(dst2_index + num_added, dst2.numParticles()); + dst2.resize(new_np2); auto *p_offsets = offsets.dataPtr(); @@ -230,6 +255,35 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst1, + 0, 0, + pc1.getUserRealAttribs(), pc1.getUserIntAttribs(), + pc1.getParticleComps(), pc1.getParticleiComps(), + pc1.getUserRealAttribParser(), + pc1.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc1.get_breit_wheeler_engine_ptr(), + pc1.get_quantum_sync_engine_ptr(), +#endif + pc1.getIonizationInitialLevel(), + old_np1, new_np1); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst2, + 0, 0, + pc2.getUserRealAttribs(), pc2.getUserIntAttribs(), + pc2.getParticleComps(), pc2.getParticleiComps(), + pc2.getUserRealAttribParser(), + pc2.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc2.get_breit_wheeler_engine_ptr(), + pc2.get_quantum_sync_engine_ptr(), +#endif + pc2.getIonizationInitialLevel(), + old_np2, new_np2); + Gpu::synchronize(); return num_added; } @@ -276,9 +330,9 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, * * \return num_added the number of particles that were written to dst. */ -template -Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, +Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, SrcTile& src, Index dst1_index, Index dst2_index, PredFunc&& filter, CopyFunc1&& copy1, CopyFunc2&& copy2, TransFunc&& transform) noexcept @@ -299,7 +353,7 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, p_mask[i] = filter(src_data, i, engine); }); - return filterCopyTransformParticles(dst1, dst2, src, mask.dataPtr(), + return filterCopyTransformParticles(pc1, pc2, dst1, dst2, src, mask.dataPtr(), dst1_index, dst2_index, std::forward(copy1), std::forward(copy2), diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 8f16530f020..2a4c9fccad0 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -8,6 +8,7 @@ #ifndef FILTER_CREATE_TRANSFORM_FROM_FAB_H_ #define FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#include "Particles/ParticleCreation/DefaultInitialization.H" #include "WarpX.H" #include @@ -42,14 +43,15 @@ * * \return num_added the number of particles that were written to dst1 and dst2. */ -template ::value, int> foo = 0> -Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::Box box, - const FAB *src_FAB, const Index* mask, - const Index dst1_index, const Index dst2_index, - CreateFunc1&& create1, CreateFunc2&& create2, - TransFunc&& transform) noexcept +Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, + DstTile& dst1, DstTile& dst2, const amrex::Box box, + const FAB *src_FAB, const Index* mask, + const Index dst1_index, const Index dst2_index, + CreateFunc1&& create1, CreateFunc2&& create2, + TransFunc&& transform) noexcept { using namespace amrex; @@ -83,8 +85,13 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B Gpu::DeviceVector offsets(ncells); auto total = amrex::Scan::ExclusiveSum(ncells, mask, offsets.data()); const Index num_added = N*total; - dst1.resize(std::max(dst1_index + num_added, dst1.numParticles())); - dst2.resize(std::max(dst2_index + num_added, dst2.numParticles())); + auto old_np1 = dst1.size(); + auto new_np1 = std::max(dst1_index + num_added, dst1.numParticles()); + dst1.resize(new_np1); + + auto old_np2 = dst2.size(); + auto new_np2 = std::max(dst2_index + num_added, dst2.numParticles()); + dst2.resize(new_np2); auto *p_offsets = offsets.dataPtr(); @@ -130,6 +137,35 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst1, + 0, 0, + pc1.getUserRealAttribs(), pc1.getUserIntAttribs(), + pc1.getParticleComps(), pc1.getParticleiComps(), + pc1.getUserRealAttribParser(), + pc1.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CreateFunc functor + pc1.get_breit_wheeler_engine_ptr(), + pc1.get_quantum_sync_engine_ptr(), +#endif + pc1.getIonizationInitialLevel(), + old_np1, new_np1); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst2, + 0, 0, + pc2.getUserRealAttribs(), pc2.getUserIntAttribs(), + pc2.getParticleComps(), pc2.getParticleiComps(), + pc2.getUserRealAttribParser(), + pc2.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CreateFunc functor + pc2.get_breit_wheeler_engine_ptr(), + pc2.get_quantum_sync_engine_ptr(), +#endif + pc2.getIonizationInitialLevel(), + old_np2, new_np2); + Gpu::synchronize(); return num_added; } @@ -166,10 +202,10 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B * * \return num_added the number of particles that were written to dst1 and dst2. */ -template -Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::Box box, +Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, const amrex::Box box, const FABs& src_FABs, const Index dst1_index, const Index dst2_index, FilterFunc&& filter, CreateFunc1&& create1, CreateFunc2&& create2, @@ -201,7 +237,7 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B p_mask[mask_position] = (arrNumPartCreation(i,j,k) > 0); }); - return filterCreateTransformFromFAB(dst1, dst2, box, &NumPartCreation, + return filterCreateTransformFromFAB(pc1, pc2, dst1, dst2, box, &NumPartCreation, mask.dataPtr(), dst1_index, dst2_index, std::forward(create1), std::forward(create2), diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index ecc6f9536e1..1ec3473e81b 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -272,8 +272,7 @@ public: NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& pinned_tile, int n_external_attr_real, - int n_external_attr_int, - const amrex::RandomEngine& engine) final; + int n_external_attr_int) final; /** * \brief Apply NCI Godfrey filter to all components of E and B before gather @@ -370,11 +369,35 @@ public: (std::shared_ptr ptr) override; //__________ + BreitWheelerEngine* get_breit_wheeler_engine_ptr () const override { + return m_shr_p_bw_engine.get(); + } + + QuantumSynchrotronEngine* get_quantum_sync_engine_ptr () const override { + return m_shr_p_qs_engine.get(); + } + PhotonEmissionFilterFunc getPhotonEmissionFilterFunc (); PairGenerationFilterFunc getPairGenerationFilterFunc (); #endif + std::vector getUserIntAttribs () const override { + return m_user_int_attribs; + } + + std::vector getUserRealAttribs () const override { + return m_user_real_attribs; + } + + amrex::Vector< amrex::Parser* > getUserIntAttribParser () const override { + return GetVecOfPtrs(m_user_int_attrib_parser); + } + + amrex::Vector< amrex::Parser* > getUserRealAttribParser () const override { + return GetVecOfPtrs(m_user_real_attrib_parser); + } + protected: std::string species_name; std::vector> plasma_injectors; @@ -425,9 +448,9 @@ protected: /* Vector of user-defined real attributes for species, species_name */ std::vector m_user_real_attribs; /* Vector of user-defined parser for initializing user-defined integer attributes */ - std::vector< std::unique_ptr > m_user_int_attrib_parser; + amrex::Vector< std::unique_ptr > m_user_int_attrib_parser; /* Vector of user-defined parser for initializing user-defined real attributes */ - std::vector< std::unique_ptr > m_user_real_attrib_parser; + amrex::Vector< std::unique_ptr > m_user_real_attrib_parser; }; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index cabd7320f58..cfd0074d616 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -21,6 +21,7 @@ #endif #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" +#include "Particles/ParticleCreation/DefaultInitialization.H" #include "Particles/Pusher/CopyParticleAttribs.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/Pusher/PushSelector.H" @@ -779,110 +780,21 @@ PhysicalParticleContainer::DefaultInitializeRuntimeAttributes ( NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& pinned_tile, const int n_external_attr_real, - const int n_external_attr_int, - const amrex::RandomEngine& engine) + const int n_external_attr_int) { - using namespace amrex::literals; - - const int np = pinned_tile.numParticles(); - - // Preparing data needed for user defined attributes - const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); - const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); - const auto get_position = GetParticlePosition(pinned_tile); - const auto soa = pinned_tile.getParticleTileData(); - const amrex::ParticleReal* AMREX_RESTRICT ux = soa.m_rdata[PIdx::ux]; - const amrex::ParticleReal* AMREX_RESTRICT uy = soa.m_rdata[PIdx::uy]; - const amrex::ParticleReal* AMREX_RESTRICT uz = soa.m_rdata[PIdx::uz]; - constexpr int lev = 0; - const amrex::Real t = WarpX::GetInstance().gett_new(lev); - -#ifndef WARPX_QED - amrex::ignore_unused(engine); -#endif - - // Initialize the last NumRuntimeRealComps() - n_external_attr_real runtime real attributes - for (int j = PIdx::nattribs + n_external_attr_real; j < NumRealComps() ; ++j) - { - amrex::Vector attr_temp(np, 0.0_prt); + ParticleCreation::DefaultInitializeRuntimeAttributes(pinned_tile, + n_external_attr_real, n_external_attr_int, + m_user_real_attribs, m_user_int_attribs, + particle_comps, particle_icomps, + amrex::GetVecOfPtrs(m_user_real_attrib_parser), + amrex::GetVecOfPtrs(m_user_int_attrib_parser), #ifdef WARPX_QED - // Current runtime comp is quantum synchrotron optical depth - if (particle_comps.find("opticalDepthQSR") != particle_comps.end() && - particle_comps["opticalDepthQSR"] == j) - { - const QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt = - m_shr_p_qs_engine->build_optical_depth_functor();; - for (int i = 0; i < np; ++i) { - attr_temp[i] = quantum_sync_get_opt(engine); - } - } - - // Current runtime comp is Breit-Wheeler optical depth - if (particle_comps.find("opticalDepthBW") != particle_comps.end() && - particle_comps["opticalDepthBW"] == j) - { - const BreitWheelerGetOpticalDepth breit_wheeler_get_opt = - m_shr_p_bw_engine->build_optical_depth_functor();; - for (int i = 0; i < np; ++i) { - attr_temp[i] = breit_wheeler_get_opt(engine); - } - } + true, + m_shr_p_bw_engine.get(), + m_shr_p_qs_engine.get(), #endif - - for (int ia = 0; ia < n_user_real_attribs; ++ia) - { - // Current runtime comp is ia-th user defined attribute - if (particle_comps.find(m_user_real_attribs[ia]) != particle_comps.end() && - particle_comps[m_user_real_attribs[ia]] == j) - { - amrex::ParticleReal xp, yp, zp; - const amrex::ParserExecutor<7> user_real_attrib_parserexec = - m_user_real_attrib_parser[ia]->compile<7>(); - for (int i = 0; i < np; ++i) { - get_position(i, xp, yp, zp); - attr_temp[i] = user_real_attrib_parserexec(xp, yp, zp, - ux[i], uy[i], uz[i], t); - } - } - } - - pinned_tile.push_back_real(j, attr_temp.data(), attr_temp.data() + np); - } - - // Initialize the last NumRuntimeIntComps() - n_external_attr_int runtime int attributes - for (int j = n_external_attr_int; j < NumIntComps() ; ++j) - { - amrex::Vector attr_temp(np, 0); - - // Current runtime comp is ionization level - if (particle_icomps.find("ionizationLevel") != particle_icomps.end() && - particle_icomps["ionizationLevel"] == j) - { - for (int i = 0; i < np; ++i) { - attr_temp[i] = ionization_initial_level; - } - } - - for (int ia = 0; ia < n_user_int_attribs; ++ia) - { - // Current runtime comp is ia-th user defined attribute - if (particle_icomps.find(m_user_int_attribs[ia]) != particle_icomps.end() && - particle_icomps[m_user_int_attribs[ia]] == j) - { - amrex::ParticleReal xp, yp, zp; - const amrex::ParserExecutor<7> user_int_attrib_parserexec = - m_user_int_attrib_parser[ia]->compile<7>(); - for (int i = 0; i < np; ++i) { - get_position(i, xp, yp, zp); - attr_temp[i] = static_cast( - user_int_attrib_parserexec(xp, yp, zp, ux[i], uy[i], uz[i], t)); - } - } - } - - pinned_tile.push_back_int(j, attr_temp.data(), attr_temp.data() + np); - } - + ionization_initial_level, + 0,pinned_tile.numParticles()); } diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 1e2f3ff3862..e791ac25d22 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -168,8 +168,7 @@ public: NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& pinned_tile, int n_external_attr_real, - int n_external_attr_int, - const amrex::RandomEngine& engine) = 0; + int n_external_attr_int) = 0; /// /// This pushes the particle positions by one half time step. @@ -407,6 +406,19 @@ public: */ void defineAllParticleTiles () noexcept; + virtual std::vector getUserIntAttribs () const { return std::vector{}; } + + virtual std::vector getUserRealAttribs () const { return std::vector{}; } + + virtual amrex::Vector< amrex::Parser* > getUserIntAttribParser () const { return amrex::Vector{}; } + + virtual amrex::Vector< amrex::Parser* > getUserRealAttribParser () const { return amrex::Vector{}; } + +#ifdef WARPX_QED + virtual BreitWheelerEngine* get_breit_wheeler_engine_ptr () const {return nullptr;} + virtual QuantumSynchrotronEngine* get_quantum_sync_engine_ptr () const {return nullptr;} +#endif + protected: int species_id; @@ -476,6 +488,9 @@ public: using TmpParticles = amrex::Vector >; TmpParticles getTmpParticleData () const noexcept {return tmp_particle_data;} + + int getIonizationInitialLevel () const noexcept {return ionization_initial_level;} + protected: TmpParticles tmp_particle_data; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 29c8cd3fddb..f1b19dd8407 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -279,9 +279,9 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, pinned_tile.push_back_int(j, attr_int[j].data() + ibegin, attr_int[j].data() + iend); } + pinned_tile.resize(np); // Default initialize the other real and integer runtime attributes - DefaultInitializeRuntimeAttributes(pinned_tile, nattr_real - 1, nattr_int, - amrex::RandomEngine{}); + DefaultInitializeRuntimeAttributes(pinned_tile, nattr_real - 1, nattr_int); auto old_np = particle_tile.numParticles(); auto new_np = old_np + pinned_tile.numParticles(); From ae600b83b7b987b1d4a70836cc36a36077d52932 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 22 Dec 2023 21:20:16 -0800 Subject: [PATCH 083/176] Update explanation of why 2D simulations are not suitable for plasma-wakefield (#4544) * Update physics explanation * Apply suggestions from code review --- Examples/Physics_applications/laser_acceleration/README.rst | 5 +++-- Examples/Physics_applications/plasma_acceleration/README.rst | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Examples/Physics_applications/laser_acceleration/README.rst b/Examples/Physics_applications/laser_acceleration/README.rst index 737cb1fe677..0ba3b5382f2 100644 --- a/Examples/Physics_applications/laser_acceleration/README.rst +++ b/Examples/Physics_applications/laser_acceleration/README.rst @@ -5,8 +5,9 @@ Laser-Wakefield Acceleration of Electrons This example shows how to model a laser-wakefield accelerator (LWFA) :cite:p:`ex-TajimaDawson1982,ex-Esarey1996`. -Laser-wakefield acceleration is best performed in 3D and quasi-cylindrical (RZ) geometry, which ensures that the plasma wavelength of the wakefield is modelled with the right scale lengths. -RZ modeling enables efficient modeling if effects of asymmetry shall be ignored (e.g., asymmetric beams and transverse profiles, hosing of the injected beam, etc.). +Laser-wakefield acceleration is best performed in 3D or quasi-cylindrical (RZ) geometry, in order to correctly capture some of the key physics (laser diffraction, beamloading, shape of the accelerating bubble in the blowout regime, etc.). +For physical situations that have close-to-cylindrical symmetry, simulations in RZ geometry capture the relevant physics at a fraction of the computational cost of a 3D simulation. +On the other hand, for physical situation with strong asymmetries (e.g., non-round laser driver, strong hosing of the accelerated beam, etc.), only 3D simulations are suitable. For LWFA scenarios with long propagation lengths, use the :ref:`boosted frame method `. An example can be seen in the :ref:`PWFA example `. diff --git a/Examples/Physics_applications/plasma_acceleration/README.rst b/Examples/Physics_applications/plasma_acceleration/README.rst index d5605dd93ef..d5775e93aa8 100644 --- a/Examples/Physics_applications/plasma_acceleration/README.rst +++ b/Examples/Physics_applications/plasma_acceleration/README.rst @@ -5,8 +5,9 @@ Beam-Driven Wakefield Acceleration of Electrons This example shows how to model a beam-driven plasma-wakefield accelerator (PWFA) :cite:p:`ex-TajimaDawson1982,ex-Esarey1996`. -PWFA is best performed in 3D and quasi-cylindrical (RZ) geometry, which ensures that the plasma wavelength of the wakefield is modelled with the right scale lengths. -RZ modeling enables efficient modeling if effects of asymmetry shall be ignored (e.g., asymmetric beams and transverse profiles, hosing of the injected beam, etc.). +PWFA is best performed in 3D or quasi-cylindrical (RZ) geometry, in order to correctly capture some of the key physics (structure of the space-charge fields, beamloading, shape of the accelerating bubble in the blowout regime, etc.). +For physical situations that have close-to-cylindrical symmetry, simulations in RZ geometry capture the relevant physics at a fraction of the computational cost of a 3D simulation. +On the other hand, for physical situation with strong asymmetries (e.g., non-round driver, strong hosing of the accelerated beam, etc.), only 3D simulations are suitable. Additionally, to speed up computation, this example uses the :ref:`boosted frame method ` to effectively model long acceleration lengths. From 5cff5e041b1c58d8c6a9b919de5bf752779b410c Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Tue, 2 Jan 2024 10:49:16 -0800 Subject: [PATCH 084/176] Change the sign of laser particles (#4568) * Change the sign of laser particles The choice is somewhat arbitrary for current use cases. But it is necessary for boosted frame restart runs (#4119). Since this changes the results for the CI tests, we make this change a separate PR so that #4119 would be easier to review. * Update benchmarks from Azure output --- .../Checksum/benchmarks_json/BTD_rz.json | 12 +-- .../LaserAccelerationBoost.json | 56 ++++++------- .../benchmarks_json/LaserAccelerationRZ.json | 60 +++++++------- .../LaserAcceleration_BTD.json | 48 ++++++------ ...erAcceleration_single_precision_comms.json | 36 ++++----- .../LaserInjectionFromLASYFile_RZ.json | 12 +-- .../benchmarks_json/LaserIonAcc2d.json | 53 +++++++------ .../benchmarks_json/PlasmaMirror.json | 42 +++++----- .../Python_LaserAcceleration.json | 48 ++++++------ .../Python_LaserAccelerationRZ.json | 74 +++++++++--------- .../benchmarks_json/Python_ionization.json | 42 +++++----- .../benchmarks_json/RefinedInjection.json | 60 +++++++------- .../comoving_2d_psatd_hybrid.json | 62 +++++++-------- .../galilean_2d_psatd_hybrid.json | 56 ++++++------- .../benchmarks_json/ionization_boost.json | 38 ++++----- .../benchmarks_json/ionization_lab.json | 44 +++++------ .../Checksum/benchmarks_json/restart.json | 66 ++++++++-------- .../benchmarks_json/restart_psatd.json | 48 ++++++------ .../restart_psatd_time_avg.json | 78 +++++++++---------- Source/Particles/LaserParticleContainer.cpp | 2 +- 20 files changed, 468 insertions(+), 469 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/BTD_rz.json b/Regression/Checksum/benchmarks_json/BTD_rz.json index 3110120b70a..01e4c687292 100644 --- a/Regression/Checksum/benchmarks_json/BTD_rz.json +++ b/Regression/Checksum/benchmarks_json/BTD_rz.json @@ -1,13 +1,13 @@ { "lev=0": { - "Br": 1.8705552174367742e-08, - "Bt": 2380179.567792259, - "Bz": 2.4079075035536954e-08, + "Br": 1.8705552264208163e-08, + "Bt": 2380179.5677922587, + "Bz": 2.4079077116116535e-08, "Er": 497571119514841.25, - "Et": 7.048225282653208, - "Ez": 137058870936728.17, + "Et": 7.048225464720808, + "Ez": 137058870936728.28, "jr": 0.0, "jt": 0.0, "jz": 0.0 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json index ee19b84c7af..94e243f21a2 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 3.535284585563231e-19, - "particle_momentum_y": 4.403094613346061e-19, - "particle_momentum_z": 5.658013779496569e-17, - "particle_position_x": 0.008317876520240174, - "particle_position_y": 1.1704335094514386, - "particle_weight": 62415090744.60765 + "lev=0": { + "Bx": 4818955.485214876, + "By": 1752.8017794063862, + "Bz": 14516.212849468406, + "Ex": 2366115511749.6064, + "Ey": 1446112026972328.0, + "Ez": 21864189477873.78, + "jx": 1996366361598696.5, + "jy": 5.312583836700398e+16, + "jz": 2.0491352591140016e+16, + "rho": 68443961.6079968 }, "electrons": { - "particle_momentum_x": 2.2135959939611405e-23, - "particle_momentum_y": 2.822519730011994e-22, - "particle_momentum_z": 5.260625039372931e-22, - "particle_position_x": 0.010800577787577741, - "particle_position_y": 0.21115060628258137, + "particle_momentum_x": 2.2135944672847805e-23, + "particle_momentum_y": 2.82245592458273e-22, + "particle_momentum_z": 5.260626007649988e-22, + "particle_position_x": 0.010800577787630073, + "particle_position_y": 0.21115060628317733, "particle_weight": 4.121554826246186e+16 }, "ions": { - "particle_momentum_x": 6.248472008953412e-23, - "particle_momentum_y": 4.449200926395666e-22, - "particle_momentum_z": 5.768167708374496e-22, - "particle_position_x": 0.010800001678510793, - "particle_position_y": 0.21114947608115497, + "particle_momentum_x": 6.24847236820344e-23, + "particle_momentum_y": 4.449097670697282e-22, + "particle_momentum_z": 5.768168724446374e-22, + "particle_position_x": 0.010800001678510515, + "particle_position_y": 0.21114947608115428, "particle_weight": 4.121554826246186e+16 }, - "lev=0": { - "Bx": 4818965.813977506, - "By": 1752.781451346485, - "Bz": 14516.29947347909, - "Ex": 2366115950699.717, - "Ey": 1446115205306590.0, - "Ez": 21864183756771.12, - "jx": 1996366451911408.0, - "jy": 5.312642483130767e+16, - "jz": 2.049134468340688e+16, - "rho": 68443935.06252655 + "beam": { + "particle_momentum_x": 3.5357352601344873e-19, + "particle_momentum_y": 4.363147101327531e-19, + "particle_momentum_z": 5.658494028187168e-17, + "particle_position_x": 0.008314957057032625, + "particle_position_y": 1.1704335719922687, + "particle_weight": 62415090744.60765 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json index 0c6f5242b87..1602058dbdc 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json @@ -1,55 +1,55 @@ { "lev=0": { - "Br": 104965.61239547668, - "Br_0_real": 0.27473110662919303, - "Br_1_imag": 104.10451752777047, + "Br": 104965.61239547684, + "Br_0_real": 0.2747311066289638, + "Br_1_imag": 104.10451752777048, "Br_1_real": 104965.62478916143, - "Bt": 1296.2723279668432, - "Btheta_0_real": 1297.3296772773876, + "Bt": 1296.2723277779253, + "Btheta_0_real": 1297.3296772773874, "Btheta_1_imag": 105725.25398122355, - "Btheta_1_real": 141.255481019691, - "Bz": 5076.744762384442, - "Bz_0_real": 0.4603819957305941, - "Bz_1_imag": 1.007360812967319, + "Btheta_1_real": 141.2554810196911, + "Bz": 5076.74476238461, + "Bz_0_real": 0.4603819957306249, + "Bz_1_imag": 1.0073608129672305, "Bz_1_real": 5076.632351216658, - "Er": 273570494222.5625, + "Er": 273570493775.55588, "Er_0_real": 271974091013.7082, "Er_1_imag": 39530787242966.72, - "Er_1_real": 42616886403.53066, - "Et": 39016542462571.72, - "Etheta_0_real": 112249482.93967965, - "Etheta_1_imag": 33602799006.8748, + "Er_1_real": 42616886403.53069, + "Et": 39016542462571.68, + "Etheta_0_real": 112249482.93975669, + "Etheta_1_imag": 33602799006.874825, "Etheta_1_real": 39016517454881.91, - "Ez": 511653064866.5733, + "Ez": 511653064863.048, "Ez_0_real": 496845125310.57947, - "Ez_1_imag": 1245709520854.4875, - "Ez_1_real": 24849976994.356285, - "Jr_0_real": 1264417193501.7915, + "Ez_1_imag": 1245709520854.488, + "Ez_1_real": 24849976994.356266, + "Jr_0_real": 1264417193503.146, "Jr_1_imag": 2.3356633334993878e+17, - "Jr_1_real": 2272922066062.2944, + "Jr_1_real": 2272922066062.586, "Jtheta_0_real": 475304056513.7001, - "Jtheta_1_imag": 1028929701334.7467, + "Jtheta_1_imag": 1028929701334.8286, "Jtheta_1_real": 2.1766379427377802e+17, "Jz_0_real": 1832468408628522.8, "Jz_1_imag": 556484600934089.9, "Jz_1_real": 602703622358902.4, - "jr": 1749994884866.3667, - "jt": 2.176637940885753e+17, - "jz": 1954078685186198.5, - "rho": 39314210.88334631, + "jr": 1749985661974.403, + "jt": 2.1766379408857264e+17, + "jz": 1954078684852864.2, + "rho": 39314210.931097575, "rho_0_real": 38889615.839967206, - "rho_1_imag": 21546499.619336687, - "rho_1_real": 2012888.5658883716 + "rho_1_imag": 21546499.619336277, + "rho_1_real": 2012888.5658883736 }, "electrons": { - "particle_momentum_x": 1.3203417227476259e-24, + "particle_momentum_x": 1.3203417227476271e-24, "particle_momentum_y": 4.0070625287284664e-22, - "particle_momentum_z": 1.2493391415020164e-23, + "particle_momentum_z": 1.2493391415020162e-23, "particle_orig_x": 0.026508328457558912, "particle_orig_z": 0.04789125000000001, "particle_position_x": 0.04160250006680718, "particle_position_y": 0.04789125046517409, - "particle_theta": 7325.121266775259, + "particle_theta": 7325.160426984388, "particle_weight": 813672305.532158 }, "beam": { @@ -61,4 +61,4 @@ "particle_theta": 151.40797316586125, "particle_weight": 6241509.074460764 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json index 6b30064365c..dd224516c5c 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json @@ -1,32 +1,32 @@ { "lev=0": { - "Bx": 497087553.4743837, - "By": 252364344.72213668, - "Bz": 16986109.62608196, - "Ex": 7.304419665394173e+16, - "Ey": 1.4583459400150573e+17, - "Ez": 2.261309965998215e+16, - "jx": 8.273582439836952e+17, - "jy": 2.1044181577417728e+18, - "jz": 2.84208498868846e+19, - "rho": 91637594416.50476 + "Bx": 497756265.44724107, + "By": 252197580.117894, + "Bz": 16988475.5444833, + "Ex": 7.299911104263803e+16, + "Ey": 1.460116488178734e+17, + "Ez": 2.26033100286114e+16, + "jx": 8.27644503757345e+17, + "jy": 2.1062078409959675e+18, + "jz": 2.8491727096438305e+19, + "rho": 91863764144.41415 }, "electrons": { - "particle_momentum_x": 5.922677819206403e-20, - "particle_momentum_y": 2.7778096010261295e-19, - "particle_momentum_z": 4.183846407463124e-19, - "particle_position_x": 0.0025865980636989934, - "particle_position_y": 0.002652673793955211, - "particle_position_z": 0.18411922027879662, - "particle_weight": 1731649095408.5505 + "particle_momentum_x": 5.76165226700654e-20, + "particle_momentum_y": 2.7567389504898156e-19, + "particle_momentum_z": 4.134562048099117e-19, + "particle_position_x": 0.0025269863605945427, + "particle_position_y": 0.0024538321295153346, + "particle_position_z": 0.17818421763751244, + "particle_weight": 1675789447169.5652 }, "beam": { - "particle_momentum_x": 1.1925686969448298e-17, - "particle_momentum_y": 2.570810132551278e-17, - "particle_momentum_z": 5.156190278524037e-14, - "particle_position_x": 0.0005608477913384702, - "particle_position_y": 0.0008431191912437461, - "particle_position_z": 2.9800048756186395, + "particle_momentum_x": 1.1926313055043134e-17, + "particle_momentum_y": 2.7056205218547404e-17, + "particle_momentum_z": 5.1562131494813424e-14, + "particle_position_x": 0.000560821518739447, + "particle_position_y": 0.000800729816549036, + "particle_position_z": 2.9800048650570097, "particle_weight": 62415.090744607616 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json index 793d8c11cb3..30eef0cedb7 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json @@ -1,25 +1,25 @@ { + "lev=0": { + "Bx": 5863879.051452597, + "By": 2411.5124004699082, + "Bz": 116025.40178726945, + "Ex": 6267725953308.365, + "Ey": 1670763222566621.8, + "Ez": 104345977762699.77, + "jx": 555688154318027.4, + "jy": 1595897074125252.8, + "jz": 1045267363178542.9, + "rho": 2205684400.3678846 + }, "electrons": { "particle_initialenergy": 0.0, - "particle_momentum_x": 1.7921229236407606e-20, - "particle_momentum_y": 7.225825184653885e-20, - "particle_momentum_z": 4.231731488281653e-20, - "particle_position_x": 0.7139122621535502, - "particle_position_y": 0.7150340889556129, - "particle_position_z": 1.3175770602802896, "particle_regionofinterest": 1936.0, + "particle_position_x": 0.7139122620952513, + "particle_position_y": 0.7150340902640349, + "particle_position_z": 1.317577060220835, + "particle_momentum_x": 1.7921232385614662e-20, + "particle_momentum_y": 7.22582607277354e-20, + "particle_momentum_z": 4.231730764749506e-20, "particle_weight": 12926557617.187498 - }, - "lev=0": { - "Bx": 5863879.02030842, - "By": 2411.501904737579, - "Bz": 116025.42462998998, - "Ex": 6267728094111.663, - "Ey": 1670763233105822.0, - "Ez": 104345989831222.4, - "jx": 555687912108453.8, - "jy": 1595896363359136.0, - "jz": 1045267552192496.5, - "rho": 2205684400.307701 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json index d98aabd84e9..242cf00f0c3 100644 --- a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json +++ b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json @@ -1,12 +1,12 @@ { "lev=0": { "Br": 100278645.72225758, - "Bz": 2509008.1146699777, - "Er": 3.258880159474263, + "Bz": 2509008.1146699786, + "Er": 3.263343388037509, "Et": 3.0045297157982412e+16, - "Ez": 0.24010979707502123, - "jr": 50.13668665903749, - "jt": 4.305139762894391e+17, + "Ez": 0.2422526411295923, + "jr": 49.67802097760541, + "jt": 4.3051397628943917e+17, "jz": 0.0 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json index 36b2bc66aed..1678e794683 100644 --- a/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json +++ b/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json @@ -1,34 +1,33 @@ { - "electrons": { - "particle_momentum_x": 3.856058278845215e-19, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.633111056849423e-18, - "particle_position_x": 0.008155341094488198, - "particle_position_y": 0.03087362289643631, - "particle_weight": 2.6462205036771795e+17 - }, - "hydrogen": { - "particle_momentum_x": 2.237944099185357e-18, - "particle_momentum_z": 1.0744000122946915e-18, - "particle_orig_x": 0.007768349609375002, - "particle_orig_z": 0.035127407226562504, - "particle_position_x": 0.007767975241766832, - "particle_position_y": 0.03512562609400349, - "particle_weight": 2.68584931046923e+17 - }, "lev=0": { "Bx": 0.0, - "By": 11399442.711372176, + "By": 11411047.898351602, "Bz": 0.0, - "Ex": 2033057399920464.5, + "Ex": 2031641076084948.5, "Ey": 0.0, - "Ez": 339244862686685.9, - "jx": 1.6477362358927364e+19, + "Ez": 334777106874158.8, + "jx": 1.6602050255725978e+19, "jy": 0.0, - "jz": 9.633815033523364e+18, - "rho": 68121082972.17873, - "rho_electrons": 17448735668294.348, - "rho_hydrogen": 17441797017887.941 + "jz": 9.545451466990307e+18, + "rho": 67237998613.86053, + "rho_electrons": 17449705826002.385, + "rho_hydrogen": 17441792654743.146 + }, + "hydrogen": { + "particle_momentum_x": 2.2575255298569512e-18, + "particle_momentum_z": 1.0773391029868316e-18, + "particle_orig_x": 0.007763427734375002, + "particle_orig_z": 0.0351975439453125, + "particle_position_x": 0.007763034484095496, + "particle_position_y": 0.035195761508116416, + "particle_weight": 2.6792730619156992e+17 + }, + "electrons": { + "particle_momentum_x": 3.8463518636930147e-19, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.6287398567676136e-18, + "particle_position_x": 0.008139313907538123, + "particle_position_y": 0.0308605164193468, + "particle_weight": 2.647983436428149e+17 } -} - +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/PlasmaMirror.json b/Regression/Checksum/benchmarks_json/PlasmaMirror.json index 9a686e78d56..861f42f2d74 100644 --- a/Regression/Checksum/benchmarks_json/PlasmaMirror.json +++ b/Regression/Checksum/benchmarks_json/PlasmaMirror.json @@ -1,29 +1,29 @@ { - "electrons": { - "particle_momentum_x": 2.000457748960032e-17, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.446670640741138e-17, - "particle_position_x": 0.37688793289418476, - "particle_position_y": 0.1714047288635171, - "particle_weight": 2.904173860599889e+18 - }, - "ions": { - "particle_momentum_x": 2.805690782833416e-17, - "particle_momentum_y": 0.0, - "particle_momentum_z": 9.605283484426823e-17, - "particle_position_x": 0.3828171295926821, - "particle_position_y": 0.17237242725257168, - "particle_weight": 2.9212132379167017e+18 - }, "lev=0": { "Bx": 0.0, - "By": 75294856.83091721, + "By": 75293687.71417463, "Bz": 0.0, - "Ex": 2.081697252453305e+16, + "Ex": 2.0817302986029584e+16, "Ey": 0.0, - "Ez": 2.350511387513211e+16, - "jx": 5.496779405360711e+19, + "Ez": 2.350518662171605e+16, + "jx": 5.496864993586538e+19, "jy": 0.0, - "jz": 7.159348923050661e+19 + "jz": 7.159302467888838e+19 + }, + "ions": { + "particle_momentum_x": 2.8056907319288714e-17, + "particle_momentum_y": 0.0, + "particle_momentum_z": 9.605283524169195e-17, + "particle_position_x": 0.382817129592504, + "particle_position_y": 0.1723724272525192, + "particle_weight": 2.9212132379167017e+18 + }, + "electrons": { + "particle_momentum_x": 2.000448978774197e-17, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.446668081246458e-17, + "particle_position_x": 0.376887964713013, + "particle_position_y": 0.17140472970668535, + "particle_weight": 2.9041738605998894e+18 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json index 0b0355b7cb6..08969db023e 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json @@ -1,32 +1,32 @@ { + "lev=0": { + "Bx": 5866866.518334419, + "By": 11175.108378013305, + "Bz": 116026.79919117832, + "Ex": 8178060925462.576, + "Ey": 1671615340558021.2, + "Ez": 106548534464553.72, + "jx": 555903248104584.56, + "jy": 1595980426561245.2, + "jz": 1366292265497407.5, + "rho": 2206755250.226452 + }, "beam": { - "particle_momentum_x": 4.707586336874016e-20, - "particle_momentum_y": 4.4850722108112576e-20, - "particle_momentum_z": 1.36054441043288e-17, - "particle_position_x": 4.058764306495361e-05, - "particle_position_y": 3.7888695722549883e-05, - "particle_position_z": 0.00019656701118308398, + "particle_momentum_x": 4.7075864374777216e-20, + "particle_momentum_y": 4.4875025179072e-20, + "particle_momentum_z": 1.3605444179112827e-17, + "particle_position_x": 4.058764328440382e-05, + "particle_position_y": 3.788866110052658e-05, + "particle_position_z": 0.00019656700944015084, "particle_weight": 6241509.074460764 }, "electrons": { - "particle_momentum_x": 1.7921232203945004e-20, - "particle_momentum_y": 7.225819894813053e-20, - "particle_momentum_z": 4.231725460173154e-20, - "particle_position_x": 0.7139122621161993, - "particle_position_y": 0.7150340887578637, - "particle_position_z": 1.3175770600690966, + "particle_momentum_x": 1.7921232210868553e-20, + "particle_momentum_y": 7.225819896136567e-20, + "particle_momentum_z": 4.2317254599358777e-20, + "particle_position_x": 0.713912262116188, + "particle_position_y": 0.7150340887578024, + "particle_position_z": 1.31757706006908, "particle_weight": 12926557617.187498 - }, - "lev=0": { - "Bx": 5866866.85492377, - "By": 11177.920546471447, - "Bz": 116026.93444649166, - "Ex": 8178548880638.266, - "Ey": 1671614207789070.8, - "Ez": 106548168484665.61, - "jx": 555903247963958.4, - "jy": 1595974150308405.2, - "jz": 1366292284444382.5, - "rho": 2206755250.226321 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json index d4c8d2b0c4e..720148888e2 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json @@ -1,45 +1,54 @@ { "lev=0": { - "Br": 142258.01161290862, - "Br_0_real": 0.36356680791862006, - "Br_1_imag": 115.41915541351702, + "Br": 142258.01161290894, + "Br_0_real": 0.36356680791855334, + "Br_1_imag": 115.41915541351706, "Br_1_real": 142258.0180774376, - "Bt": 1301.5691011565148, + "Bt": 1301.5691003159159, "Btheta_0_real": 1299.8813681794945, "Btheta_1_imag": 143318.04184449295, "Btheta_1_real": 155.37799428725944, - "Bz": 5993.6411733668865, - "Bz_0_real": 0.47374186122724626, - "Bz_1_imag": 1.140891753831192, - "Bz_1_real": 5993.529100973487, - "Er": 278531478924.55994, + "Bz": 5993.641173367065, + "Bz_0_real": 0.4737418612272665, + "Bz_1_imag": 1.1408917538312353, + "Bz_1_real": 5993.529100973486, + "Er": 278531478379.8872, "Er_0_real": 276179480674.09875, - "Er_1_imag": 47911368149173.87, - "Er_1_real": 46900731766.21943, - "Et": 47328876349791.84, - "Etheta_0_real": 135868018.29309198, - "Etheta_1_imag": 36802947213.299355, + "Er_1_imag": 47911368149173.86, + "Er_1_real": 46900731766.219345, + "Et": 47328876349791.85, + "Etheta_0_real": 135868018.2930996, + "Etheta_1_imag": 36802947213.299324, "Etheta_1_real": 47328835574237.43, - "Ez": 514006688163.9632, + "Ez": 514006688162.3537, "Ez_0_real": 499008057521.7594, - "Ez_1_imag": 1565161964565.9727, - "Ez_1_real": 28898941539.846138, - "Jr_0_real": 1458783813352.3413, + "Ez_1_imag": 1565161964565.9724, + "Ez_1_real": 28898941539.846146, + "Jr_0_real": 1458783813352.7048, "Jr_1_imag": 2.335663323286376e+17, - "Jr_1_real": 2725720908169.8604, - "Jtheta_0_real": 499386783042.54987, - "Jtheta_1_imag": 1179203035377.7158, + "Jr_1_real": 2725720908168.6816, + "Jtheta_0_real": 499386783042.5499, + "Jtheta_1_imag": 1179203035377.2214, "Jtheta_1_real": 2.1766371791828432e+17, "Jz_0_real": 1832469778455967.8, "Jz_1_imag": 621923937700173.0, "Jz_1_real": 660910016274555.4, - "jr": 2108653492279.1746, - "jt": 2.1766371088503258e+17, - "jz": 1954236547127886.2, - "rho": 39480728.028152466, + "jr": 2108641839684.7007, + "jt": 2.1766371088502803e+17, + "jz": 1954236547059862.2, + "rho": 39480728.08547403, "rho_0_real": 39055923.162689775, - "rho_1_imag": 21660762.696674928, - "rho_1_real": 2131498.588887921 + "rho_1_imag": 21660762.696674515, + "rho_1_real": 2131498.5888879234 + }, + "electrons": { + "particle_momentum_x": 1.237777638089911e-24, + "particle_momentum_y": 3.9465600403988994e-22, + "particle_momentum_z": 1.0097949593654349e-23, + "particle_position_x": 0.04160250006989259, + "particle_position_y": 0.047891250488567585, + "particle_theta": 7325.162679147719, + "particle_weight": 813672305.532158 }, "beam": { "particle_momentum_x": 3.8798818966213747e-20, @@ -49,14 +58,5 @@ "particle_position_y": 0.0026764363296859625, "particle_theta": 151.40797595010355, "particle_weight": 6241509.074460764 - }, - "electrons": { - "particle_momentum_x": 1.2377776380899228e-24, - "particle_momentum_y": 3.9465600403988994e-22, - "particle_momentum_z": 1.009794959365435e-23, - "particle_position_x": 0.04160250006989259, - "particle_position_y": 0.047891250488567585, - "particle_theta": 7325.119014611929, - "particle_weight": 813672305.532158 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_ionization.json b/Regression/Checksum/benchmarks_json/Python_ionization.json index 35aa2c07a60..31f426aa362 100644 --- a/Regression/Checksum/benchmarks_json/Python_ionization.json +++ b/Regression/Checksum/benchmarks_json/Python_ionization.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bx": 0.0, + "By": 26296568.434868, + "Bz": 0.0, + "Ex": 7878103122971888.0, + "Ey": 0.0, + "Ez": 3027.995293554466, + "jx": 1.2111358330750162e+16, + "jy": 0.0, + "jz": 1.3483401471475687e-07 + }, "electrons": { - "particle_momentum_x": 4.410992240916769e-18, + "particle_momentum_x": 4.4206237143449475e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.637306716013194e-18, - "particle_position_x": 0.1095060155214851, - "particle_position_y": 0.6411875726789248, - "particle_weight": 3.44453125e-10 + "particle_momentum_z": 2.6361297302081026e-18, + "particle_position_x": 0.11009154442846772, + "particle_position_y": 0.6414658436421568, + "particle_weight": 3.4450781249999996e-10 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.761324019342538e-18, + "particle_momentum_x": 1.76132401934254e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.644887006212893e-23, - "particle_position_x": 0.03199998810579664, - "particle_position_y": 0.12800000462171646, + "particle_momentum_z": 3.644887053263054e-23, + "particle_position_x": 0.03200001189420337, + "particle_position_y": 0.1280000046901387, "particle_weight": 9.999999999999999e-11 - }, - "lev=0": { - "Bx": 0.0, - "By": 26296568.43487075, - "Bz": 0.0, - "Ex": 7878103122973058.0, - "Ey": 0.0, - "Ez": 3027.995293554496, - "jx": 1.2111358333819302e+16, - "jy": 0.0, - "jz": 1.355517269126987e-07 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/RefinedInjection.json b/Regression/Checksum/benchmarks_json/RefinedInjection.json index 8be545f3887..43cc8e3072c 100644 --- a/Regression/Checksum/benchmarks_json/RefinedInjection.json +++ b/Regression/Checksum/benchmarks_json/RefinedInjection.json @@ -1,11 +1,27 @@ { - "beam": { - "particle_momentum_x": 4.5603842543853726e-20, - "particle_momentum_y": 4.191542263916011e-20, - "particle_momentum_z": 1.363947302075425e-17, - "particle_position_x": 4.672031050423284e-05, - "particle_position_y": 0.00024387147640533396, - "particle_weight": 12483018148921.525 + "lev=0": { + "Bx": 26338425.87401078, + "By": 160257.72293376247, + "Bz": 694883.1248007242, + "Ex": 56513906294889.77, + "Ey": 7806060318610865.0, + "Ez": 65816740142650.89, + "jx": 6873283794395257.0, + "jy": 4286847636945161.5, + "jz": 7506503405703726.0, + "rho": 278060758.9118884 + }, + "lev=1": { + "Bx": 41971351.93872842, + "By": 347834.73215718823, + "Bz": 1093501.9056767717, + "Ex": 136247073074384.92, + "Ey": 1.2435868091440666e+16, + "Ez": 92449081296414.34, + "jx": 1.6499002361434856e+16, + "jy": 1568934856430368.5, + "jz": 2.2190671920159212e+16, + "rho": 298322648.1883917 }, "electrons": { "particle_momentum_x": 4.599605609558194e-19, @@ -15,28 +31,12 @@ "particle_position_y": 0.3695766840020302, "particle_weight": 216500976562500.75 }, - "lev=0": { - "Bx": 26338374.3731115, - "By": 160257.72168459874, - "Bz": 694868.2285976049, - "Ex": 56513906233209.63, - "Ey": 7806074074131210.0, - "Ez": 65816739650512.98, - "jx": 6873283793354275.0, - "jy": 4287548168426209.0, - "jz": 7506499988368183.0, - "rho": 278060758.9118884 - }, - "lev=1": { - "Bx": 41971136.54330983, - "By": 347834.7278931723, - "Bz": 1093489.2683699469, - "Ex": 136247072743280.97, - "Ey": 1.2435931966910504e+16, - "Ez": 92449079327862.72, - "jx": 1.6499002357270928e+16, - "jy": 1571736982354559.0, - "jz": 2.219065825081704e+16, - "rho": 298322648.1883917 + "beam": { + "particle_momentum_x": 4.560382242970367e-20, + "particle_momentum_y": 4.2263074230751857e-20, + "particle_momentum_z": 1.3639472455787962e-17, + "particle_position_x": 4.672031008809431e-05, + "particle_position_y": 0.0002438714470005561, + "particle_weight": 12483018148921.525 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json index d62cd008670..c8571869cd5 100644 --- a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 6.875024052604206e-19, - "particle_momentum_y": 4.375444487966637e-19, - "particle_momentum_z": 6.432593176909762e-18, - "particle_position_x": 0.0012933598223665212, - "particle_position_y": 0.35872101659459354, - "particle_weight": 3120754537230.3823 - }, - "electrons": { - "particle_momentum_x": 7.058217306102507e-19, - "particle_momentum_y": 2.2042314720139012e-18, - "particle_momentum_z": 2.5305214299582585e-16, - "particle_position_x": 1.5006580327952221, - "particle_position_y": 16.454388306646475, - "particle_weight": 1.234867020725368e+18 + "lev=0": { + "Bx": 1118808.3734374465, + "By": 3248957.1122179274, + "Bz": 280612.78289064515, + "Ex": 975532732112639.6, + "Ey": 402861836732114.4, + "Ez": 159049610399317.66, + "jx": 2.9997053130250828e+16, + "jy": 8.866654890775573e+16, + "jz": 3.163974708948631e+17, + "rho": 1059977729.0184418 }, "ions": { - "particle_momentum_x": 1.6150569264030943e-18, - "particle_momentum_y": 2.2334239793537862e-18, - "particle_momentum_z": 4.279249530683602e-13, - "particle_position_x": 1.4883816864868449, - "particle_position_y": 16.452386504130835, + "particle_momentum_x": 1.6150569180478943e-18, + "particle_momentum_y": 2.2334266828401142e-18, + "particle_momentum_z": 4.2792495306800117e-13, + "particle_position_x": 1.4883816864865955, + "particle_position_y": 16.45238650413084, "particle_weight": 1.234867369440658e+18 }, - "lev=0": { - "Bx": 1118823.5061574595, - "By": 3248957.084212374, - "Bz": 280624.78102759377, - "Ex": 975532719061463.4, - "Ey": 402851207946673.1, - "Ez": 159049601047523.06, - "jx": 2.9997052529118484e+16, - "jy": 8.866616130905646e+16, - "jz": 3.163974567840701e+17, - "rho": 1059977703.1166165 + "electrons": { + "particle_momentum_x": 7.05821754019506e-19, + "particle_momentum_y": 2.2042393263043917e-18, + "particle_momentum_z": 2.5305214316289944e-16, + "particle_position_x": 1.5006580331362074, + "particle_position_y": 16.45438830674347, + "particle_weight": 1.234867020725368e+18 + }, + "beam": { + "particle_momentum_x": 6.874634694077579e-19, + "particle_momentum_y": 4.374677735660533e-19, + "particle_momentum_z": 6.432600800266472e-18, + "particle_position_x": 0.0012933700124436584, + "particle_position_y": 0.358720803656086, + "particle_weight": 3120754537230.3823 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json index cfd4bc83a85..03cf06b3158 100644 --- a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 7.005046981277183e-19, - "particle_momentum_y": 4.375209352729912e-19, - "particle_momentum_z": 6.175459223479959e-18, - "particle_position_x": 0.001602571749982411, - "particle_position_y": 0.35897924403061005, - "particle_weight": 3120754537230.3823 + "lev=0": { + "Bx": 1086729.9983020213, + "By": 2886537.292820136, + "Bz": 264259.5410465989, + "Ex": 867382749986595.4, + "Ey": 392666737316432.8, + "Ez": 146897959667920.22, + "jx": 2.7028854305928356e+16, + "jy": 8.615938520657634e+16, + "jz": 2.7328721771319978e+17, + "rho": 915931445.5917195 }, "electrons": { - "particle_momentum_x": 6.240989852249656e-19, - "particle_momentum_y": 1.5790301798509742e-18, - "particle_momentum_z": 2.5064352637575283e-16, - "particle_position_x": 1.501413662801212, - "particle_position_y": 16.523781706919216, + "particle_momentum_x": 6.2409905560361695e-19, + "particle_momentum_y": 1.5790611507526441e-18, + "particle_momentum_z": 2.506435264953701e-16, + "particle_position_x": 1.5014136629221837, + "particle_position_y": 16.52378170697116, "particle_weight": 1.2372401466086835e+18 }, "ions": { - "particle_momentum_x": 1.4394968631660078e-18, - "particle_momentum_y": 1.5967458174525868e-18, - "particle_momentum_z": 4.2873406586841774e-13, - "particle_position_x": 1.4911814217840753, - "particle_position_y": 16.52196497877435, + "particle_momentum_x": 1.4394967936107095e-18, + "particle_momentum_y": 1.5967629122067375e-18, + "particle_momentum_z": 4.287340658682592e-13, + "particle_position_x": 1.4911814217840271, + "particle_position_y": 16.521964978774346, "particle_weight": 1.2372405194129536e+18 }, - "lev=0": { - "Bx": 1086731.2285090145, - "By": 2886537.258822082, - "Bz": 264280.80501631316, - "Ex": 867382744066315.0, - "Ey": 392669698675093.25, - "Ez": 146897949570681.72, - "jx": 2.7028852724044516e+16, - "jy": 8.615686606708202e+16, - "jz": 2.7328720244417827e+17, - "rho": 915931431.8889401 + "beam": { + "particle_momentum_x": 7.0050974522936195e-19, + "particle_momentum_y": 4.374915691594075e-19, + "particle_momentum_z": 6.1754662837862646e-18, + "particle_position_x": 0.0016025830388130494, + "particle_position_y": 0.35897909980539305, + "particle_weight": 3120754537230.3823 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ionization_boost.json b/Regression/Checksum/benchmarks_json/ionization_boost.json index cb02eeba5b0..35b2db84a1a 100644 --- a/Regression/Checksum/benchmarks_json/ionization_boost.json +++ b/Regression/Checksum/benchmarks_json/ionization_boost.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bx": 0.0, + "By": 18263123.342891, + "Bz": 0.0, + "Ex": 5472992180428804.0, + "Ey": 0.0, + "Ez": 922.5589707939612, + "jx": 12440856508004.96, + "jy": 0.0, + "jz": 78616.0000011086 + }, "electrons": { - "particle_momentum_x": 2.137334032699983e-17, + "particle_momentum_x": 2.1386770170623736e-17, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.729179869336611e-17, - "particle_position_x": 0.11036860836114047, - "particle_position_y": 1.7121959960266107, - "particle_weight": 3.075501431906104e-09 + "particle_momentum_z": 1.7287241262743654e-17, + "particle_position_x": 0.11064981928849067, + "particle_position_y": 1.7121826057017473, + "particle_weight": 3.0755014319061045e-09 }, "ions": { "particle_ionizationLevel": 52741.0, - "particle_momentum_x": 3.63061972838759e-18, + "particle_momentum_x": 3.630619728387593e-18, "particle_momentum_y": 0.0, "particle_momentum_z": 1.0432995297946715e-13, - "particle_position_x": 0.021440031727258925, + "particle_position_x": 0.021439968272741083, "particle_position_y": 0.4742770090248555, "particle_weight": 5.000948082142308e-10 - }, - "lev=0": { - "Bx": 0.0, - "By": 18263123.342890996, - "Bz": 0.0, - "Ex": 5472992180428804.0, - "Ey": 0.0, - "Ez": 922.6036749671132, - "jx": 12440856508004.96, - "jy": 0.0, - "jz": 78616.00000110848 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ionization_lab.json b/Regression/Checksum/benchmarks_json/ionization_lab.json index f1be9ec3017..82984141b59 100644 --- a/Regression/Checksum/benchmarks_json/ionization_lab.json +++ b/Regression/Checksum/benchmarks_json/ionization_lab.json @@ -1,31 +1,31 @@ { - "electrons": { - "particle_momentum_x": 4.407898469197755e-18, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.642991174223682e-18, - "particle_orig_z": 0.43016526372226926, - "particle_position_x": 0.1095015206652257, - "particle_position_y": 0.6413864600981052, - "particle_weight": 3.443203125e-10 + "lev=0": { + "Bx": 0.0, + "By": 26296568.434868, + "Bz": 0.0, + "Ex": 7878103122971888.0, + "Ey": 0.0, + "Ez": 3027.995293554467, + "jx": 1.2111358330750162e+16, + "jy": 0.0, + "jz": 1.3575651149270143e-07 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.761324005206226e-18, + "particle_momentum_x": 1.7613240052056494e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.618696690270335e-23, - "particle_position_x": 0.03199998819289686, + "particle_momentum_z": 3.6186967375178554e-23, + "particle_position_x": 0.0320000118071032, "particle_position_y": 0.12800000462171646, "particle_weight": 9.999999999999999e-11 }, - "lev=0": { - "Bx": 0.0, - "By": 26296568.43487075, - "Bz": 0.0, - "Ex": 7878103122973058.0, - "Ey": 0.0, - "Ez": 3027.995293554496, - "jx": 1.2111358333819302e+16, - "jy": 0.0, - "jz": 1.346772722412443e-07 + "electrons": { + "particle_momentum_x": 4.428135211584547e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.6627518442038486e-18, + "particle_orig_z": 0.43043207622230534, + "particle_position_x": 0.11023346490802505, + "particle_position_y": 0.6417814148540429, + "particle_weight": 3.44453125e-10 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart.json b/Regression/Checksum/benchmarks_json/restart.json index 9ed02d0faad..d8aa0788997 100644 --- a/Regression/Checksum/benchmarks_json/restart.json +++ b/Regression/Checksum/benchmarks_json/restart.json @@ -1,12 +1,15 @@ { - "beam": { - "particle_momentum_x": 4.178482505909375e-19, - "particle_momentum_y": 4.56492260137707e-19, - "particle_momentum_z": 2.733972888170628e-17, - "particle_position_x": 0.0003995213395426269, - "particle_position_y": 0.0004148795632360405, - "particle_position_z": 1.9019426942919677, - "particle_weight": 3120754537.230381 + "lev=0": { + "Bx": 115361.74185283793, + "By": 242516.32397638055, + "Bz": 62078.1602361236, + "Ex": 119701543932824.69, + "Ey": 20420953176049.12, + "Ez": 53561498853753.336, + "jx": 2.1550943713704252e+16, + "jy": 328452566832977.2, + "jz": 4525238578330174.0, + "rho": 22112877.392750103 }, "driver": { "particle_momentum_x": 4.700436405078562e+21, @@ -26,34 +29,31 @@ "particle_position_z": 0.4899808854005956, "particle_weight": 6241509074.460762 }, - "lev=0": { - "Bx": 115361.85363382133, - "By": 242516.41036288123, - "Bz": 62078.12299476949, - "Ex": 119701563770527.33, - "Ey": 20420746160110.2, - "Ez": 53561518084714.88, - "jx": 2.155094272131813e+16, - "jy": 328438528912277.4, - "jz": 4525204093100645.0, - "rho": 22113102.212642767 + "beam": { + "particle_momentum_x": 4.178482505909375e-19, + "particle_momentum_y": 4.56492260137707e-19, + "particle_momentum_z": 2.733972888170628e-17, + "particle_position_x": 0.0003995213395426269, + "particle_position_y": 0.0004148795632360405, + "particle_position_z": 1.9019426942919677, + "particle_weight": 3120754537.230381 }, - "plasma_e": { - "particle_momentum_x": 2.6421618201433585e-19, - "particle_momentum_y": 1.3141433018232061e-20, - "particle_momentum_z": 2.6326692402728787e-17, - "particle_position_x": 0.49916042906025937, - "particle_position_y": 0.4991834641855549, - "particle_position_z": 0.45626372602154797, + "plasma_p": { + "particle_momentum_x": 2.6392309174575904e-19, + "particle_momentum_y": 5.301543151656233e-21, + "particle_momentum_z": 4.829600619848831e-14, + "particle_position_x": 0.4991250033821341, + "particle_position_y": 0.49912499879083855, + "particle_position_z": 0.4563845472726038, "particle_weight": 33067341227104.594 }, - "plasma_p": { - "particle_momentum_x": 2.639231047445633e-19, - "particle_momentum_y": 5.302522423284582e-21, - "particle_momentum_z": 4.829600619851657e-14, - "particle_position_x": 0.4991250033821394, - "particle_position_y": 0.49912499879083844, - "particle_position_z": 0.45638454727260425, + "plasma_e": { + "particle_momentum_x": 2.6421607111722265e-19, + "particle_momentum_y": 1.3141424232991868e-20, + "particle_momentum_z": 2.6326692443085376e-17, + "particle_position_x": 0.49916042919135184, + "particle_position_y": 0.49918346422905135, + "particle_position_z": 0.4562637258155577, "particle_weight": 33067341227104.594 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd.json b/Regression/Checksum/benchmarks_json/restart_psatd.json index 81df3c64c4c..d22fb5b57e7 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd.json @@ -1,4 +1,16 @@ { + "lev=0": { + "Bx": 116102.13010406512, + "By": 245467.5412026496, + "Bz": 65305.35287114741, + "Ex": 118565481099326.73, + "Ey": 18640100573642.96, + "Ez": 51777969821120.38, + "jx": 2.1177246490642464e+16, + "jy": 351544815339900.3, + "jz": 4573466562652713.0, + "rho": 22294533.587530266 + }, "beam": { "particle_momentum_x": 4.178482505909375e-19, "particle_momentum_y": 4.56492260137707e-19, @@ -26,34 +38,22 @@ "particle_position_z": 0.4899808854005956, "particle_weight": 6241509074.460762 }, - "lev=0": { - "Bx": 116102.28415732287, - "By": 245468.57438100342, - "Bz": 65305.38781838063, - "Ex": 118565257880521.4, - "Ey": 18640027127418.004, - "Ez": 51777964618631.914, - "jx": 2.1177240176169996e+16, - "jy": 351529204246424.9, - "jz": 4573419547907262.0, - "rho": 22294773.912022192 - }, "plasma_e": { - "particle_momentum_x": 2.0912370354448323e-19, - "particle_momentum_y": 1.3510736736168866e-20, - "particle_momentum_z": 2.633607024821822e-17, - "particle_position_x": 0.4991733347875822, - "particle_position_y": 0.49919828067421096, - "particle_position_z": 0.4562416755755165, + "particle_momentum_x": 2.0912364079892767e-19, + "particle_momentum_y": 1.3510804924903876e-20, + "particle_momentum_z": 2.6336070297356917e-17, + "particle_position_x": 0.49917333523960167, + "particle_position_y": 0.49919828074154127, + "particle_position_z": 0.45624167532314724, "particle_weight": 33067341227104.625 }, "plasma_p": { - "particle_momentum_x": 2.105303160755537e-19, - "particle_momentum_y": 3.3281768098792752e-21, - "particle_momentum_z": 4.829599953650214e-14, - "particle_position_x": 0.4991250029759827, - "particle_position_y": 0.49912499825258744, - "particle_position_z": 0.4563845470979585, + "particle_momentum_x": 2.1053025446547977e-19, + "particle_momentum_y": 3.3272838305044046e-21, + "particle_momentum_z": 4.829599953646669e-14, + "particle_position_x": 0.49912500297599893, + "particle_position_y": 0.49912499825258866, + "particle_position_z": 0.45638454709795806, "particle_weight": 33067341227104.625 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json index 344495f8144..50971f0d45e 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json @@ -1,4 +1,25 @@ { + "lev=0": { + "Bx": 115622.1878941125, + "By": 243403.66715242947, + "Bz": 63602.66529020351, + "Ex": 117118934931289.56, + "Ey": 18448377440588.27, + "Ez": 50821967818817.24, + "jx": 2.1044522674691452e+16, + "jy": 329314843111847.94, + "jz": 4524787623275627.0, + "rho": 22128883.53068064 + }, + "driverback": { + "particle_momentum_x": 4.813131349021332e+21, + "particle_momentum_y": 5.16548074090123e+21, + "particle_momentum_z": 3.005830430844926e+25, + "particle_position_x": 0.001649481123084974, + "particle_position_y": 0.0016172218745428432, + "particle_position_z": 0.4899808854005956, + "particle_weight": 6241509074.460762 + }, "beam": { "particle_momentum_x": 4.178482505909375e-19, "particle_momentum_y": 4.56492260137707e-19, @@ -8,6 +29,24 @@ "particle_position_z": 1.901942694291968, "particle_weight": 3120754537.230381 }, + "plasma_e": { + "particle_momentum_x": 1.9905288525315664e-19, + "particle_momentum_y": 1.2685401564810352e-20, + "particle_momentum_z": 2.6334746597885943e-17, + "particle_position_x": 0.4991702885212871, + "particle_position_y": 0.4991949978513417, + "particle_position_z": 0.4562491611716817, + "particle_weight": 33067341227104.625 + }, + "plasma_p": { + "particle_momentum_x": 2.002897995377179e-19, + "particle_momentum_y": 2.87423099731012e-21, + "particle_momentum_z": 4.8296000849128737e-14, + "particle_position_x": 0.49912500259846493, + "particle_position_y": 0.4991249979476612, + "particle_position_z": 0.45638454712861487, + "particle_weight": 33067341227104.625 + }, "driver": { "particle_momentum_x": 4.700436405078562e+21, "particle_momentum_y": 4.6785862113093076e+21, @@ -16,44 +55,5 @@ "particle_position_y": 0.0016212373149699414, "particle_position_z": 0.30417779498653386, "particle_weight": 6241509074.460762 - }, - "driverback": { - "particle_momentum_x": 4.813131349021332e+21, - "particle_momentum_y": 5.16548074090123e+21, - "particle_momentum_z": 3.005830430844926e+25, - "particle_position_x": 0.001649481123084974, - "particle_position_y": 0.0016172218745428432, - "particle_position_z": 0.4899808854005956, - "particle_weight": 6241509074.460762 - }, - "lev=0": { - "Bx": 115622.38124168929, - "By": 243405.25199746038, - "Bz": 63602.713209078014, - "Ex": 117118723119070.72, - "Ey": 18448325716024.688, - "Ez": 50821980097172.29, - "jx": 2.104451599585796e+16, - "jy": 329301545706958.4, - "jz": 4524740919108259.0, - "rho": 22129124.901803844 - }, - "plasma_e": { - "particle_momentum_x": 1.9905296213155052e-19, - "particle_momentum_y": 1.2685326003720766e-20, - "particle_momentum_z": 2.6334746549552728e-17, - "particle_position_x": 0.4991702881158205, - "particle_position_y": 0.4991949977965001, - "particle_position_z": 0.45624916139143146, - "particle_weight": 33067341227104.625 - }, - "plasma_p": { - "particle_momentum_x": 2.0028987523417718e-19, - "particle_momentum_y": 2.875037979316222e-21, - "particle_momentum_z": 4.8296000849162443e-14, - "particle_position_x": 0.49912500259844994, - "particle_position_y": 0.4991249979476608, - "particle_position_z": 0.4563845471286154, - "particle_weight": 33067341227104.625 } } \ No newline at end of file diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index 88da5b0afad..fd4e9397253 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -870,7 +870,7 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, np, [=] AMREX_GPU_DEVICE (int i) { // Calculate the velocity according to the amplitude of E - const Real sign_charge = (pwp[i]>0) ? 1 : -1; + const Real sign_charge = (pwp[i]>0) ? -1 : 1; const Real v_over_c = sign_charge * tmp_mobility * amplitude[i]; AMREX_ALWAYS_ASSERT_WITH_MESSAGE(amrex::Math::abs(v_over_c) < amrex::Real(1.), "Error: calculated laser particle velocity greater than c." From 2444f15216c49d4fa6684caad2286c21d6a94433 Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 2 Jan 2024 10:49:41 -0800 Subject: [PATCH 085/176] Add semi-implicit CI test (#4565) * Add semi-implicit CI test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update CI benchmark values --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Examples/Tests/Implicit/analysis_1d.py | 7 +- .../Tests/Implicit/inputs_1d_semiimplicit | 81 +++++++++++++++++++ .../SemiImplicitPicard_1d.json | 29 +++++++ Regression/WarpX-tests.ini | 17 ++++ 4 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 Examples/Tests/Implicit/inputs_1d_semiimplicit create mode 100644 Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json diff --git a/Examples/Tests/Implicit/analysis_1d.py b/Examples/Tests/Implicit/analysis_1d.py index 3d4fc375bc9..0f00010a505 100755 --- a/Examples/Tests/Implicit/analysis_1d.py +++ b/Examples/Tests/Implicit/analysis_1d.py @@ -10,6 +10,7 @@ # This is a script that analyses the simulation results from # the script `inputs_1d`. This simulates a 1D periodic plasma using the implicit solver. import os +import re import sys import numpy as np @@ -28,7 +29,11 @@ delta_E = (total_energy - total_energy[0])/total_energy[0] max_delta_E = np.abs(delta_E).max() -tolerance_rel = 1.e-14 +if re.match('SemiImplicitPicard_1d', fn): + tolerance_rel = 2.5e-5 +elif re.match('ImplicitPicard_1d', fn): + # This case should have near machine precision conservation of energy + tolerance_rel = 1.e-14 print(f"max change in energy: {max_delta_E}") print(f"tolerance: {tolerance_rel}") diff --git a/Examples/Tests/Implicit/inputs_1d_semiimplicit b/Examples/Tests/Implicit/inputs_1d_semiimplicit new file mode 100644 index 00000000000..2271a0bb1bc --- /dev/null +++ b/Examples/Tests/Implicit/inputs_1d_semiimplicit @@ -0,0 +1,81 @@ +################################# +############ CONSTANTS ############# +################################# + +my_constants.n0 = 1.e30 # plasma densirty, m^-3 +my_constants.nz = 40 # number of grid cells +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.nppcz = 100 # number of particles/cell in z +my_constants.dt = 0.1/wpe # time step size, s + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_step = 100 +amr.n_cell = nz +amr.max_level = 0 + +geometry.dims = 1 +geometry.prob_lo = 0.0 +geometry.prob_hi = 10.*de0 +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic + +################################# +############ NUMERICS ########### +################################# + +warpx.const_dt = dt +algo.evolve_scheme = semi_implicit_picard +algo.max_picard_iterations = 5 +algo.picard_iteration_tolerance = 0. +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ PLASMA ############# +################################# + +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 100 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = w ux uy uz +diag1.protons.variables = w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json new file mode 100644 index 00000000000..2c9859b037d --- /dev/null +++ b/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 3559.0541122456157, + "By": 1685.942868827529, + "Bz": 0.0, + "Ex": 796541204346.5195, + "Ey": 961740397927.6577, + "Ez": 3528140764527.8877, + "divE": 2.0829159085076083e+21, + "jx": 7.683674095607745e+17, + "jy": 1.4132459141738875e+18, + "jz": 1.350650739310074e+18, + "rho": 18442528652.19583 + }, + "protons": { + "particle_momentum_x": 5.231109104020749e-19, + "particle_momentum_y": 5.367985047933474e-19, + "particle_momentum_z": 5.253213505843665e-19, + "particle_position_x": 0.00010628272743703473, + "particle_weight": 5.314093261582036e+22 + }, + "electrons": { + "particle_momentum_x": 1.196357181461066e-20, + "particle_momentum_y": 1.2271903040162696e-20, + "particle_momentum_z": 1.2277743615209627e-20, + "particle_position_x": 0.0001064956905491333, + "particle_weight": 5.314093261582036e+22 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index da05ed4d5bc..7273140a369 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4479,3 +4479,20 @@ compileTest = 0 doVis = 0 compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py + +[SemiImplicitPicard_1d] +buildDir = . +inputFile = Examples/Tests/Implicit/inputs_1d_semiimplicit +runtime_params = warpx.abort_on_warning_threshold=high +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_1d.py From 8d15065f6661ae347a172d5b37e2c4913055ab44 Mon Sep 17 00:00:00 2001 From: Kale Weichman <46941919+kale-j@users.noreply.github.com> Date: Wed, 3 Jan 2024 11:58:22 -0500 Subject: [PATCH 086/176] Add filters for boundary scraping diagnostics (#4371) * Edited openPMD diagnostic routines to use particle filters with boundary scraping diagnostics The modification separates the cases for backtransformed diagnostics and boundary scraping diagnostics in WriteOpenPMDParticles, and implements filters for the latter (folowing the procedure used for fullDiagnostics) This allows the filters available for fullDiagnostics (random_filter, uniform_filter, parser_filter, and geometry_filter) to be used with boundary scraping diagnostics Some notes: 1. All particles are still flushed from the boundary scraping particle buffer after the diagnostic file is written, regardless of whether all particles were written 2. The filter functions are called at the time the diagnostic is written, not the time the particle crosses the boundary This has a couple of implications: i. The boundary scraping diagnostic should be written frequently to flush the buffer if a lot of particles cross the scraped boundary, even if only a small subset of those particles are written to file ii. "t" in plot_filter_function is the time the diagnostic file is written rather than the time the particle crossed the boundary * Fixed spacing * Added regression test for filtering boundary scraping diagnostics Added a new regression test scraping_filter, based on scraping This tests whether boundary scraping diagnostics are set up to use filtering In this test, plot_filter_function is used to indicateas whether filters are implemented -- if one wants to test the other filtering options (e.g., random_fraction), more tests will be needed in the future The regression test scraping was copied and plot_filter_function is used to write only the particles with z>0 Because this represents half the total particles, the check on the total number of particles now uses twice the number of particles scraped * Updated scraping_filter test Added check that all scraped particles have z > 0 as specified in inputs Removed checksum test that was redundant with scraping test * Fix indentation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Remi Lehe Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/usage/parameters.rst | 2 + Examples/Tests/scraping/analysis_rz_filter.py | 63 +++++++++++++++++++ Examples/Tests/scraping/inputs_rz_filter | 58 +++++++++++++++++ Regression/WarpX-tests.ini | 17 +++++ Source/Diagnostics/WarpXOpenPMD.cpp | 32 +++++++--- 5 files changed, 163 insertions(+), 9 deletions(-) create mode 100755 Examples/Tests/scraping/analysis_rz_filter.py create mode 100644 Examples/Tests/scraping/inputs_rz_filter diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 88d8b4dbe6f..99758d52848 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2737,6 +2737,8 @@ This can be important if a large number of particles are lost, avoiding filling In addition to their usual attributes, the saved particles have an integer attribute ``timestamp``, which indicates the PIC iteration at which each particle was absorbed at the boundary. +``BoundaryScrapingDiagnostics`` can be used with ``..random_fraction``, ``..uniform_stride``, and ``..plot_filter_function``, which have the same behavior as for ``FullDiagnostics``. For ``BoundaryScrapingDiagnostics``, these filters are applied at the time the data is written to file. An implication of this is that more particles may initially be accumulated in memory than are ultimately written. ``t`` in ``plot_filter_function`` refers to the time the diagnostic is written rather than the time the particle crossed the boundary. + .. _running-cpp-parameters-diagnostics-reduced: Reduced Diagnostics diff --git a/Examples/Tests/scraping/analysis_rz_filter.py b/Examples/Tests/scraping/analysis_rz_filter.py new file mode 100755 index 00000000000..b97b6e0eb5a --- /dev/null +++ b/Examples/Tests/scraping/analysis_rz_filter.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# Copyright 2023 Kale Weichman +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests the particle scraping for the embedded boundary in RZ. +# Particles are initialized between r=0.15 and r=0.2 +# having a negative radial velocity. +# A cylindrical embedded surface is placed at r=0.1. +# Upon reaching the surface, particles should be removed. +# At the end of the simulation, i.e., at time step 37, +# there should be 512 particles left. +# This test checks that plot_filter_fucntion works with the boundary scraping diagnostic +# by making sure that the particles removed from only half the domain (z>0) have been recorded. + +# Possible errors: 0 +# tolerance: 0 +# Possible running time: < 1 s + +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +import yt + +tolerance = 0 + +fn = sys.argv[1] +ds = yt.load( fn ) +ad = ds.all_data() +x = ad['electron', 'particle_position_x'].v + +error = len(x)-512 +print('error = ', error) +print('tolerance = ', tolerance) +assert(error==tolerance) + +# Check that all the removed particles are properly recorded +# by making sure that, at each iteration, the sum of the number of +# remaining particles and scraped particles is equal to half the +# original number of particles +# also check that no particles with z <= 0 have been scraped +ts_full = OpenPMDTimeSeries('./diags/diag2/') +ts_scraping = OpenPMDTimeSeries('./diags/diag3/particles_at_eb') + +def n_remaining_particles( iteration ): + w, = ts_full.get_particle(['w'], iteration=iteration) + return len(w) +def n_scraped_particles( iteration ): + timestamp = ts_scraping.get_particle( ['timestamp'], iteration=ts_scraping.iterations[0] ) + return (timestamp <= iteration).sum() +def n_scraped_z_leq_zero( iteration ): + z_pos, = ts_scraping.get_particle( ['z'], iteration=ts_scraping.iterations[0] ) + return (z_pos <= 0).sum() +n_remaining = np.array([ n_remaining_particles(iteration) for iteration in ts_full.iterations ]) +n_scraped = np.array([ n_scraped_particles(iteration) for iteration in ts_full.iterations ]) +n_z_leq_zero = np.array([ n_scraped_z_leq_zero(iteration) for iteration in ts_full.iterations ]) +n_total = n_remaining[0] +assert np.all( 2*n_scraped+n_remaining == n_total) +assert np.all( n_z_leq_zero == 0) diff --git a/Examples/Tests/scraping/inputs_rz_filter b/Examples/Tests/scraping/inputs_rz_filter new file mode 100644 index 00000000000..0d67fb96f6c --- /dev/null +++ b/Examples/Tests/scraping/inputs_rz_filter @@ -0,0 +1,58 @@ +amr.max_level = 1 +warpx.fine_tag_lo = 0.0 -0.5 +warpx.fine_tag_hi = 0.25 0.0 + +max_step = 37 + +amr.n_cell = 64 64 +amr.blocking_factor = 8 +amr.max_grid_size = 128 + +geometry.dims = RZ +geometry.prob_lo = 0.0 -0.5 +geometry.prob_hi = 0.5 0.5 + +boundary.field_lo = none periodic +boundary.field_hi = pec periodic +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +warpx.const_dt = 1.216119097e-11 +warpx.eb_implicit_function = "-(x**2-0.1**2)" + +# Do not evolve the E and B fields +algo.maxwell_solver = none +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 4 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "(x*x+y*y>0.15*0.15)*(x*x+y*y<0.2*0.2)*1.0e21" +electron.momentum_distribution_type = parse_momentum_function +electron.momentum_function_ux(x,y,z) = "if(x*x+y*y>0.0, -1.0*x/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uy(x,y,z) = "if(x*x+y*y>0.0, -1.0*y/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uz(x,y,z) = "0" +electron.save_particles_at_eb = 1 + +diagnostics.diags_names = diag1 diag2 diag3 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Er + +diag2.intervals = 1 +diag2.diag_type = Full +diag2.fields_to_plot = Er +diag2.format = openpmd + +diag3.diag_type = BoundaryScraping +diag3.format = openpmd +diag3.electron.plot_filter_function(t,x,y,z,ux,uy,uz) = "z > 0" diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 7273140a369..610c5befa67 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4240,6 +4240,23 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/scraping/analysis_rz.py +[scraping_filter] +buildDir = . +inputFile = Examples/Tests/scraping/inputs_rz_filter +runtime_params = warpx.abort_on_warning_threshold = medium +dim = 2 +addToCompileString = USE_EB=TRUE USE_RZ=TRUE USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/scraping/analysis_rz_filter.py + [silver_mueller_1d] buildDir = . inputFile = Examples/Tests/silver_mueller/inputs_1d diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 5100291a94a..891ba5a3189 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -588,21 +588,35 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { GeometryFilter const geometry_filter(particle_diags[i].m_do_geom_filter, particle_diags[i].m_diag_domain); - if (isBTD || use_pinned_pc) { + if (isBTD) { tmp.copyParticles(*pinned_pc, true); particlesConvertUnits(ConvertDirection::WarpX_to_SI, &tmp, mass); + } else if (use_pinned_pc) { + // pinned pc, but not BTD (i.e., boundary scraping diagnostic) + particlesConvertUnits(ConvertDirection::WarpX_to_SI, pinned_pc, mass); + using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; + tmp.copyParticles(*pinned_pc, + [random_filter,uniform_filter,parser_filter,geometry_filter] + AMREX_GPU_HOST_DEVICE + (const SrcData& src, int ip, const amrex::RandomEngine& engine) + { + const SuperParticleType& p = src.getSuperParticle(ip); + return random_filter(p, engine) * uniform_filter(p, engine) + * parser_filter(p, engine) * geometry_filter(p, engine); + }, true); + particlesConvertUnits(ConvertDirection::SI_to_WarpX, pinned_pc, mass); } else { particlesConvertUnits(ConvertDirection::WarpX_to_SI, pc, mass); using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; tmp.copyParticles(*pc, - [random_filter,uniform_filter,parser_filter,geometry_filter] - AMREX_GPU_HOST_DEVICE - (const SrcData& src, int ip, const amrex::RandomEngine& engine) - { - const SuperParticleType& p = src.getSuperParticle(ip); - return random_filter(p, engine) * uniform_filter(p, engine) - * parser_filter(p, engine) * geometry_filter(p, engine); - }, true); + [random_filter,uniform_filter,parser_filter,geometry_filter] + AMREX_GPU_HOST_DEVICE + (const SrcData& src, int ip, const amrex::RandomEngine& engine) + { + const SuperParticleType& p = src.getSuperParticle(ip); + return random_filter(p, engine) * uniform_filter(p, engine) + * parser_filter(p, engine) * geometry_filter(p, engine); + }, true); particlesConvertUnits(ConvertDirection::SI_to_WarpX, pc, mass); } From 4f894c6f1cf1d7870389678a8ccbba02f29e860f Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 3 Jan 2024 10:09:52 -0800 Subject: [PATCH 087/176] Update the field probe diagnostic documentation (#4564) --- Docs/source/usage/parameters.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 99758d52848..81580f5c342 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2870,8 +2870,9 @@ Reduced Diagnostics defaulting to ``1``. In RZ geometry, this only saves the 0'th azimuthal mode component of the fields. - Integrated electric and magnetic field components can instead be obtained by specifying + Time integrated electric and magnetic field components can instead be obtained by specifying ``.integrate = true``. + The integration is done every time step even when the data is written out less often. In a *moving window* simulation, the FieldProbe can be set to follow the moving frame by specifying ``.do_moving_window_FP = 1`` (default 0). .. warning:: From ea8cadfb8f83a1474be96bc547e846475e4cd1ff Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 3 Jan 2024 12:45:41 -0800 Subject: [PATCH 088/176] Apply particle selection to BTDiagnostics (#4570) * Edited openPMD diagnostic routines to use particle filters with boundary scraping diagnostics The modification separates the cases for backtransformed diagnostics and boundary scraping diagnostics in WriteOpenPMDParticles, and implements filters for the latter (folowing the procedure used for fullDiagnostics) This allows the filters available for fullDiagnostics (random_filter, uniform_filter, parser_filter, and geometry_filter) to be used with boundary scraping diagnostics Some notes: 1. All particles are still flushed from the boundary scraping particle buffer after the diagnostic file is written, regardless of whether all particles were written 2. The filter functions are called at the time the diagnostic is written, not the time the particle crosses the boundary This has a couple of implications: i. The boundary scraping diagnostic should be written frequently to flush the buffer if a lot of particles cross the scraped boundary, even if only a small subset of those particles are written to file ii. "t" in plot_filter_function is the time the diagnostic file is written rather than the time the particle crossed the boundary * Fixed spacing * Added regression test for filtering boundary scraping diagnostics Added a new regression test scraping_filter, based on scraping This tests whether boundary scraping diagnostics are set up to use filtering In this test, plot_filter_function is used to indicateas whether filters are implemented -- if one wants to test the other filtering options (e.g., random_fraction), more tests will be needed in the future The regression test scraping was copied and plot_filter_function is used to write only the particles with z>0 Because this represents half the total particles, the check on the total number of particles now uses twice the number of particles scraped * Updated scraping_filter test Added check that all scraped particles have z > 0 as specified in inputs Removed checksum test that was redundant with scraping test * Fix indentation * Apply particle selection to BTDiagnostics * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add automated check * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Kale Weichman Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Examples/Tests/boosted_diags/analysis.py | 7 ++++++- Examples/Tests/boosted_diags/inputs_3d | 1 + Source/Diagnostics/WarpXOpenPMD.cpp | 6 +----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Examples/Tests/boosted_diags/analysis.py b/Examples/Tests/boosted_diags/analysis.py index c6c089f9807..c0b03f4a20b 100755 --- a/Examples/Tests/boosted_diags/analysis.py +++ b/Examples/Tests/boosted_diags/analysis.py @@ -21,6 +21,7 @@ import numpy as np import openpmd_api as io +from openpmd_viewer import OpenPMDTimeSeries import yt yt.funcs.mylog.setLevel(0) @@ -48,9 +49,13 @@ Ez_openpmd = ds_openpmd.meshes['E']['z'].load_chunk() Ez_openpmd = Ez_openpmd.transpose() series.flush() - # Compare arrays to check consistency between new BTD formats (plotfile and openPMD) assert(np.allclose(Ez_plotfile, Ez_openpmd, rtol=rtol, atol=atol)) +# Check that particle random sub-selection has been applied +ts = OpenPMDTimeSeries('./diags/diag2/') +w, = ts.get_particle(['w'], species='beam', iteration=3) +assert (400 < len(w)) & (len(w) < 600) + test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/boosted_diags/inputs_3d b/Examples/Tests/boosted_diags/inputs_3d index ba98558be47..1b6b3448f26 100644 --- a/Examples/Tests/boosted_diags/inputs_3d +++ b/Examples/Tests/boosted_diags/inputs_3d @@ -122,3 +122,4 @@ diag2.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diag2.format = openpmd diag2.buffer_size = 32 diag2.openpmd_backend = h5 +diag2.beam.random_fraction = 0.5 diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 891ba5a3189..71d96a47927 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -588,11 +588,7 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { GeometryFilter const geometry_filter(particle_diags[i].m_do_geom_filter, particle_diags[i].m_diag_domain); - if (isBTD) { - tmp.copyParticles(*pinned_pc, true); - particlesConvertUnits(ConvertDirection::WarpX_to_SI, &tmp, mass); - } else if (use_pinned_pc) { - // pinned pc, but not BTD (i.e., boundary scraping diagnostic) + if (isBTD || use_pinned_pc) { particlesConvertUnits(ConvertDirection::WarpX_to_SI, pinned_pc, mass); using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; tmp.copyParticles(*pinned_pc, From 54486e7146ede599c4fec2eedc392470b1acc3a4 Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 3 Jan 2024 17:40:10 -0800 Subject: [PATCH 089/176] Update dsmc and fusion binary collisions (#4572) * Set random seed in capacitive_discharge/PICMI_inputs_1d.py * In collision, use expm1 to replace 1-exp --- .../capacitive_discharge/PICMI_inputs_1d.py | 9 ++++++--- .../BinaryCollision/DSMC/CollisionFilterFunc.H | 14 ++------------ .../NuclearFusion/SingleNuclearFusionEvent.H | 14 ++------------ 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index dd970cc15ea..38b83e46948 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -187,6 +187,9 @@ def __init__(self, n=0, test=False, pythonsolver=False, dsmc=False): else: self.mcc_subcycling_steps = None + if self.dsmc: + self.rng = np.random.default_rng(23094290) + self.ion_density_array = np.zeros(self.nz + 1) self.setup_run() @@ -397,9 +400,9 @@ def rethermalize_neutrals(self): vel_std = np.sqrt(constants.kb * self.gas_temp / self.m_ion) for ii in range(len(ux_arrays)): nps = len(ux_arrays[ii]) - ux_arrays[ii][:] = vel_std * np.random.normal(size=nps) - uy_arrays[ii][:] = vel_std * np.random.normal(size=nps) - uz_arrays[ii][:] = vel_std * np.random.normal(size=nps) + ux_arrays[ii][:] = vel_std * self.rng.normal(size=nps) + uy_arrays[ii][:] = vel_std * self.rng.normal(size=nps) + uz_arrays[ii][:] = vel_std * self.rng.normal(size=nps) def _get_rho_ions(self): # deposit the ion density in rho_fp diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H index a847b841ee2..c714cfdb133 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H @@ -77,18 +77,8 @@ void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleR // In principle this is obtained by computing 1 - exp(-probability_estimate) // However, the computation of this quantity can fail numerically when probability_estimate is // too small (e.g. exp(-probability_estimate) returns 1 and the computation returns 0). - // In this case, we simply use "probability_estimate" instead of 1 - exp(-probability_estimate) - // The threshold exp_threshold at which we switch between the two formulas is determined by the - // fact that computing the exponential is only useful if it can resolve the x^2/2 term of its - // Taylor expansion, i.e. the square of probability_estimate should be greater than the - // machine epsilon. -#ifdef AMREX_SINGLE_PRECISION_PARTICLES - constexpr auto exp_threshold = amrex::ParticleReal(1.e-3); -#else - constexpr auto exp_threshold = amrex::ParticleReal(5.e-8); -#endif - const amrex::ParticleReal probability = (exponent < exp_threshold) ? - exponent: 1._prt - std::exp(-exponent); + // std::expm1 is used since it maintains correctness for small exponent. + const amrex::ParticleReal probability = -std::expm1(-exponent); // Now we determine if a collision should occur if (amrex::Random(engine) < probability) diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H index bd0c6b06d6f..07e0174438b 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H @@ -113,18 +113,8 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part // In principle this is obtained by computing 1 - exp(-probability_estimate) // However, the computation of this quantity can fail numerically when probability_estimate is // too small (e.g. exp(-probability_estimate) returns 1 and the computation returns 0). - // In this case, we simply use "probability_estimate" instead of 1 - exp(-probability_estimate) - // The threshold exp_threshold at which we switch between the two formulas is determined by the - // fact that computing the exponential is only useful if it can resolve the x^2/2 term of its - // Taylor expansion, i.e. the square of probability_estimate should be greater than the - // machine epsilon. -#ifdef AMREX_SINGLE_PRECISION_PARTICLES - constexpr auto exp_threshold = amrex::ParticleReal(1.e-3); -#else - constexpr auto exp_threshold = amrex::ParticleReal(5.e-8); -#endif - const amrex::ParticleReal probability = (probability_estimate < exp_threshold) ? - probability_estimate: 1._prt - std::exp(-probability_estimate); + // std::expm1 is used since it maintains correctness for small exponent. + const amrex::ParticleReal probability = -std::expm1(-probability_estimate); // Get a random number const amrex::ParticleReal random_number = amrex::Random(engine); From 6aa2fc11a67791fc01fd07baba50df8e9bc4fa29 Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Thu, 4 Jan 2024 14:23:53 -0800 Subject: [PATCH 090/176] Beam-beam collision example (#4546) * adds beam beam collision test * updated inputs * add benchmark for BeamBeamCollision * updated test and ini * update checksums and inputs * adds docs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * added image * removed unused imports * add openpmd compiling flag * Update example * Update Examples/Physics_applications/beam-beam_collision/README.rst --------- Co-authored-by: Tools Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Remi Lehe --- Docs/source/refs.bib | 15 ++ Docs/source/usage/examples.rst | 1 + .../source/usage/examples/beam-beam_collision | 1 + .../beam-beam_collision/README.rst | 70 +++++ .../beam-beam_collision/inputs | 241 ++++++++++++++++++ .../benchmarks_json/BeamBeamCollision.json | 128 ++++++++++ Regression/WarpX-tests.ini | 17 ++ 7 files changed, 473 insertions(+) create mode 120000 Docs/source/usage/examples/beam-beam_collision create mode 100644 Examples/Physics_applications/beam-beam_collision/README.rst create mode 100644 Examples/Physics_applications/beam-beam_collision/inputs create mode 100644 Regression/Checksum/benchmarks_json/BeamBeamCollision.json diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 4b1cd1bd1a1..d94938775c6 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -277,6 +277,21 @@ @Inbook{VanLeerBookChapter1997 year = {1997} } +@article{Yakimenko2019, + title = {Prospect of Studying Nonperturbative QED with Beam-Beam Collisions}, + author = {Yakimenko, V. and Meuren, S. and Del Gaudio, F. and Baumann, C. and Fedotov, A. and Fiuza, F. and Grismayer, T. and Hogan, M. J. and Pukhov, A. and Silva, L. O. and White, G.}, + journal = {Phys. Rev. Lett.}, + volume = {122}, + issue = {19}, + pages = {190404}, + numpages = {7}, + year = {2019}, + month = {May}, + publisher = {American Physical Society}, + doi = {10.1103/PhysRevLett.122.190404}, + url = {https://link.aps.org/doi/10.1103/PhysRevLett.122.190404} +} + @article{Groenewald2023, author = {Groenewald, R. E. and Veksler, A. and Ceccherini, F. and Necas, A. and Nicks, B. S. and Barnes, D. C. and Tajima, T. and Dettrick, S. A.}, title = "{Accelerated kinetic model for global macro stability studies of high-beta fusion reactors}", diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index 5490068e448..855faab392a 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -56,6 +56,7 @@ Particle Accelerator & Beam Physics :maxdepth: 1 examples/gaussian_beam/README.rst + examples/beam-beam_collision/README.rst Coming soon: diff --git a/Docs/source/usage/examples/beam-beam_collision b/Docs/source/usage/examples/beam-beam_collision new file mode 120000 index 00000000000..8c6ac6b30b1 --- /dev/null +++ b/Docs/source/usage/examples/beam-beam_collision @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/beam-beam_collision \ No newline at end of file diff --git a/Examples/Physics_applications/beam-beam_collision/README.rst b/Examples/Physics_applications/beam-beam_collision/README.rst new file mode 100644 index 00000000000..559a81277db --- /dev/null +++ b/Examples/Physics_applications/beam-beam_collision/README.rst @@ -0,0 +1,70 @@ +.. _examples-beam-beam_collision: + +Beam-beam collision +==================== + +This example shows how to simulate the collision between two ultra-relativistic particle beams. +This is representative of what happens at the interaction point of a linear collider. +We consider a right-propagating electron bunch colliding against a left-propagating positron bunch. + +We turn on the Quantum Synchrotron QED module for photon emission (also known as beamstrahlung in the collider community) and +the Breit-Wheeler QED module for the generation of electron-positron pairs (also known as coherent pair generation in the collider community). + +To solve for the electromagnetic field we use the nodal version of the electrostatic relativistic solver. +This solver computes the average velocity of each species, and solves the corresponding relativistic Poisson equation (see the WarpX documentation for `warpx.do_electrostatic = relativistic` for more detail). This solver accurately reproduced the subtle cancellation that occur for some component of the ``E + v x B`` terms which are crucial in simulations of relativistic particles. + + +This example is based on the following paper :cite:t:`ex-Yakimenko2019`. + + +Run +--- + +The PICMI input file is not available for this example yet. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: inputs + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/beam-beam_collision/inputs``. + + +Visualize +--------- + +The figure below shows the number of photons emitted per beam particle (left) and the number of secondary pairs generated per beam particle (right). + +We compare different results: +* (red) simplified WarpX simulation as the example stored in the directory ``/Examples/Physics_applications/beam-beam_collision``; +* (blue) large-scale WarpX simulation (high resolution and ad hoc generated tables ; +* (black) literature results from :cite:t:`ex-Yakimenko2019`. + +The small-scale simulation has been performed with a resolution of ``nx = 64, ny = 64, nz = 128`` grid cells, while the large-scale one has a much higher resolution of ``nx = 512, ny = 512, nz = 1024``. Moreover, the large-scale simulation uses dedicated QED lookup tables instead of the builtin tables. To generate the tables within WarpX, the code must be compiled with the flag ``-DWarpX_QED_TABLE_GEN=ON``. For the large-scale simulation we have used the following options: + +.. code-block:: ini + + qed_qs.lookup_table_mode = generate + qed_bw.lookup_table_mode = generate + qed_qs.tab_dndt_chi_min=1e-3 + qed_qs.tab_dndt_chi_max=2e3 + qed_qs.tab_dndt_how_many=512 + qed_qs.tab_em_chi_min=1e-3 + qed_qs.tab_em_chi_max=2e3 + qed_qs.tab_em_chi_how_many=512 + qed_qs.tab_em_frac_how_many=512 + qed_qs.tab_em_frac_min=1e-12 + qed_qs.save_table_in=my_qs_table.txt + qed_bw.tab_dndt_chi_min=1e-2 + qed_bw.tab_dndt_chi_max=2e3 + qed_bw.tab_dndt_how_many=512 + qed_bw.tab_pair_chi_min=1e-2 + qed_bw.tab_pair_chi_max=2e3 + qed_bw.tab_pair_chi_how_many=512 + qed_bw.tab_pair_frac_how_many=512 + qed_bw.save_table_in=my_bw_table.txt + +.. figure:: https://user-images.githubusercontent.com/17280419/291749626-aa61fff2-e6d2-45a3-80ee-84b2851ea0bf.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTEiLCJleHAiOjE3MDMwMzQzNTEsIm5iZiI6MTcwMzAzNDA1MSwicGF0aCI6Ii8xNzI4MDQxOS8yOTE3NDk2MjYtYWE2MWZmZjItZTZkMi00NWEzLTgwZWUtODRiMjg1MWVhMGJmLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFJV05KWUFYNENTVkVINTNBJTJGMjAyMzEyMjAlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjMxMjIwVDAxMDA1MVomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPWFiYzY2MGQyYzIyZGIzYzUxOWI3MzNjZTk5ZDM1YzgyNmY4ZDYxOGRlZjAyZTIwNTAyMTc3NTgwN2Q0YjEwNGMmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0JmFjdG9yX2lkPTAma2V5X2lkPTAmcmVwb19pZD0wIn0.I96LQpjqmFXirPDVnBlFQIkCuenR6IuOSY0OIIQvtCo + :alt: Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. + :width: 100% + + Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. diff --git a/Examples/Physics_applications/beam-beam_collision/inputs b/Examples/Physics_applications/beam-beam_collision/inputs new file mode 100644 index 00000000000..e66f0e63050 --- /dev/null +++ b/Examples/Physics_applications/beam-beam_collision/inputs @@ -0,0 +1,241 @@ +################################# +########## MY CONSTANTS ######### +################################# +my_constants.mc2 = m_e*clight*clight +my_constants.nano = 1.0e-9 +my_constants.GeV = q_e*1.e9 + +# BEAMS +my_constants.beam_energy = 125.*GeV +my_constants.beam_uz = beam_energy/(mc2) +my_constants.beam_charge = 0.14*nano +my_constants.sigmax = 10*nano +my_constants.sigmay = 10*nano +my_constants.sigmaz = 10*nano +my_constants.beam_uth = 0.1/100.*beam_uz +my_constants.n0 = beam_charge / (q_e * sigmax * sigmay * sigmaz * (2.*pi)**(3./2.)) +my_constants.omegab = sqrt(n0 * q_e**2 / (epsilon0*m_e)) +my_constants.mux = 0.0 +my_constants.muy = 0.0 +my_constants.muz = -0.5*Lz+3.2*sigmaz + +# BOX +my_constants.Lx = 100.0*clight/omegab +my_constants.Ly = 100.0*clight/omegab +my_constants.Lz = 180.0*clight/omegab + +# for a full scale simulation use: nx, ny, nz = 512, 512, 1024 +my_constants.nx = 64 +my_constants.ny = 64 +my_constants.nz = 128 + + +# TIME +my_constants.T = 0.7*Lz/clight +my_constants.dt = sigmaz/clight/10. + +# DIAGS +my_constants.every_red = 1. +warpx.used_inputs_file = warpx_used_inputs.txt + +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = T +amr.n_cell = nx ny nz +amr.max_grid_size = 128 +amr.blocking_factor = 2 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.5*Lx -0.5*Ly -0.5*Lz +geometry.prob_hi = 0.5*Lx 0.5*Ly 0.5*Lz + +################################# +######## BOUNDARY CONDITION ##### +################################# +boundary.field_lo = PEC PEC PEC +boundary.field_hi = PEC PEC PEC +boundary.particle_lo = Absorbing Absorbing Absorbing +boundary.particle_hi = Absorbing Absorbing Absorbing + +################################# +############ NUMERICS ########### +################################# +warpx.do_electrostatic = relativistic +warpx.const_dt = dt +warpx.grid_type = collocated +algo.particle_shape = 3 +algo.load_balance_intervals=100 +algo.particle_pusher = vay + +################################# +########### PARTICLES ########### +################################# +particles.species_names = beam1 beam2 pho1 pho2 ele_nlbw1 pos_nlbw1 ele_nlbw2 pos_nlbw2 pho ele pos +particles.photon_species = pho1 pho2 pho + +beam1.species_type = electron +beam1.injection_style = NUniformPerCell +beam1.num_particles_per_cell_each_dim = 1 1 1 +beam1.profile = parse_density_function +beam1.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z-muz)**2/(2*sigmaz**2))" +beam1.density_min = n0 / 1e3 +beam1.momentum_distribution_type = gaussian +beam1.uz_m = beam_uz +beam1.uy_m = 0.0 +beam1.ux_m = 0.0 +beam1.ux_th = beam_uth +beam1.uy_th = beam_uth +beam1.uz_th = beam_uth +beam1.initialize_self_fields = 1 +beam1.self_fields_required_precision = 5e-10 +beam1.self_fields_max_iters = 10000 +beam1.do_qed_quantum_sync = 1 +beam1.qed_quantum_sync_phot_product_species = pho1 +beam1.do_classical_radiation_reaction = 0 + +beam2.species_type = positron +beam2.injection_style = NUniformPerCell +beam2.num_particles_per_cell_each_dim = 1 1 1 +beam2.profile = parse_density_function +beam2.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z+muz)**2/(2*sigmaz**2))" +beam2.density_min = n0 / 1e3 +beam2.momentum_distribution_type = gaussian +beam2.uz_m = -beam_uz +beam2.uy_m = 0.0 +beam2.ux_m = 0.0 +beam2.ux_th = beam_uth +beam2.uy_th = beam_uth +beam2.uz_th = beam_uth +beam2.initialize_self_fields = 1 +beam2.self_fields_required_precision = 5e-10 +beam2.self_fields_max_iters = 10000 +beam2.do_qed_quantum_sync = 1 +beam2.qed_quantum_sync_phot_product_species = pho2 +beam2.do_classical_radiation_reaction = 0 + +pho1.species_type = photon +pho1.injection_style = none +pho1.do_qed_breit_wheeler = 1 +pho1.qed_breit_wheeler_ele_product_species = ele_nlbw1 +pho1.qed_breit_wheeler_pos_product_species = pos_nlbw1 + +pho2.species_type = photon +pho2.injection_style = none +pho2.do_qed_breit_wheeler = 1 +pho2.qed_breit_wheeler_ele_product_species = ele_nlbw2 +pho2.qed_breit_wheeler_pos_product_species = pos_nlbw2 + +ele_nlbw1.species_type = electron +ele_nlbw1.injection_style = none +ele_nlbw1.self_fields_required_precision = 1e-11 +ele_nlbw1.self_fields_max_iters = 10000 +ele_nlbw1.do_qed_quantum_sync = 1 +ele_nlbw1.qed_quantum_sync_phot_product_species = pho +ele_nlbw1.do_classical_radiation_reaction = 0 + +pos_nlbw1.species_type = positron +pos_nlbw1.injection_style = none +pos_nlbw1.self_fields_required_precision = 1e-11 +pos_nlbw1.self_fields_max_iters = 10000 +pos_nlbw1.do_qed_quantum_sync = 1 +pos_nlbw1.qed_quantum_sync_phot_product_species = pho +pos_nlbw1.do_classical_radiation_reaction = 0 + +ele_nlbw2.species_type = electron +ele_nlbw2.injection_style = none +ele_nlbw2.self_fields_required_precision = 1e-11 +ele_nlbw2.self_fields_max_iters = 10000 +ele_nlbw2.do_qed_quantum_sync = 1 +ele_nlbw2.qed_quantum_sync_phot_product_species = pho +ele_nlbw2.do_classical_radiation_reaction = 0 + +pos_nlbw2.species_type = positron +pos_nlbw2.injection_style = none +pos_nlbw2.self_fields_required_precision = 1e-11 +pos_nlbw2.self_fields_max_iters = 10000 +pos_nlbw2.do_qed_quantum_sync = 1 +pos_nlbw2.qed_quantum_sync_phot_product_species = pho +pos_nlbw2.do_classical_radiation_reaction = 0 + +pho.species_type = photon +pho.injection_style = none +pho.do_qed_breit_wheeler = 1 +pho.qed_breit_wheeler_ele_product_species = ele +pho.qed_breit_wheeler_pos_product_species = pos + +ele.species_type = electron +ele.injection_style = none +ele.self_fields_required_precision = 1e-11 +ele.self_fields_max_iters = 10000 +ele.do_qed_quantum_sync = 1 +ele.qed_quantum_sync_phot_product_species = pho +ele.do_classical_radiation_reaction = 0 + +pos.species_type = positron +pos.injection_style = none +pos.self_fields_required_precision = 1e-11 +pos.self_fields_max_iters = 10000 +pos.do_qed_quantum_sync = 1 +pos.qed_quantum_sync_phot_product_species = pho +pos.do_classical_radiation_reaction = 0 + +################################# +############# QED ############### +################################# +qed_qs.photon_creation_energy_threshold = 0. + +qed_qs.lookup_table_mode = builtin +qed_qs.chi_min = 1.e-3 + +qed_bw.lookup_table_mode = builtin +qed_bw.chi_min = 1.e-2 + +# for accurate results use the generated tables with +# the following parameters +# note: must compile with -DWarpX_QED_TABLE_GEN=ON +#qed_qs.lookup_table_mode = generate +#qed_bw.lookup_table_mode = generate +#qed_qs.tab_dndt_chi_min=1e-3 +#qed_qs.tab_dndt_chi_max=2e3 +#qed_qs.tab_dndt_how_many=512 +#qed_qs.tab_em_chi_min=1e-3 +#qed_qs.tab_em_chi_max=2e3 +#qed_qs.tab_em_chi_how_many=512 +#qed_qs.tab_em_frac_how_many=512 +#qed_qs.tab_em_frac_min=1e-12 +#qed_qs.save_table_in=my_qs_table.txt +#qed_bw.tab_dndt_chi_min=1e-2 +#qed_bw.tab_dndt_chi_max=2e3 +#qed_bw.tab_dndt_how_many=512 +#qed_bw.tab_pair_chi_min=1e-2 +#qed_bw.tab_pair_chi_max=2e3 +#qed_bw.tab_pair_chi_how_many=512 +#qed_bw.tab_pair_frac_how_many=512 +#qed_bw.save_table_in=my_bw_table.txt + +warpx.do_qed_schwinger = 0. + +################################# +######### DIAGNOSTICS ########### +################################# +# FULL +diagnostics.diags_names = diag1 + +diag1.intervals = 0 +diag1.diag_type = Full +diag1.write_species = 1 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho_beam1 rho_beam2 rho_ele_nlbw1 rho_pos_nlbw1 rho_ele_nlbw2 rho_pos_nlbw2 rho_ele rho_pos +diag1.format = openpmd +diag1.dump_last_timestep = 1 +diag1.species = pho1 pho2 pho ele_nlbw1 pos_nlbw1 ele_nlbw2 pos_nlbw2 ele pos beam1 beam2 + +# REDUCED +warpx.reduced_diags_names = ParticleNumber ColliderRelevant_beam1_beam2 + +ColliderRelevant_beam1_beam2.type = ColliderRelevant +ColliderRelevant_beam1_beam2.intervals = every_red +ColliderRelevant_beam1_beam2.species = beam1 beam2 + +ParticleNumber.type = ParticleNumber +ParticleNumber.intervals = every_red diff --git a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json new file mode 100644 index 00000000000..da2a8500b53 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json @@ -0,0 +1,128 @@ +{ + "lev=0": { + "Bx": 970235841795.1099, + "By": 970175630167.5084, + "Bz": 51789226.68307851, + "Ex": 2.90874251210462e+20, + "Ey": 2.908934769466038e+20, + "Ez": 1.70819200999153e+17, + "rho_beam1": 7.969271809437626e+16, + "rho_beam2": 7.969216746820773e+16, + "rho_ele": 0.0, + "rho_ele_nlbw1": 299758753127495.56, + "rho_ele_nlbw2": 289860995480960.4, + "rho_pos": 613963689581.3057, + "rho_pos_nlbw1": 287891375746148.75, + "rho_pos_nlbw2": 295214367940598.1 + }, + "beam1": { + "particle_opticalDepthQSR": 104868.583557123, + "particle_position_x": 0.0015001423941487055, + "particle_position_y": 0.001500178877546805, + "particle_position_z": 0.004965525054740317, + "particle_momentum_x": 6.20719343794678e-15, + "particle_momentum_y": 6.1639864036830354e-15, + "particle_momentum_z": 6.807872002761295e-12, + "particle_weight": 635864683.4991333 + }, + "beam2": { + "particle_opticalDepthQSR": 104166.96636860794, + "particle_position_x": 0.001500093343835549, + "particle_position_y": 0.0015001802698114421, + "particle_position_z": 0.00496555337137147, + "particle_momentum_x": 6.1976310171196696e-15, + "particle_momentum_y": 6.1844493116189144e-15, + "particle_momentum_z": 6.796731846337045e-12, + "particle_weight": 635863931.615053 + }, + "ele": { + "particle_opticalDepthQSR": 0.0, + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + }, + "ele_nlbw1": { + "particle_opticalDepthQSR": 405.16545575146574, + "particle_position_x": 5.119886480641899e-06, + "particle_position_y": 5.266875857997872e-06, + "particle_position_z": 1.855770689791573e-05, + "particle_momentum_x": 5.744620543080891e-18, + "particle_momentum_y": 5.463133845399354e-18, + "particle_momentum_z": 2.562592228511742e-15, + "particle_weight": 2359597.3527386785 + }, + "ele_nlbw2": { + "particle_opticalDepthQSR": 354.8472581769572, + "particle_position_x": 5.235909912617143e-06, + "particle_position_y": 5.068613695636293e-06, + "particle_position_z": 1.6665840940849017e-05, + "particle_momentum_x": 4.47916023711595e-18, + "particle_momentum_y": 4.092494634513803e-18, + "particle_momentum_z": 2.2277004588526963e-15, + "particle_weight": 2354612.144268994 + }, + "pho": { + "particle_opticalDepthBW": 462.76829758918615, + "particle_position_x": 6.163512168886876e-06, + "particle_position_y": 5.4614856451855956e-06, + "particle_position_z": 2.0316590334121627e-05, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 2923343.7532989895 + }, + "pho1": { + "particle_opticalDepthBW": 9769.560457892825, + "particle_position_x": 0.00013930069652083352, + "particle_position_y": 0.00014046343589264903, + "particle_position_z": 0.00046498427057467117, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 59881947.934441805 + }, + "pho2": { + "particle_opticalDepthBW": 9986.841892753177, + "particle_position_x": 0.000143478808886347, + "particle_position_y": 0.00014235781306129165, + "particle_position_z": 0.0004760406640177353, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 60412035.77539255 + }, + "pos": { + "particle_opticalDepthQSR": 1.0913353138611541, + "particle_position_x": 1.297022467502343e-08, + "particle_position_y": 6.533761995664651e-09, + "particle_position_z": 5.270388297177834e-08, + "particle_momentum_x": 9.474214808949084e-21, + "particle_momentum_y": 3.664691658234474e-21, + "particle_momentum_z": 5.559027877774493e-20, + "particle_weight": 4777.4993625771185 + }, + "pos_nlbw1": { + "particle_opticalDepthQSR": 353.1921715878159, + "particle_position_x": 5.413239275792192e-06, + "particle_position_y": 5.599083716468022e-06, + "particle_position_z": 1.729829030925741e-05, + "particle_momentum_x": 5.130129014453181e-18, + "particle_momentum_y": 5.151236001734455e-18, + "particle_momentum_z": 2.7592435303859743e-15, + "particle_weight": 2298018.034763882 + }, + "pos_nlbw2": { + "particle_opticalDepthQSR": 350.113008933412, + "particle_position_x": 4.925666132330728e-06, + "particle_position_y": 4.677928261139806e-06, + "particle_position_z": 1.7528432992559524e-05, + "particle_momentum_x": 5.3410973872204896e-18, + "particle_momentum_y": 5.0088980554153576e-18, + "particle_momentum_z": 2.4803849492900593e-15, + "particle_weight": 2379194.375363683 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 610c5befa67..974615ad725 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4480,6 +4480,23 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/nodal_electrostatic/analysis_3d.py +[BeamBeamCollision] +buildDir = . +inputFile = Examples/Physics_applications/beam-beam_collision/inputs +runtime_params = warpx.abort_on_warning_threshold=high +dim = 3 +addToCompileString = USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +outputFile = BeamBeamCollision_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py + [ImplicitPicard_1d] buildDir = . inputFile = Examples/Tests/Implicit/inputs_1d From fa2c8558c8c29147edfb7dba08ffda039159d047 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 5 Jan 2024 01:20:24 +0100 Subject: [PATCH 091/176] Perlmutter: Load `gpu` Module (#4573) If a user loaded the `cpu` module, they might lack the CUDA modules. This makes the load a bit more robust, in lieu of a working `module purge` on Cray/HPE systems. --- .../perlmutter-nersc/perlmutter_gpu_warpx.profile.example | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 13ab2ead605..54cd33fe476 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -6,6 +6,12 @@ export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi # required dependencies +module load gpu +module load PrgEnv-gnu +module load craype +module load craype-x86-milan +module load craype-accel-nvidia80 +module load cudatoolkit module load cmake/3.22.0 # optional: for QED support with detailed tables From 685ae3fee9ba02b38b9675c4ea68b4f893283433 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 4 Jan 2024 22:28:20 -0800 Subject: [PATCH 092/176] Scale fields by a dimensionless number between 0 and 1 (#4575) * Scale fields by a number between 0 and 1 * Update Source/Particles/Gather/ScaleFields.H --- .../LaserAccelerationBoost.json | 52 ++++++++--------- .../comoving_2d_psatd_hybrid.json | 50 ++++++++--------- .../galilean_2d_psatd_hybrid.json | 56 +++++++++---------- Source/Particles/Gather/ScaleFields.H | 4 +- 4 files changed, 81 insertions(+), 81 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json index 94e243f21a2..0efcdaeca3c 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json @@ -1,38 +1,38 @@ { "lev=0": { - "Bx": 4818955.485214876, - "By": 1752.8017794063862, - "Bz": 14516.212849468406, - "Ex": 2366115511749.6064, - "Ey": 1446112026972328.0, - "Ez": 21864189477873.78, - "jx": 1996366361598696.5, - "jy": 5.312583836700398e+16, - "jz": 2.0491352591140016e+16, - "rho": 68443961.6079968 + "Bx": 4818955.480792835, + "By": 1752.8025402207227, + "Bz": 14516.21278267981, + "Ex": 2366115496505.249, + "Ey": 1446112025634143.0, + "Ez": 21864189507353.19, + "jx": 1996366349839211.5, + "jy": 5.312583827165288e+16, + "jz": 2.049135262445976e+16, + "rho": 68443961.71835628 }, "electrons": { - "particle_momentum_x": 2.2135944672847805e-23, - "particle_momentum_y": 2.82245592458273e-22, - "particle_momentum_z": 5.260626007649988e-22, - "particle_position_x": 0.010800577787630073, - "particle_position_y": 0.21115060628317733, + "particle_momentum_x": 2.2135945391227107e-23, + "particle_momentum_y": 2.8224559499572622e-22, + "particle_momentum_z": 5.260626010211241e-22, + "particle_position_x": 0.010800577787628053, + "particle_position_y": 0.2111506062831815, "particle_weight": 4.121554826246186e+16 }, "ions": { - "particle_momentum_x": 6.24847236820344e-23, - "particle_momentum_y": 4.449097670697282e-22, - "particle_momentum_z": 5.768168724446374e-22, - "particle_position_x": 0.010800001678510515, - "particle_position_y": 0.21114947608115428, + "particle_momentum_x": 6.248472277235318e-23, + "particle_momentum_y": 4.449097689427615e-22, + "particle_momentum_z": 5.768168724780326e-22, + "particle_position_x": 0.010800001678510512, + "particle_position_y": 0.21114947608115425, "particle_weight": 4.121554826246186e+16 }, "beam": { - "particle_momentum_x": 3.5357352601344873e-19, - "particle_momentum_y": 4.363147101327531e-19, - "particle_momentum_z": 5.658494028187168e-17, - "particle_position_x": 0.008314957057032625, - "particle_position_y": 1.1704335719922687, + "particle_momentum_x": 3.535745635169933e-19, + "particle_momentum_y": 4.363391839372122e-19, + "particle_momentum_z": 5.658606416951657e-17, + "particle_position_x": 0.008314723025211447, + "particle_position_y": 1.1704335743854242, "particle_weight": 62415090744.60765 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json index c8571869cd5..8b03899369b 100644 --- a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json @@ -1,38 +1,38 @@ { "lev=0": { - "Bx": 1118808.3734374465, - "By": 3248957.1122179274, - "Bz": 280612.78289064515, - "Ex": 975532732112639.6, - "Ey": 402861836732114.4, - "Ez": 159049610399317.66, - "jx": 2.9997053130250828e+16, - "jy": 8.866654890775573e+16, - "jz": 3.163974708948631e+17, - "rho": 1059977729.0184418 + "Bx": 1118808.3686978193, + "By": 3248970.5506422943, + "Bz": 280612.7921641442, + "Ex": 975536649649286.1, + "Ey": 402861835403418.1, + "Ez": 159049265640492.28, + "jx": 2.9996888133195436e+16, + "jy": 8.866654944519546e+16, + "jz": 3.164008885453435e+17, + "rho": 1059988299.6088305 }, "ions": { - "particle_momentum_x": 1.6150569180478943e-18, - "particle_momentum_y": 2.2334266828401142e-18, - "particle_momentum_z": 4.2792495306800117e-13, - "particle_position_x": 1.4883816864865955, - "particle_position_y": 16.45238650413084, + "particle_momentum_x": 1.6150513873065298e-18, + "particle_momentum_y": 2.233426695677123e-18, + "particle_momentum_z": 4.279249529993671e-13, + "particle_position_x": 1.4883816864183497, + "particle_position_y": 16.452386504127254, "particle_weight": 1.234867369440658e+18 }, "electrons": { - "particle_momentum_x": 7.05821754019506e-19, - "particle_momentum_y": 2.2042393263043917e-18, - "particle_momentum_z": 2.5305214316289944e-16, - "particle_position_x": 1.5006580331362074, - "particle_position_y": 16.45438830674347, + "particle_momentum_x": 7.058167362825288e-19, + "particle_momentum_y": 2.204239326446281e-18, + "particle_momentum_z": 2.530521998715408e-16, + "particle_position_x": 1.5006581263609764, + "particle_position_y": 16.454388313398017, "particle_weight": 1.234867020725368e+18 }, "beam": { - "particle_momentum_x": 6.874634694077579e-19, - "particle_momentum_y": 4.374677735660533e-19, - "particle_momentum_z": 6.432600800266472e-18, - "particle_position_x": 0.0012933700124436584, - "particle_position_y": 0.358720803656086, + "particle_momentum_x": 6.869222298759882e-19, + "particle_momentum_y": 4.374719809060106e-19, + "particle_momentum_z": 6.4523206583503136e-18, + "particle_position_x": 0.001290816359726098, + "particle_position_y": 0.3586691102823157, "particle_weight": 3120754537230.3823 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json index 03cf06b3158..dd56f8170a9 100644 --- a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json @@ -1,38 +1,38 @@ { "lev=0": { - "Bx": 1086729.9983020213, - "By": 2886537.292820136, - "Bz": 264259.5410465989, - "Ex": 867382749986595.4, - "Ey": 392666737316432.8, - "Ez": 146897959667920.22, - "jx": 2.7028854305928356e+16, - "jy": 8.615938520657634e+16, - "jz": 2.7328721771319978e+17, - "rho": 915931445.5917195 - }, - "electrons": { - "particle_momentum_x": 6.2409905560361695e-19, - "particle_momentum_y": 1.5790611507526441e-18, - "particle_momentum_z": 2.506435264953701e-16, - "particle_position_x": 1.5014136629221837, - "particle_position_y": 16.52378170697116, - "particle_weight": 1.2372401466086835e+18 + "Bx": 1086729.9718613266, + "By": 2886554.482275311, + "Bz": 264259.55093734514, + "Ex": 867387781289915.2, + "Ey": 392666724461952.5, + "Ez": 146897592531660.03, + "jx": 2.702866174672266e+16, + "jy": 8.615938361747776e+16, + "jz": 2.7329155817806224e+17, + "rho": 915945723.7934376 }, "ions": { - "particle_momentum_x": 1.4394967936107095e-18, - "particle_momentum_y": 1.5967629122067375e-18, - "particle_momentum_z": 4.287340658682592e-13, - "particle_position_x": 1.4911814217840271, - "particle_position_y": 16.521964978774346, + "particle_momentum_x": 1.4394902513923003e-18, + "particle_momentum_y": 1.5967629157922875e-18, + "particle_momentum_z": 4.287340658051679e-13, + "particle_position_x": 1.4911814217142487, + "particle_position_y": 16.521964978771, "particle_weight": 1.2372405194129536e+18 }, + "electrons": { + "particle_momentum_x": 6.240933687389075e-19, + "particle_momentum_y": 1.5790611427694247e-18, + "particle_momentum_z": 2.5064357834741096e-16, + "particle_position_x": 1.501413766926399, + "particle_position_y": 16.523781713952324, + "particle_weight": 1.2372401466086835e+18 + }, "beam": { - "particle_momentum_x": 7.0050974522936195e-19, - "particle_momentum_y": 4.374915691594075e-19, - "particle_momentum_z": 6.1754662837862646e-18, - "particle_position_x": 0.0016025830388130494, - "particle_position_y": 0.35897909980539305, + "particle_momentum_x": 7.000932845220306e-19, + "particle_momentum_y": 4.374936866729326e-19, + "particle_momentum_z": 6.194468548032543e-18, + "particle_position_x": 0.0016030835496557787, + "particle_position_y": 0.3589262705964349, "particle_weight": 3120754537230.3823 } } \ No newline at end of file diff --git a/Source/Particles/Gather/ScaleFields.H b/Source/Particles/Gather/ScaleFields.H index 10a10cfe190..4cddfa5a342 100644 --- a/Source/Particles/Gather/ScaleFields.H +++ b/Source/Particles/Gather/ScaleFields.H @@ -47,8 +47,8 @@ struct ScaleFields // This only approximates what should be happening. The particles // should by advanced a fraction of a time step instead. // Scaling the fields is much easier and may be good enough. - const amrex::Real dtscale = m_dt - (m_z_plane_previous - zp)/(m_vz_ave_boosted + m_v_boost); - if (0._rt < dtscale && dtscale < m_dt) + const amrex::Real dtscale = 1._rt - (m_z_plane_previous - zp)/(m_vz_ave_boosted + m_v_boost)/m_dt; + if (0._rt < dtscale && dtscale < 1._rt) { Exp *= dtscale; Eyp *= dtscale; From 920b3e8445d1181bf3360455bcceda2a9498d50c Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Fri, 5 Jan 2024 11:19:54 -0800 Subject: [PATCH 093/176] Beam-beam collision example follow-up (#4578) * Update species * Update checksum --------- Co-authored-by: Remi Lehe --- .../beam-beam_collision/inputs | 114 +++++------ .../benchmarks_json/BeamBeamCollision.json | 180 +++++++----------- 2 files changed, 126 insertions(+), 168 deletions(-) diff --git a/Examples/Physics_applications/beam-beam_collision/inputs b/Examples/Physics_applications/beam-beam_collision/inputs index e66f0e63050..488e997f895 100644 --- a/Examples/Physics_applications/beam-beam_collision/inputs +++ b/Examples/Physics_applications/beam-beam_collision/inputs @@ -71,8 +71,8 @@ algo.particle_pusher = vay ################################# ########### PARTICLES ########### ################################# -particles.species_names = beam1 beam2 pho1 pho2 ele_nlbw1 pos_nlbw1 ele_nlbw2 pos_nlbw2 pho ele pos -particles.photon_species = pho1 pho2 pho +particles.species_names = beam1 beam2 pho1 pho2 ele1 pos1 ele2 pos2 +particles.photon_species = pho1 pho2 beam1.species_type = electron beam1.injection_style = NUniformPerCell @@ -117,68 +117,58 @@ beam2.do_classical_radiation_reaction = 0 pho1.species_type = photon pho1.injection_style = none pho1.do_qed_breit_wheeler = 1 -pho1.qed_breit_wheeler_ele_product_species = ele_nlbw1 -pho1.qed_breit_wheeler_pos_product_species = pos_nlbw1 +pho1.qed_breit_wheeler_ele_product_species = ele1 +pho1.qed_breit_wheeler_pos_product_species = pos1 pho2.species_type = photon pho2.injection_style = none pho2.do_qed_breit_wheeler = 1 -pho2.qed_breit_wheeler_ele_product_species = ele_nlbw2 -pho2.qed_breit_wheeler_pos_product_species = pos_nlbw2 - -ele_nlbw1.species_type = electron -ele_nlbw1.injection_style = none -ele_nlbw1.self_fields_required_precision = 1e-11 -ele_nlbw1.self_fields_max_iters = 10000 -ele_nlbw1.do_qed_quantum_sync = 1 -ele_nlbw1.qed_quantum_sync_phot_product_species = pho -ele_nlbw1.do_classical_radiation_reaction = 0 - -pos_nlbw1.species_type = positron -pos_nlbw1.injection_style = none -pos_nlbw1.self_fields_required_precision = 1e-11 -pos_nlbw1.self_fields_max_iters = 10000 -pos_nlbw1.do_qed_quantum_sync = 1 -pos_nlbw1.qed_quantum_sync_phot_product_species = pho -pos_nlbw1.do_classical_radiation_reaction = 0 - -ele_nlbw2.species_type = electron -ele_nlbw2.injection_style = none -ele_nlbw2.self_fields_required_precision = 1e-11 -ele_nlbw2.self_fields_max_iters = 10000 -ele_nlbw2.do_qed_quantum_sync = 1 -ele_nlbw2.qed_quantum_sync_phot_product_species = pho -ele_nlbw2.do_classical_radiation_reaction = 0 - -pos_nlbw2.species_type = positron -pos_nlbw2.injection_style = none -pos_nlbw2.self_fields_required_precision = 1e-11 -pos_nlbw2.self_fields_max_iters = 10000 -pos_nlbw2.do_qed_quantum_sync = 1 -pos_nlbw2.qed_quantum_sync_phot_product_species = pho -pos_nlbw2.do_classical_radiation_reaction = 0 - -pho.species_type = photon -pho.injection_style = none -pho.do_qed_breit_wheeler = 1 -pho.qed_breit_wheeler_ele_product_species = ele -pho.qed_breit_wheeler_pos_product_species = pos - -ele.species_type = electron -ele.injection_style = none -ele.self_fields_required_precision = 1e-11 -ele.self_fields_max_iters = 10000 -ele.do_qed_quantum_sync = 1 -ele.qed_quantum_sync_phot_product_species = pho -ele.do_classical_radiation_reaction = 0 - -pos.species_type = positron -pos.injection_style = none -pos.self_fields_required_precision = 1e-11 -pos.self_fields_max_iters = 10000 -pos.do_qed_quantum_sync = 1 -pos.qed_quantum_sync_phot_product_species = pho -pos.do_classical_radiation_reaction = 0 +pho2.qed_breit_wheeler_ele_product_species = ele2 +pho2.qed_breit_wheeler_pos_product_species = pos2 + +ele1.species_type = electron +ele1.injection_style = none +ele1.self_fields_required_precision = 1e-11 +ele1.self_fields_max_iters = 10000 +ele1.do_qed_quantum_sync = 1 +ele1.qed_quantum_sync_phot_product_species = pho1 +ele1.do_classical_radiation_reaction = 0 + +pos1.species_type = positron +pos1.injection_style = none +pos1.self_fields_required_precision = 1e-11 +pos1.self_fields_max_iters = 10000 +pos1.do_qed_quantum_sync = 1 +pos1.qed_quantum_sync_phot_product_species = pho1 +pos1.do_classical_radiation_reaction = 0 + +ele2.species_type = electron +ele2.injection_style = none +ele2.self_fields_required_precision = 1e-11 +ele2.self_fields_max_iters = 10000 +ele2.do_qed_quantum_sync = 1 +ele2.qed_quantum_sync_phot_product_species = pho2 +ele2.do_classical_radiation_reaction = 0 + +pos2.species_type = positron +pos2.injection_style = none +pos2.self_fields_required_precision = 1e-11 +pos2.self_fields_max_iters = 10000 +pos2.do_qed_quantum_sync = 1 +pos2.qed_quantum_sync_phot_product_species = pho2 +pos2.do_classical_radiation_reaction = 0 + +pho1.species_type = photon +pho1.injection_style = none +pho1.do_qed_breit_wheeler = 1 +pho1.qed_breit_wheeler_ele_product_species = ele1 +pho1.qed_breit_wheeler_pos_product_species = pos1 + +pho2.species_type = photon +pho2.injection_style = none +pho2.do_qed_breit_wheeler = 1 +pho2.qed_breit_wheeler_ele_product_species = ele2 +pho2.qed_breit_wheeler_pos_product_species = pos2 ################################# ############# QED ############### @@ -225,10 +215,10 @@ diagnostics.diags_names = diag1 diag1.intervals = 0 diag1.diag_type = Full diag1.write_species = 1 -diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho_beam1 rho_beam2 rho_ele_nlbw1 rho_pos_nlbw1 rho_ele_nlbw2 rho_pos_nlbw2 rho_ele rho_pos +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho_beam1 rho_beam2 rho_ele1 rho_pos1 rho_ele2 rho_pos2 diag1.format = openpmd diag1.dump_last_timestep = 1 -diag1.species = pho1 pho2 pho ele_nlbw1 pos_nlbw1 ele_nlbw2 pos_nlbw2 ele pos beam1 beam2 +diag1.species = pho1 pho2 ele1 pos1 ele2 pos2 beam1 beam2 # REDUCED warpx.reduced_diags_names = ParticleNumber ColliderRelevant_beam1_beam2 diff --git a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json index da2a8500b53..16c27055ab5 100644 --- a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json +++ b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json @@ -1,128 +1,96 @@ { "lev=0": { - "Bx": 970235841795.1099, - "By": 970175630167.5084, - "Bz": 51789226.68307851, - "Ex": 2.90874251210462e+20, - "Ey": 2.908934769466038e+20, - "Ez": 1.70819200999153e+17, - "rho_beam1": 7.969271809437626e+16, - "rho_beam2": 7.969216746820773e+16, - "rho_ele": 0.0, - "rho_ele_nlbw1": 299758753127495.56, - "rho_ele_nlbw2": 289860995480960.4, - "rho_pos": 613963689581.3057, - "rho_pos_nlbw1": 287891375746148.75, - "rho_pos_nlbw2": 295214367940598.1 + "Bx": 971135657171.612, + "By": 971078812454.5405, + "Bz": 20140193.235893946, + "Ex": 2.9111756162943966e+20, + "Ey": 2.9113725115697712e+20, + "Ez": 1.0213536367191107e+17, + "rho_beam1": 7.970337929028706e+16, + "rho_beam2": 7.969213804851568e+16, + "rho_ele1": 343600677331163.7, + "rho_ele2": 302746939366837.25, + "rho_pos1": 333855946230626.06, + "rho_pos2": 310879461124837.44 }, "beam1": { - "particle_opticalDepthQSR": 104868.583557123, - "particle_position_x": 0.0015001423941487055, - "particle_position_y": 0.001500178877546805, - "particle_position_z": 0.004965525054740317, - "particle_momentum_x": 6.20719343794678e-15, - "particle_momentum_y": 6.1639864036830354e-15, - "particle_momentum_z": 6.807872002761295e-12, - "particle_weight": 635864683.4991333 + "particle_opticalDepthQSR": 104909.59461909423, + "particle_position_x": 0.001500222221634118, + "particle_position_y": 0.0015002445303035634, + "particle_position_z": 0.0049656251227976015, + "particle_momentum_x": 6.205341799808723e-15, + "particle_momentum_y": 6.1592257603817594e-15, + "particle_momentum_z": 6.806886719670214e-12, + "particle_weight": 635949610.5135971 }, "beam2": { - "particle_opticalDepthQSR": 104166.96636860794, - "particle_position_x": 0.001500093343835549, - "particle_position_y": 0.0015001802698114421, - "particle_position_z": 0.00496555337137147, - "particle_momentum_x": 6.1976310171196696e-15, - "particle_momentum_y": 6.1844493116189144e-15, - "particle_momentum_z": 6.796731846337045e-12, - "particle_weight": 635863931.615053 + "particle_opticalDepthQSR": 104164.848014815, + "particle_position_x": 0.0015001011957527532, + "particle_position_y": 0.001500139975740741, + "particle_position_z": 0.004965479176845744, + "particle_momentum_x": 6.200690794877584e-15, + "particle_momentum_y": 6.186048913459861e-15, + "particle_momentum_z": 6.7990490255176515e-12, + "particle_weight": 635863144.251134 }, - "ele": { - "particle_opticalDepthQSR": 0.0, - "particle_position_x": 0.0, - "particle_position_y": 0.0, - "particle_position_z": 0.0, - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_weight": 0.0 + "ele1": { + "particle_opticalDepthQSR": 435.73003117907257, + "particle_position_x": 4.882183530045367e-06, + "particle_position_y": 4.841391483672882e-06, + "particle_position_z": 1.8449175055560687e-05, + "particle_momentum_x": 5.6656608489971696e-18, + "particle_momentum_y": 5.724425295258085e-18, + "particle_momentum_z": 2.6277553331470036e-15, + "particle_weight": 2696555.9200674472 }, - "ele_nlbw1": { - "particle_opticalDepthQSR": 405.16545575146574, - "particle_position_x": 5.119886480641899e-06, - "particle_position_y": 5.266875857997872e-06, - "particle_position_z": 1.855770689791573e-05, - "particle_momentum_x": 5.744620543080891e-18, - "particle_momentum_y": 5.463133845399354e-18, - "particle_momentum_z": 2.562592228511742e-15, - "particle_weight": 2359597.3527386785 - }, - "ele_nlbw2": { - "particle_opticalDepthQSR": 354.8472581769572, - "particle_position_x": 5.235909912617143e-06, - "particle_position_y": 5.068613695636293e-06, - "particle_position_z": 1.6665840940849017e-05, - "particle_momentum_x": 4.47916023711595e-18, - "particle_momentum_y": 4.092494634513803e-18, - "particle_momentum_z": 2.2277004588526963e-15, - "particle_weight": 2354612.144268994 - }, - "pho": { - "particle_opticalDepthBW": 462.76829758918615, - "particle_position_x": 6.163512168886876e-06, - "particle_position_y": 5.4614856451855956e-06, - "particle_position_z": 2.0316590334121627e-05, - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_weight": 2923343.7532989895 + "ele2": { + "particle_opticalDepthQSR": 340.82229684726735, + "particle_position_x": 5.233059654483856e-06, + "particle_position_y": 4.781569085220371e-06, + "particle_position_z": 1.6293559425324337e-05, + "particle_momentum_x": 4.802611971470525e-18, + "particle_momentum_y": 4.3556825243407754e-18, + "particle_momentum_z": 2.41587659230925e-15, + "particle_weight": 2481137.860727036 }, "pho1": { - "particle_opticalDepthBW": 9769.560457892825, - "particle_position_x": 0.00013930069652083352, - "particle_position_y": 0.00014046343589264903, - "particle_position_z": 0.00046498427057467117, + "particle_opticalDepthBW": 9894.64959724129, + "particle_position_x": 0.00014191369823397644, + "particle_position_y": 0.00014347545392717968, + "particle_position_z": 0.00047442826029322116, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_weight": 59881947.934441805 + "particle_weight": 61063948.68610941 }, "pho2": { - "particle_opticalDepthBW": 9986.841892753177, - "particle_position_x": 0.000143478808886347, - "particle_position_y": 0.00014235781306129165, - "particle_position_z": 0.0004760406640177353, + "particle_opticalDepthBW": 10292.1955840901, + "particle_position_x": 0.00014731892710321073, + "particle_position_y": 0.00014515617182809124, + "particle_position_z": 0.00048756513452074315, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_weight": 60412035.77539255 - }, - "pos": { - "particle_opticalDepthQSR": 1.0913353138611541, - "particle_position_x": 1.297022467502343e-08, - "particle_position_y": 6.533761995664651e-09, - "particle_position_z": 5.270388297177834e-08, - "particle_momentum_x": 9.474214808949084e-21, - "particle_momentum_y": 3.664691658234474e-21, - "particle_momentum_z": 5.559027877774493e-20, - "particle_weight": 4777.4993625771185 + "particle_weight": 62299636.622087024 }, - "pos_nlbw1": { - "particle_opticalDepthQSR": 353.1921715878159, - "particle_position_x": 5.413239275792192e-06, - "particle_position_y": 5.599083716468022e-06, - "particle_position_z": 1.729829030925741e-05, - "particle_momentum_x": 5.130129014453181e-18, - "particle_momentum_y": 5.151236001734455e-18, - "particle_momentum_z": 2.7592435303859743e-15, - "particle_weight": 2298018.034763882 + "pos1": { + "particle_opticalDepthQSR": 387.7441392212553, + "particle_position_x": 5.1462880118803425e-06, + "particle_position_y": 5.2613832293016684e-06, + "particle_position_z": 1.7054223425917483e-05, + "particle_momentum_x": 4.6437665862693495e-18, + "particle_momentum_y": 4.761862836969051e-18, + "particle_momentum_z": 2.3776996599289627e-15, + "particle_weight": 2625121.7841375084 }, - "pos_nlbw2": { - "particle_opticalDepthQSR": 350.113008933412, - "particle_position_x": 4.925666132330728e-06, - "particle_position_y": 4.677928261139806e-06, - "particle_position_z": 1.7528432992559524e-05, - "particle_momentum_x": 5.3410973872204896e-18, - "particle_momentum_y": 5.0088980554153576e-18, - "particle_momentum_z": 2.4803849492900593e-15, - "particle_weight": 2379194.375363683 + "pos2": { + "particle_opticalDepthQSR": 361.943365907597, + "particle_position_x": 4.969019565031149e-06, + "particle_position_y": 4.361394970806125e-06, + "particle_position_z": 1.7413304358612675e-05, + "particle_momentum_x": 5.6348322786528905e-18, + "particle_momentum_y": 4.8171439953214205e-18, + "particle_momentum_z": 2.1937254860708963e-15, + "particle_weight": 2529794.7740602638 } } From 5b765dc92b291c93bfe30e594041ea9b99a64a83 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Fri, 5 Jan 2024 12:59:49 -0800 Subject: [PATCH 094/176] Reduce tolerance for DSMC CI test (#4576) * reduce tolerance for DSMC CI test * update json file with new benchmark values * update reference density array --- .../capacitive_discharge/analysis_dsmc.py | 72 +++++++++---------- .../benchmarks_json/Python_dsmc_1d.json | 40 +++++------ 2 files changed, 55 insertions(+), 57 deletions(-) diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py index df773dd9deb..a7a76be46ad 100755 --- a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py +++ b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py @@ -15,46 +15,44 @@ fn = sys.argv[1] test_name = os.path.split(os.getcwd())[1] -my_check = checksumAPI.evaluate_checksum( - test_name, fn, do_particles=True, rtol=0.01 -) +my_check = checksumAPI.evaluate_checksum(test_name, fn, do_particles=True) ref_density = np.array([ - 1.27953969e+14, 2.23553999e+14, 2.55384510e+14, 2.55663110e+14, - 2.55805760e+14, 2.55812087e+14, 2.55813911e+14, 2.55754104e+14, - 2.55929601e+14, 2.56085472e+14, 2.55932867e+14, 2.55828121e+14, - 2.55901711e+14, 2.55985074e+14, 2.56182697e+14, 2.56446847e+14, - 2.56483696e+14, 2.56301187e+14, 2.56245301e+14, 2.56797584e+14, - 2.57257907e+14, 2.57023627e+14, 2.56500876e+14, 2.56106851e+14, - 2.56283546e+14, 2.56723967e+14, 2.56960855e+14, 2.56825486e+14, - 2.56674669e+14, 2.56567191e+14, 2.56310927e+14, 2.56361171e+14, - 2.56692197e+14, 2.56743606e+14, 2.56653108e+14, 2.56883854e+14, - 2.56763228e+14, 2.56343726e+14, 2.56385489e+14, 2.56570110e+14, - 2.56538112e+14, 2.56472179e+14, 2.56322922e+14, 2.56195384e+14, - 2.56474576e+14, 2.56764233e+14, 2.56533016e+14, 2.56257170e+14, - 2.56362463e+14, 2.56363962e+14, 2.56311292e+14, 2.56678788e+14, - 2.57061138e+14, 2.56785892e+14, 2.56406603e+14, 2.56334908e+14, - 2.56120051e+14, 2.56003269e+14, 2.56132187e+14, 2.56329572e+14, - 2.56535713e+14, 2.56708950e+14, 2.56661860e+14, 2.56448986e+14, - 2.56386823e+14, 2.56233660e+14, 2.56137632e+14, 2.56206263e+14, - 2.56364996e+14, 2.56483536e+14, 2.56308741e+14, 2.56447231e+14, - 2.56896301e+14, 2.56691405e+14, 2.56170780e+14, 2.56122216e+14, - 2.56427399e+14, 2.56897558e+14, 2.56928868e+14, 2.56659033e+14, - 2.56749993e+14, 2.56952497e+14, 2.56798907e+14, 2.56377081e+14, - 2.56453057e+14, 2.56796632e+14, 2.56944576e+14, 2.57248469e+14, - 2.57279426e+14, 2.56849516e+14, 2.56601834e+14, 2.56850545e+14, - 2.56953072e+14, 2.56442586e+14, 2.56329006e+14, 2.56790661e+14, - 2.57083582e+14, 2.57075550e+14, 2.56719615e+14, 2.56220486e+14, - 2.56222323e+14, 2.56547365e+14, 2.56499423e+14, 2.56434041e+14, - 2.56378587e+14, 2.56249892e+14, 2.56380492e+14, 2.56504513e+14, - 2.56337631e+14, 2.56204891e+14, 2.56325116e+14, 2.56297798e+14, - 2.56112782e+14, 2.56054218e+14, 2.56320120e+14, 2.56580938e+14, - 2.56446800e+14, 2.56267011e+14, 2.56372853e+14, 2.56617592e+14, - 2.56630745e+14, 2.56615242e+14, 2.56625259e+14, 2.56561320e+14, - 2.56640072e+14, 2.56693273e+14, 2.56613237e+14, 2.24169847e+14, - 1.27683197e+14 + 1.27957355e+14, 2.23554080e+14, 2.55373436e+14, 2.55659492e+14, + 2.55814670e+14, 2.55818418e+14, 2.55811882e+14, 2.55742272e+14, + 2.55912888e+14, 2.56086072e+14, 2.55944486e+14, 2.55830183e+14, + 2.55909337e+14, 2.56008609e+14, 2.56205930e+14, 2.56421940e+14, + 2.56369990e+14, 2.56151020e+14, 2.55925823e+14, 2.55924941e+14, + 2.56067211e+14, 2.56264104e+14, 2.56435035e+14, 2.56543804e+14, + 2.56715146e+14, 2.56639305e+14, 2.56509438e+14, 2.56478881e+14, + 2.56406748e+14, 2.56194832e+14, 2.56126186e+14, 2.56442221e+14, + 2.56603784e+14, 2.56592554e+14, 2.56475838e+14, 2.56304135e+14, + 2.56310993e+14, 2.56298883e+14, 2.56386742e+14, 2.56555670e+14, + 2.56588013e+14, 2.56851444e+14, 2.56928531e+14, 2.56637559e+14, + 2.56678652e+14, 2.56827322e+14, 2.56630197e+14, 2.56295404e+14, + 2.56285079e+14, 2.56558116e+14, 2.56676094e+14, 2.56577780e+14, + 2.56599749e+14, 2.56540500e+14, 2.56292984e+14, 2.56230350e+14, + 2.56363607e+14, 2.56553909e+14, 2.56501054e+14, 2.56249684e+14, + 2.56280268e+14, 2.56558208e+14, 2.56437837e+14, 2.56152650e+14, + 2.56143349e+14, 2.56067330e+14, 2.56020624e+14, 2.56039223e+14, + 2.56306096e+14, 2.56693084e+14, 2.56649778e+14, 2.56589778e+14, + 2.56594097e+14, 2.56368788e+14, 2.56290090e+14, 2.56420940e+14, + 2.56581419e+14, 2.56642649e+14, 2.56426887e+14, 2.56360122e+14, + 2.56573424e+14, 2.56679138e+14, 2.56488767e+14, 2.56217444e+14, + 2.56353118e+14, 2.56640765e+14, 2.56809490e+14, 2.56933226e+14, + 2.56633538e+14, 2.56203430e+14, 2.56202958e+14, 2.56564020e+14, + 2.56816347e+14, 2.56709830e+14, 2.56557382e+14, 2.56573904e+14, + 2.56745541e+14, 2.56784430e+14, 2.56580054e+14, 2.56210130e+14, + 2.56271415e+14, 2.56821160e+14, 2.56703292e+14, 2.56169296e+14, + 2.56166549e+14, 2.56467777e+14, 2.56573240e+14, 2.56437594e+14, + 2.56253730e+14, 2.56176123e+14, 2.56351125e+14, 2.56569916e+14, + 2.56761101e+14, 2.56891411e+14, 2.56628312e+14, 2.56180062e+14, + 2.56063564e+14, 2.56189728e+14, 2.56609454e+14, 2.57263643e+14, + 2.57097673e+14, 2.56666761e+14, 2.56622585e+14, 2.56432378e+14, + 2.56386718e+14, 2.56734491e+14, 2.57042448e+14, 2.24471147e+14, + 1.27720853e+14 ]) density_data = np.load( 'ion_density_case_1.npy' ) print(repr(density_data)) -assert np.allclose(density_data, ref_density, rtol=0.01) +assert np.allclose(density_data, ref_density) diff --git a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json index 4d30f741bfb..0b38f78e6f9 100644 --- a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json @@ -1,27 +1,27 @@ { "lev=0": { - "rho_electrons": 0.004436602398896733, - "rho_he_ions": 0.0052003262664415285 + "rho_electrons": 0.00443743609125863, + "rho_he_ions": 0.005198801518328451 + }, + "neutrals": { + "particle_momentum_x": 1.404700281648976e-19, + "particle_momentum_y": 1.4028127127618884e-19, + "particle_momentum_z": 1.4090901433394346e-19, + "particle_position_x": 1120.7727446759352, + "particle_weight": 6.4588e+19 }, "he_ions": { - "particle_momentum_x": 2.7735512966774165e-19, - "particle_momentum_y": 2.7574111491186894e-19, - "particle_momentum_z": 3.620520352986572e-19, - "particle_position_x": 2201.236370518716, - "particle_weight": 17190734375000.002 + "particle_momentum_x": 2.770386771117138e-19, + "particle_momentum_y": 2.7568040242914223e-19, + "particle_momentum_z": 3.619756966185903e-19, + "particle_position_x": 2200.683185473434, + "particle_weight": 17185500000000.002 }, "electrons": { - "particle_momentum_x": 3.50212700099208e-20, - "particle_momentum_y": 3.5368926859820716e-20, - "particle_momentum_z": 1.2588108956625115e-19, - "particle_position_x": 2139.6498177543617, - "particle_weight": 14582968750000.002 - }, - "neutrals": { - "particle_momentum_x": 1.405588503355727e-19, - "particle_momentum_y": 1.408077689882847e-19, - "particle_momentum_z": 1.4024616940779626e-19, - "particle_position_x": 1121.2330379095083, - "particle_weight": 6.4588e+19 + "particle_momentum_x": 3.5129762363657864e-20, + "particle_momentum_y": 3.5431134517510143e-20, + "particle_momentum_z": 1.2592093336142964e-19, + "particle_position_x": 2142.0662480700303, + "particle_weight": 14593699218750.002 } -} \ No newline at end of file +} From 9b48df9e5973ad18e31c94842d7bc91691d6e8aa Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 5 Jan 2024 23:22:47 +0100 Subject: [PATCH 095/176] Doc: Clean Example TODOs (#4584) Moves TODOs to issues. Thanks everyone who added new user-facing docs! --- Docs/source/usage/examples.rst | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index 855faab392a..0492372b4e6 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -25,14 +25,6 @@ Plasma-Based Acceleration examples/pwfa/README.rst pwfa.rst -Coming soon: - -* LWFA: External injection in the boosted frame -* LWFA: Ionization injection in the lab frame using a LASY data file -* PWFA: External injection in the boosted frame -* PWFA: Self-injection in the lab frame -* MR case? - Laser-Plasma Interaction ------------------------ @@ -43,11 +35,6 @@ Laser-Plasma Interaction examples/laser_ion/README.rst examples/plasma_mirror/README.rst -Coming soon: - -* MVA (3D & RZ) -* MR for the planar example? - Particle Accelerator & Beam Physics ----------------------------------- @@ -58,12 +45,6 @@ Particle Accelerator & Beam Physics examples/gaussian_beam/README.rst examples/beam-beam_collision/README.rst -Coming soon: - -* Beam-Beam Collision -* Beam Transport or Injector -* Cathode/source - High Energy Astrophysical Plasma Physics ---------------------------------------- @@ -90,11 +71,6 @@ Nuclear Fusion TODO -Coming soon: - -* Microchannel -* Magnetically Confined Plasma with a Single Coil - Magnetic bottle: simple geometry with an external field - Fundamental Plasma Physics -------------------------- @@ -105,9 +81,6 @@ Fundamental Plasma Physics examples/langmuir/README.rst examples/capacitive_discharge/README.rst -Coming soon: - -* Expanding Sphere example .. _examples-hybrid-model: @@ -150,7 +123,7 @@ Manipulating fields via Python .. note:: - TODO: The section needs to be sorted into either science cases (above) or later sections (workflows and Python API details). + TODO: The section needs to be sorted into either science cases (above) or later sections (:ref:`workflows and Python API details `). An example of using Python to access the simulation charge density, solve the Poisson equation (using ``superLU``) and write the resulting electrostatic potential back to the simulation is given in the input file below. This example uses the ``fields.py`` module included in the ``pywarpx`` library. From 125ccef4317fe7c8e6a669049ffd32dc31b03264 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 5 Jan 2024 17:08:13 -0800 Subject: [PATCH 096/176] Read external fields from file in restarted simulation (#4547) * Read external fields from file also in restarted simulation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add automated test with restart * Add function to avoid duplication * Fix restart test * Move function outside of #ifdef * Fix restart test * Update Source/WarpX.H * Update Source/Initialization/WarpXInitData.cpp * Clarify documentation on external fields * Add docstring --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/usage/parameters.rst | 14 ++++- Examples/Tests/LoadExternalField/inputs_rz | 15 +++--- Regression/WarpX-tests.ini | 5 +- Source/Initialization/WarpXInitData.cpp | 63 ++++++++++++---------- Source/WarpX.H | 10 ++++ 5 files changed, 67 insertions(+), 40 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 81580f5c342..d3cf9b61aba 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1508,9 +1508,15 @@ Laser initialization External fields --------------- -Grid initialization +Applied to the grid ^^^^^^^^^^^^^^^^^^^ +The external fields defined with input parameters that start with ``warpx.B_ext_grid_init_`` or ``warpx.E_ext_grid_init_`` +are applied to the grid directly. In particular, these fields can be seen in the diagnostics that output the fields on the grid. + + - When using an **electromagnetic** field solver, these fields are applied to the grid at the beginning of the simulation, and serve as initial condition for the Maxwell solver. + - When using an **electrostatic** or **magnetostatic** field solver, these fields are added to the fields computed by the Poisson solver, at each timestep. + * ``warpx.B_ext_grid_init_style`` (string) optional This parameter determines the type of initialization for the external magnetic field. By default, the @@ -1597,6 +1603,9 @@ Grid initialization Applied to Particles ^^^^^^^^^^^^^^^^^^^^ +The external fields defined with input parameters that start with ``warpx.B_ext_particle_init_`` or ``warpx.E_ext_particle_init_`` +are applied to the particles directly, at each timestep. As a results, these fields **cannot** be seen in the diagnostics that output the fields on the grid. + * ``particles.E_ext_particle_init_style`` & ``particles.B_ext_particle_init_style`` (string) optional (default "none") These parameters determine the type of the external electric and magnetic fields respectively that are applied directly to the particles at every timestep. @@ -1657,6 +1666,9 @@ Applied to Particles Applied to Cold Relativistic Fluids ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The external fields defined with input parameters that start with ``warpx.B_ext_init_`` or ``warpx.E_ext_init_`` +are applied to the fluids directly, at each timestep. As a results, these fields **cannot** be seen in the diagnostics that output the fields on the grid. + * ``.E_ext_init_style`` & ``.B_ext_init_style`` (string) optional (default "none") These parameters determine the type of the external electric and magnetic fields respectively that are applied directly to the cold relativistic fluids at every timestep. diff --git a/Examples/Tests/LoadExternalField/inputs_rz b/Examples/Tests/LoadExternalField/inputs_rz index 050118d036f..2e22ca299ea 100644 --- a/Examples/Tests/LoadExternalField/inputs_rz +++ b/Examples/Tests/LoadExternalField/inputs_rz @@ -46,7 +46,7 @@ algo.particle_shape = 1 ################################# ############ PLASMA ############# ################################# -particles.species_names = proton #electron +particles.species_names = proton proton.injection_style = "SingleParticle" proton.single_particle_pos = 0.0 0.2 2.5 proton.single_particle_u = 9.506735958279367e-05 0.0 0.00013435537232359165 @@ -55,14 +55,11 @@ proton.do_not_deposit = 1 proton.mass = m_p proton.charge = q_e -#electron.injection_style = "SingleParticle" -#electron.single_particle_pos = 0.0 0.2 2.5 -#electron.single_particle_u = 0.0 0.0 0.0 -#electron.single_particle_weight = 1.0 -#electron.mass = 1.0 -#electron.charge = -q_e*1.0e-20 - # Diagnostics -diagnostics.diags_names = diag1 +diagnostics.diags_names = diag1 chk diag1.intervals = 300 diag1.diag_type = Full + +chk.intervals = 150 +chk.diag_type = Full +chk.format = checkpoint diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 974615ad725..dd1aef17877 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -2214,11 +2214,12 @@ analysisRoutine = Examples/Tests/resampling/analysis_leveling_thinning.py [LoadExternalFieldRZ] buildDir = . inputFile = Examples/Tests/LoadExternalField/inputs_rz -runtime_params = warpx.abort_on_warning_threshold=medium +runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZ_chk chk.file_min_digits=5 dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON -restartTest = 0 +restartTest = 1 +restartFileNum = 150 useMPI = 1 numprocs = 1 useOMP = 1 diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 3b36a6336bf..169453a6e99 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -728,6 +728,9 @@ void WarpX::PostRestart () { mypc->PostRestart(); + for (int lev = 0; lev <= maxLevel(); ++lev) { + LoadExternalFieldsFromFile(lev); + } } @@ -901,34 +904,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) } } - // Reading external fields from data file - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - -#if defined(WARPX_DIM_RZ) - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, - "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "r"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "t"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); -#else - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "x"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "y"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); -#endif - } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { -#if defined(WARPX_DIM_RZ) - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, - "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "r"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "t"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); -#else - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); - ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); -#endif - } + LoadExternalFieldsFromFile(lev); if (costs[lev]) { const auto iarr = costs[lev]->IndexArray(); @@ -1324,6 +1300,37 @@ void WarpX::CheckKnownIssues() #endif } +void +WarpX::LoadExternalFieldsFromFile (int const lev) +{ + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); +#else + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); +#endif + } + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); +#else + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); +#endif + } +} + #if defined(WARPX_USE_OPENPMD) && !defined(WARPX_DIM_1D_Z) && !defined(WARPX_DIM_XZ) void WarpX::ReadExternalFieldFromFile ( diff --git a/Source/WarpX.H b/Source/WarpX.H index 7a7c5ee89c1..72cb5b336ec 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1020,6 +1020,16 @@ public: char field, int lev, PatchType patch_type); + /** + * \brief Load field values from a user-specified openPMD file, + * for the fields Ex, Ey, Ez, Bx, By, Bz + */ + void LoadExternalFieldsFromFile (int lev); + + /** + * \brief Load field values from a user-specified openPMD file + * for a specific field (specified by `F_name`) + */ void ReadExternalFieldFromFile ( std::string read_fields_from_path, amrex::MultiFab* mf, std::string F_name, std::string F_component); From 0b147c7add89470ef47a61d3ddc3491d8194d400 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 6 Jan 2024 21:09:47 +0100 Subject: [PATCH 097/176] Release 24.01 (#4586) * AMReX: 24.01 * pyAMReX: 24.01 * WarpX: 24.01 --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 29a434dc579..830c33e9a91 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 && cd - + cd ../amrex && git checkout --detach 24.01 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index e6aa3c8174f..7a0b28c9f86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 23.12) +project(WarpX VERSION 24.01) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index f21dea15f18..48a02c5d216 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'23.12' +version = u'24.01' # The full version, including alpha/beta/rc tags. -release = u'23.12' +release = u'24.01' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index 47028642cce..f82e6563a31 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '23.12', + version = '24.01', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 0566184848b..68620e45b62 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 +branch = 24.01 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index dd1aef17877..da4da53e6a4 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 +branch = 24.01 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 886ee9c951f..911d70ba215 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 23.12 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.01 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "75571e2dcbf2417529c5ed8e24113580e8e1f3f1" +set(WarpX_amrex_branch "24.01" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 2ad63917965..7db3863d6ee 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 23.12 CONFIG REQUIRED) + find_package(pyAMReX 24.01 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "eb24d03fac522d36fb27d20c6e026d8b59dfa908" +set(WarpX_pyamrex_branch "24.01" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index deb4939da51..252e1cb89ae 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 75571e2dcbf2417529c5ed8e24113580e8e1f3f1 && cd - +cd amrex && git checkout --detach 24.01 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index 5228b7b3521..d3efeaaacd5 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '23.12', + version = '24.01', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From e944c7c461e9bbe68fc43a97c981d31d283af9c7 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Mon, 8 Jan 2024 17:27:11 +0100 Subject: [PATCH 098/176] Clang-tidy: enforce clang-tidy_readability_make_member_function_const check (#4510) * enforce clang-tidy_readability_make_member_function_const check * fix bug * address additional issues found with clang-tidy * Fix more issues. --------- Co-authored-by: Weiqun Zhang --- .clang-tidy | 1 - Source/AcceleratorLattice/AcceleratorLattice.cpp | 5 +++-- Source/AcceleratorLattice/LatticeElementFinder.H | 6 +++--- Source/AcceleratorLattice/LatticeElementFinder.cpp | 4 ++-- Source/BoundaryConditions/WarpXFieldBoundaries.cpp | 2 +- Source/Diagnostics/BTD_Plotfile_Header_Impl.H | 10 +++++----- Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp | 2 +- Source/Diagnostics/BTDiagnostics.H | 8 ++++---- Source/Diagnostics/BTDiagnostics.cpp | 5 +++-- Source/Diagnostics/Diagnostics.H | 2 +- Source/Diagnostics/MultiDiagnostics.H | 2 +- Source/Diagnostics/ReducedDiags/ReducedDiags.H | 2 +- Source/Diagnostics/ReducedDiags/ReducedDiags.cpp | 2 +- Source/Diagnostics/WarpXOpenPMD.H | 2 +- .../HybridPICModel/HybridPICModel.H | 2 +- .../HybridPICModel/HybridPICModel.cpp | 2 +- Source/Initialization/InjectorMomentum.cpp | 2 +- .../Collision/BackgroundMCC/BackgroundMCCCollision.H | 2 +- .../Collision/BackgroundMCC/BackgroundMCCCollision.cpp | 2 +- Source/Particles/Collision/CollisionBase.H | 2 +- Source/Particles/ParticleBoundaries.H | 2 +- Source/Particles/ParticleBoundaries.cpp | 2 +- Source/Particles/PhysicalParticleContainer.H | 4 ++-- Source/Particles/PhysicalParticleContainer.cpp | 4 ++-- Source/WarpX.H | 2 +- Source/ablastr/utils/timer/Timer.H | 4 ++-- Source/ablastr/utils/timer/Timer.cpp | 4 ++-- 27 files changed, 44 insertions(+), 43 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 4e1463affba..21f0343519c 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -41,7 +41,6 @@ Checks: ' -readability-implicit-bool-conversion, -readability-isolate-declaration, -readability-magic-numbers, - -readability-make-member-function-const, -readability-named-parameter, -readability-uppercase-literal-suffix ' diff --git a/Source/AcceleratorLattice/AcceleratorLattice.cpp b/Source/AcceleratorLattice/AcceleratorLattice.cpp index e05167c261d..8dc0983d622 100644 --- a/Source/AcceleratorLattice/AcceleratorLattice.cpp +++ b/Source/AcceleratorLattice/AcceleratorLattice.cpp @@ -88,8 +88,9 @@ AcceleratorLattice::InitElementFinder (int const lev, amrex::BoxArray const & ba } void -AcceleratorLattice::UpdateElementFinder (int const lev) -{ +AcceleratorLattice::UpdateElementFinder (int const lev) // NOLINT(readability-make-member-function-const) +{ // Techniquely clang-tidy is correct because + // m_element_finder is unique_ptr, not const*. if (m_lattice_defined) { for (amrex::MFIter mfi(*m_element_finder); mfi.isValid(); ++mfi) { diff --git a/Source/AcceleratorLattice/LatticeElementFinder.H b/Source/AcceleratorLattice/LatticeElementFinder.H index b7ca556d9ee..6773ed56a65 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.H +++ b/Source/AcceleratorLattice/LatticeElementFinder.H @@ -72,8 +72,8 @@ struct LatticeElementFinder * @param[in] a_offset particle index offset needed to access particle info * @param[in] accelerator_lattice a reference to the accelerator lattice at the refinement level */ - LatticeElementFinderDevice GetFinderDeviceInstance (WarpXParIter const& a_pti, int a_offset, - AcceleratorLattice const& accelerator_lattice); + [[nodiscard]] LatticeElementFinderDevice GetFinderDeviceInstance ( + WarpXParIter const& a_pti, int a_offset, AcceleratorLattice const& accelerator_lattice) const; /* The index lookup tables for each lattice element type */ amrex::Gpu::DeviceVector d_quad_indices; @@ -89,7 +89,7 @@ struct LatticeElementFinder */ void setup_lattice_indices (amrex::Gpu::DeviceVector const & zs, amrex::Gpu::DeviceVector const & ze, - amrex::Gpu::DeviceVector & indices); + amrex::Gpu::DeviceVector & indices) const; }; /** diff --git a/Source/AcceleratorLattice/LatticeElementFinder.cpp b/Source/AcceleratorLattice/LatticeElementFinder.cpp index bb2fe9c34a5..ba2e51715fb 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.cpp +++ b/Source/AcceleratorLattice/LatticeElementFinder.cpp @@ -78,7 +78,7 @@ LatticeElementFinder::UpdateIndices (int const lev, amrex::MFIter const& a_mfi, LatticeElementFinderDevice LatticeElementFinder::GetFinderDeviceInstance (WarpXParIter const& a_pti, int const a_offset, - AcceleratorLattice const& accelerator_lattice) + AcceleratorLattice const& accelerator_lattice) const { LatticeElementFinderDevice result; result.InitLatticeElementFinderDevice(a_pti, a_offset, accelerator_lattice, *this); @@ -124,7 +124,7 @@ LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& void LatticeElementFinder::setup_lattice_indices (amrex::Gpu::DeviceVector const & zs, amrex::Gpu::DeviceVector const & ze, - amrex::Gpu::DeviceVector & indices) + amrex::Gpu::DeviceVector & indices) const { using namespace amrex::literals; diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 0864c00067b..af7a385b729 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -119,7 +119,7 @@ void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, #ifdef WARPX_DIM_RZ // Applies the boundary conditions that are specific to the axis when in RZ. void -WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev) +WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev) const { const amrex::IntVect ngE = get_ng_fieldgather(); diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H index 209113dedeb..b87c54be9f0 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H @@ -171,11 +171,11 @@ class BTDMultiFabHeaderImpl void WriteMultiFabHeader (); /** Returns size, m_ba_size, of the Box Array, m_ba.*/ - int ba_size () {return m_ba_size;} + [[nodiscard]] int ba_size () const {return m_ba_size;} /** Returns box corresponding to the ith box in the BoxArray, m_ba. * \param[in] ibox index of the box in the BoxArray. */ - amrex::Box ba_box (int ibox) {return m_ba[ibox]; } + [[nodiscard]] amrex::Box ba_box (int ibox) const {return m_ba[ibox]; } /** Returns prefix of the ith-fab on disk, i.e., ith fab of the MultiFab data. * \param[in] ifab index of the ith fab in the MultiFab data. */ @@ -316,9 +316,9 @@ public: /** Reads the particle header file at m_Header_path and stores its data*/ void ReadHeader (); /** Writes the meta-data of particle box array in header file, with path, m_Header_path*/ - void WriteHeader (); + void WriteHeader () const; /** Returns the size of the box array, m_ba_size */ - int ba_size () {return m_ba_size; } + [[nodiscard]] int ba_size () const {return m_ba_size; } /** Increases Box array size, m_ba_size, by add_size * \param[in] add_size */ @@ -326,7 +326,7 @@ public: /** Returns box corresponding to the ith box in the BoxArray, m_ba. * \param[in] ibox index of the box in the BoxArray. */ - amrex::Box ba_box (int ibox) {return m_ba[ibox]; } + [[nodiscard]] amrex::Box ba_box (int ibox) const {return m_ba[ibox]; } /** Resize boxArray, m_ba, to size, m_ba_size. */ void ResizeBoxArray () { m_ba.resize(m_ba_size); } /** Set Box indices of the ith-box in Box Array, m_ba, to the new Box, ba_box. diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp index 8486a487f40..da4a3a79547 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp @@ -516,7 +516,7 @@ BTDParticleDataHeaderImpl::ReadHeader () } void -BTDParticleDataHeaderImpl::WriteHeader () +BTDParticleDataHeaderImpl::WriteHeader () const { if (amrex::FileExists(m_Header_path)) { amrex::FileSystem::Remove(m_Header_path); diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index 8940a89711a..ab894da69c2 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -275,7 +275,7 @@ private: * \param[in] t_lab lab-frame time of the snapshot * \param[in] t_boost boosted-frame time at level, lev */ - amrex::Real UpdateCurrentZBoostCoordinate(amrex::Real t_lab, amrex::Real t_boost) + [[nodiscard]] amrex::Real UpdateCurrentZBoostCoordinate(amrex::Real t_lab, amrex::Real t_boost) const { const amrex::Real current_z_boost = (t_lab / m_gamma_boost - t_boost) * PhysConst::c / m_beta_boost; return current_z_boost; @@ -284,7 +284,7 @@ private: * \param[in] t_lab lab-frame time of the snapshot * \param[in] t_boost boosted-frame time at level, lev */ - amrex::Real UpdateCurrentZLabCoordinate(amrex::Real t_lab, amrex::Real t_boost) + [[nodiscard]] amrex::Real UpdateCurrentZLabCoordinate(amrex::Real t_lab, amrex::Real t_boost) const { const amrex::Real current_z_lab = (t_lab - t_boost / m_gamma_boost ) * PhysConst::c / m_beta_boost; return current_z_lab; @@ -294,13 +294,13 @@ private: * \param[in] ref_ratio refinement ratio in the z-direction at level, lev-1. * The ref-ratio in the z-direction for single-level diagnostics is 1. */ - amrex::Real dz_lab (amrex::Real dt, amrex::Real ref_ratio); + [[nodiscard]] amrex::Real dz_lab (amrex::Real dt, amrex::Real ref_ratio) const; /** Compute k-index corresponding to current lab-frame z co-ordinate (m_current_z_lab) * for the ith buffer i_buffer, and at level, lev. * \param[in] i_buffer snapshot index * \param[in] lev mesh-refinement level at which the lab-frame z-index is computed */ - int k_index_zlab (int i_buffer, int lev); + [[nodiscard]] int k_index_zlab (int i_buffer, int lev) const; /** whether field buffer is full * \param[in] i_buffer buffer id for which the buffer size is checked. * returns bool = true is buffer is full, that is, diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index fd939b0370b..0e517e8190c 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -863,13 +863,14 @@ BTDiagnostics::PrepareFieldDataForOutput () amrex::Real -BTDiagnostics::dz_lab (amrex::Real dt, amrex::Real ref_ratio){ +BTDiagnostics::dz_lab (amrex::Real dt, amrex::Real ref_ratio) const +{ return PhysConst::c * dt * 1._rt/m_beta_boost * 1._rt/m_gamma_boost * 1._rt/ref_ratio; } int -BTDiagnostics::k_index_zlab (int i_buffer, int lev) +BTDiagnostics::k_index_zlab (int i_buffer, int lev) const { auto & warpx = WarpX::GetInstance(); const amrex::Real prob_domain_zmin_lab = m_snapshot_domain_lab[i_buffer].lo( m_moving_window_dir ); diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 21d5ff699d0..53ce319d747 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -114,7 +114,7 @@ public: /** Whether the last timestep is always dumped */ [[nodiscard]] bool DoDumpLastTimestep () const {return m_dump_last_timestep;} /** Returns the number of snapshots used in BTD. For Full-Diagnostics, the value is 1*/ - int getnumbuffers() {return m_num_buffers;} + [[nodiscard]] int getnumbuffers() const {return m_num_buffers;} /** Time in lab-frame associated with the ith snapshot * \param[in] i_buffer index of the buffer */ diff --git a/Source/Diagnostics/MultiDiagnostics.H b/Source/Diagnostics/MultiDiagnostics.H index f9907b8bdc6..d220396ed12 100644 --- a/Source/Diagnostics/MultiDiagnostics.H +++ b/Source/Diagnostics/MultiDiagnostics.H @@ -38,7 +38,7 @@ public: /** Start a new iteration, i.e., dump has not been done yet. */ void NewIteration (); Diagnostics& GetDiag(int idiag) {return *alldiags[idiag]; } - int GetTotalDiags() {return ndiags;} + [[nodiscard]] int GetTotalDiags() const {return ndiags;} DiagTypes diagstypes(int idiag) {return diags_types[idiag];} private: /** Vector of pointers to all diagnostics */ diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.H b/Source/Diagnostics/ReducedDiags/ReducedDiags.H index 01847a1ff12..09aa85586be 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.H +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.H @@ -94,7 +94,7 @@ public: * This function queries deprecated input parameters and aborts * the run if one of them is specified. */ - void BackwardCompatibility (); + void BackwardCompatibility () const; }; diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp index b4f617a8520..ae6dae25a71 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp @@ -86,7 +86,7 @@ void ReducedDiags::LoadBalance () // load balancing operations } -void ReducedDiags::BackwardCompatibility () +void ReducedDiags::BackwardCompatibility () const { const amrex::ParmParse pp_rd_name(m_rd_name); std::vector backward_strings; diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index bf83c1ebb0b..e3b7b893d0a 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -44,7 +44,7 @@ public: using ParticleIter = typename amrex::ParIter<0, 0, PIdx::nattribs, 0, amrex::PinnedArenaAllocator>; WarpXParticleCounter (ParticleContainer* pc); - unsigned long GetTotalNumParticles () {return m_Total;} + [[nodiscard]] unsigned long GetTotalNumParticles () const {return m_Total;} std::vector m_ParticleOffsetAtRank; std::vector m_ParticleSizeAtRank; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index c7ea9b46458..750a5e06933 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -126,7 +126,7 @@ public: */ void FillElectronPressureMF ( std::unique_ptr const& Pe_field, - amrex::MultiFab* const& rho_field ); + amrex::MultiFab* const& rho_field ) const; // Declare variables to hold hybrid-PIC model parameters /** Number of substeps to take when evolving B */ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 5a6f0018b2a..2cc6a8ec437 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -508,7 +508,7 @@ void HybridPICModel::CalculateElectronPressure(const int lev, DtType a_dt_type) void HybridPICModel::FillElectronPressureMF ( std::unique_ptr const& Pe_field, - amrex::MultiFab* const& rho_field ) + amrex::MultiFab* const& rho_field ) const { const auto n0_ref = m_n0_ref; const auto elec_temp = m_elec_temp; diff --git a/Source/Initialization/InjectorMomentum.cpp b/Source/Initialization/InjectorMomentum.cpp index 35513260827..4642d3a0cc6 100644 --- a/Source/Initialization/InjectorMomentum.cpp +++ b/Source/Initialization/InjectorMomentum.cpp @@ -9,7 +9,7 @@ using namespace amrex; -void InjectorMomentum::clear () +void InjectorMomentum::clear () // NOLINT(readability-make-member-function-const) { switch (type) { diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H index 1db3922d178..1de6b999e0b 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H @@ -32,7 +32,7 @@ public: BackgroundMCCCollision ( BackgroundMCCCollision&& ) = delete; BackgroundMCCCollision& operator= ( BackgroundMCCCollision&& ) = delete; - amrex::ParticleReal get_nu_max (amrex::Vector const& mcc_processes); + [[nodiscard]] amrex::ParticleReal get_nu_max (amrex::Vector const& mcc_processes) const; /** Perform the collisions * diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index 672c6456ec7..4cb16f6fa50 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -162,7 +162,7 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name * ranges from 1e-4 to 5000 eV in 0.2 eV increments */ amrex::ParticleReal -BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processes) +BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processes) const { using namespace amrex::literals; amrex::ParticleReal nu, nu_max = 0.0; diff --git a/Source/Particles/Collision/CollisionBase.H b/Source/Particles/Collision/CollisionBase.H index 272cc056b1a..db79eaf1a01 100644 --- a/Source/Particles/Collision/CollisionBase.H +++ b/Source/Particles/Collision/CollisionBase.H @@ -29,7 +29,7 @@ public: virtual ~CollisionBase() = default; - int get_ndt() {return m_ndt;} + [[nodiscard]] int get_ndt() const {return m_ndt;} protected: diff --git a/Source/Particles/ParticleBoundaries.H b/Source/Particles/ParticleBoundaries.H index 1f4fb0372eb..78b090056b1 100644 --- a/Source/Particles/ParticleBoundaries.H +++ b/Source/Particles/ParticleBoundaries.H @@ -27,7 +27,7 @@ struct ParticleBoundaries void SetBoundsY (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); void SetBoundsZ (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); - bool CheckAll (ParticleBoundaryType bc); + [[nodiscard]] bool CheckAll (ParticleBoundaryType bc) const; void BuildReflectionModelParsers (); diff --git a/Source/Particles/ParticleBoundaries.cpp b/Source/Particles/ParticleBoundaries.cpp index a6e80717e81..5b51fa3fd25 100644 --- a/Source/Particles/ParticleBoundaries.cpp +++ b/Source/Particles/ParticleBoundaries.cpp @@ -54,7 +54,7 @@ ParticleBoundaries::SetBoundsZ (ParticleBoundaryType bc_lo, ParticleBoundaryType } bool -ParticleBoundaries::CheckAll (ParticleBoundaryType bc) +ParticleBoundaries::CheckAll (ParticleBoundaryType bc) const { return (data.xmin_bc == bc && data.xmax_bc == bc #ifdef WARPX_DIM_3D diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 1ec3473e81b..a12ae75f629 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -224,7 +224,7 @@ public: void MapParticletoBoostedFrame (amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz, - amrex::Real t_lab = 0.); + amrex::Real t_lab = 0.) const; void AddGaussianBeam ( PlasmaInjector const& plasma_injector, @@ -253,7 +253,7 @@ public: amrex::Gpu::HostVector& particle_uy, amrex::Gpu::HostVector& particle_uz, amrex::Gpu::HostVector& particle_w, - amrex::Real t_lab= 0.); + amrex::Real t_lab= 0.) const; /** * \brief Default initialize runtime attributes in a tile. This routine does not initialize the diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index cfd0074d616..0fc718c7355 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -471,7 +471,7 @@ void PhysicalParticleContainer::InitData () } void PhysicalParticleContainer::MapParticletoBoostedFrame ( - ParticleReal& x, ParticleReal& y, ParticleReal& z, ParticleReal& ux, ParticleReal& uy, ParticleReal& uz, Real t_lab) + ParticleReal& x, ParticleReal& y, ParticleReal& z, ParticleReal& ux, ParticleReal& uy, ParticleReal& uz, Real t_lab) const { // Map the particles from the lab frame to the boosted frame. // This boosts the particle to the lab frame and calculates @@ -810,7 +810,7 @@ PhysicalParticleContainer::CheckAndAddParticle ( Gpu::HostVector& particle_uy, Gpu::HostVector& particle_uz, Gpu::HostVector& particle_w, - Real t_lab) + Real t_lab) const { if (WarpX::gamma_boost > 1.) { MapParticletoBoostedFrame(x, y, z, ux, uy, uz, t_lab); diff --git a/Source/WarpX.H b/Source/WarpX.H index 72cb5b336ec..c9aa24d3973 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -727,7 +727,7 @@ public: #ifdef WARPX_DIM_RZ // Applies the boundary conditions that are specific to the axis when in RZ. - void ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev); + void ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev) const; #endif /** diff --git a/Source/ablastr/utils/timer/Timer.H b/Source/ablastr/utils/timer/Timer.H index efbb7d6a2bb..5df37493d61 100644 --- a/Source/ablastr/utils/timer/Timer.H +++ b/Source/ablastr/utils/timer/Timer.H @@ -44,7 +44,7 @@ namespace ablastr::utils::timer * * @return the duration */ - double get_duration () noexcept; + [[nodiscard]] double get_duration () const noexcept; /** @@ -53,7 +53,7 @@ namespace ablastr::utils::timer * * @return the maximum duration across all the MPI ranks */ - double get_global_duration (); + [[nodiscard]] double get_global_duration () const; private: diff --git a/Source/ablastr/utils/timer/Timer.cpp b/Source/ablastr/utils/timer/Timer.cpp index 5682a07c290..096c079fa2a 100644 --- a/Source/ablastr/utils/timer/Timer.cpp +++ b/Source/ablastr/utils/timer/Timer.cpp @@ -24,13 +24,13 @@ Timer::record_stop_time() noexcept } double -Timer::get_duration () noexcept +Timer::get_duration () const noexcept { return m_stop_time - m_start_time; } double -Timer::get_global_duration () +Timer::get_global_duration () const { auto duration = this->get_duration(); amrex::ParallelDescriptor::ReduceRealMax( From 42e8c562345e0b67ee8026601b96d95879cf1aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxence=20Th=C3=A9venet?= Date: Mon, 8 Jan 2024 17:28:22 +0100 Subject: [PATCH 099/176] Implement correction to ADK for high fields (#4505) * Implement correction to ADK for high fields * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix Const Type and Typo * add documentation --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- Docs/source/usage/parameters.rst | 5 +++++ .../Particles/ElementaryProcess/Ionization.H | 13 +++++++++++- .../ElementaryProcess/Ionization.cpp | 4 ++++ .../Particles/PhysicalParticleContainer.cpp | 21 ++++++++++++++++++- Source/Particles/WarpXParticleContainer.H | 3 +++ .../Utils/Physics/IonizationEnergiesTable.H | 7 +++++++ 6 files changed, 51 insertions(+), 2 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index d3cf9b61aba..0597b21c393 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1167,6 +1167,11 @@ Particle initialization * ``.do_field_ionization`` (`0` or `1`) optional (default `0`) Do field ionization for this species (using the ADK theory). +* ``.do_adk_correction`` (`0` or `1`) optional (default `0`) + Whether to apply the correction to the ADK theory proposed by Zhang, Lan and Lu in `Q. Zhang et al. (Phys. Rev. A 90, 043410, 2014) `__. + If so, the probability of ionization is modified using an empirical model that should be more accurate in the regime of high electric fields. + Currently, this is only implemented for Hydrogen, although Argon is also available in the same reference. + * ``.physical_element`` (`string`) Only read if `do_field_ionization = 1`. Symbol of chemical element for this species. Example: for Helium, use ``physical_element = He``. diff --git a/Source/Particles/ElementaryProcess/Ionization.H b/Source/Particles/ElementaryProcess/Ionization.H index 573491645ea..61aeb19aea1 100644 --- a/Source/Particles/ElementaryProcess/Ionization.H +++ b/Source/Particles/ElementaryProcess/Ionization.H @@ -33,9 +33,11 @@ struct IonizationFilterFunc const amrex::Real* AMREX_RESTRICT m_adk_prefactor; const amrex::Real* AMREX_RESTRICT m_adk_exp_prefactor; const amrex::Real* AMREX_RESTRICT m_adk_power; + const amrex::Real* AMREX_RESTRICT m_adk_correction_factors; int comp; int m_atomic_number; + int m_do_adk_correction = 0; GetParticlePosition m_get_position; GetExternalEBField m_get_externalEB; @@ -82,8 +84,10 @@ struct IonizationFilterFunc const amrex::Real* AMREX_RESTRICT a_adk_prefactor, const amrex::Real* AMREX_RESTRICT a_adk_exp_prefactor, const amrex::Real* AMREX_RESTRICT a_adk_power, + const amrex::Real* AMREX_RESTRICT a_adk_correction_factors, int a_comp, int a_atomic_number, + int a_do_adk_correction, int a_offset = 0) noexcept; template @@ -133,9 +137,16 @@ struct IonizationFilterFunc ); // Compute probability of ionization p - const amrex::Real w_dtau = (E <= 0._rt) ? 0._rt : 1._rt/ ga * m_adk_prefactor[ion_lev] * + amrex::Real w_dtau = (E <= 0._rt) ? 0._rt : 1._rt/ ga * m_adk_prefactor[ion_lev] * std::pow(E, m_adk_power[ion_lev]) * std::exp( m_adk_exp_prefactor[ion_lev]/E ); + // if requested, do Zhang's correction of ADK + if (m_do_adk_correction) { + const amrex::Real r = E / m_adk_correction_factors[3]; + w_dtau *= std::exp(m_adk_correction_factors[0]*r*r+m_adk_correction_factors[1]*r+ + m_adk_correction_factors[2]); + } + const amrex::Real p = 1._rt - std::exp( - w_dtau ); const amrex::Real random_draw = amrex::Random(engine); diff --git a/Source/Particles/ElementaryProcess/Ionization.cpp b/Source/Particles/ElementaryProcess/Ionization.cpp index c3681a30cad..b7b91e4d4e3 100644 --- a/Source/Particles/ElementaryProcess/Ionization.cpp +++ b/Source/Particles/ElementaryProcess/Ionization.cpp @@ -30,15 +30,19 @@ IonizationFilterFunc::IonizationFilterFunc (const WarpXParIter& a_pti, int lev, const amrex::Real* const AMREX_RESTRICT a_adk_prefactor, const amrex::Real* const AMREX_RESTRICT a_adk_exp_prefactor, const amrex::Real* const AMREX_RESTRICT a_adk_power, + const amrex::Real* const AMREX_RESTRICT a_adk_correction_factors, int a_comp, int a_atomic_number, + int a_do_adk_correction, int a_offset) noexcept: m_ionization_energies{a_ionization_energies}, m_adk_prefactor{a_adk_prefactor}, m_adk_exp_prefactor{a_adk_exp_prefactor}, m_adk_power{a_adk_power}, + m_adk_correction_factors{a_adk_correction_factors}, comp{a_comp}, m_atomic_number{a_atomic_number}, + m_do_adk_correction{a_do_adk_correction}, m_Ex_external_particle{E_external_particle[0]}, m_Ey_external_particle{E_external_particle[1]}, m_Ez_external_particle{E_external_particle[2]}, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 0fc718c7355..0d9180c090e 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -3174,10 +3174,15 @@ PhysicalParticleContainer::InitIonizationModule () "overriding user value and setting charge = q_e."); charge = PhysConst::q_e; } + utils::parser::queryWithParser(pp_species_name, "do_adk_correction", do_adk_correction); + utils::parser::queryWithParser( pp_species_name, "ionization_initial_level", ionization_initial_level); pp_species_name.get("ionization_product_species", ionization_product_name); pp_species_name.get("physical_element", physical_element); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + physical_element == "H" || !do_adk_correction, + "Correction to ADK by Zhang et al., PRA 90, 043410 (2014) only works with Hydrogen"); // Add runtime integer component for ionization level AddIntComp("ionizationLevel"); // Get atomic number and ionization energies from file @@ -3212,6 +3217,18 @@ PhysicalParticleContainer::InitIonizationModule () h_ionization_energies.begin(), h_ionization_energies.end(), ionization_energies.begin()); + adk_correction_factors.resize(4); + if (do_adk_correction) { + Vector h_correction_factors(4); + constexpr int offset_corr = 0; // hard-coded: only Hydrogen + for(int i=0; i<4; i++){ + h_correction_factors[i] = table_correction_factors[i+offset_corr]; + } + Gpu::copyAsync(Gpu::hostToDevice, + h_correction_factors.begin(), h_correction_factors.end(), + adk_correction_factors.begin()); + } + Real const* AMREX_RESTRICT p_ionization_energies = ionization_energies.data(); Real * AMREX_RESTRICT p_adk_power = adk_power.data(); Real * AMREX_RESTRICT p_adk_prefactor = adk_prefactor.data(); @@ -3249,8 +3266,10 @@ PhysicalParticleContainer::getIonizationFunc (const WarpXParIter& pti, adk_prefactor.dataPtr(), adk_exp_prefactor.dataPtr(), adk_power.dataPtr(), + adk_correction_factors.dataPtr(), particle_icomps["ionizationLevel"], - ion_atomic_number}; + ion_atomic_number, + do_adk_correction}; } PlasmaInjector* PhysicalParticleContainer::GetPlasmaInjector (int i) diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index e791ac25d22..33aa71d1c7d 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -445,6 +445,7 @@ protected: int do_continuous_injection = 0; int do_field_ionization = 0; + int do_adk_correction = 0; int ionization_product; std::string ionization_product_name; int ion_atomic_number; @@ -453,6 +454,8 @@ protected: amrex::Gpu::DeviceVector adk_power; amrex::Gpu::DeviceVector adk_prefactor; amrex::Gpu::DeviceVector adk_exp_prefactor; + /** for correction in Zhang et al., PRA 90, 043410 (2014). a1, a2, a3, Ecrit. */ + amrex::Gpu::DeviceVector adk_correction_factors; std::string physical_element; int do_resampling = 0; diff --git a/Source/Utils/Physics/IonizationEnergiesTable.H b/Source/Utils/Physics/IonizationEnergiesTable.H index f956e308dbe..b7c82e88247 100644 --- a/Source/Utils/Physics/IonizationEnergiesTable.H +++ b/Source/Utils/Physics/IonizationEnergiesTable.H @@ -1963,4 +1963,11 @@ namespace utils::physics } +constexpr int correction_factors_length = 4; +// a1, a2, a3, Ecrit from Zhang et al., PRA 90, 043410 (2014). +constexpr amrex::Real table_correction_factors[correction_factors_length]{ + // H + amrex::Real(0.11714), amrex::Real(-0.90933), amrex::Real(-0.06034), amrex::Real(32125000000.) +}; + #endif // #ifndef WARPX_UTILS_PHYSICS_IONIZATION_TABLE_H_ From 3b7bcd89f09986ea02f9dc909f23f20e7803bbde Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Mon, 8 Jan 2024 09:26:15 -0800 Subject: [PATCH 100/176] fix doc for coarsening (#4588) --- Docs/source/usage/parameters.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 0597b21c393..674f86143e3 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2602,8 +2602,8 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a Only works with ``.format = plotfile``. * ``.coarsening_ratio`` (list of `int`) optional (default `1 1 1`) - Reduce size of the field output by this ratio in each dimension. - (This is done by averaging the field over 1 or 2 points along each direction, depending on the staggering). + Reduce size of the selected diagnostic fields output by this ratio in each dimension. + (For a ratio of N, this is done by averaging the fields over N or (N+1) points depending on the staggering). If ``blocking_factor`` and ``max_grid_size`` are used for the domain decomposition, as detailed in the :ref:`domain decomposition ` section, ``coarsening_ratio`` should be an integer divisor of ``blocking_factor``. If ``warpx.numprocs`` is used instead, the total number of cells in a given From 69e9262c4ab8aabc29c8241cf22138b3e64bbb31 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 8 Jan 2024 14:35:43 -0800 Subject: [PATCH 101/176] AMReX/pyAMReX/PICSAR: Weekly Update (#4590) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 830c33e9a91..b50989f2f5c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.01 && cd - + cd ../amrex && git checkout --detach f1ec8df75c562d2a4822cea84d284cf8e72c2e14 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 68620e45b62..2a98d7b3736 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 24.01 +branch = f1ec8df75c562d2a4822cea84d284cf8e72c2e14 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index da4da53e6a4..f5ef38e881b 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 24.01 +branch = f1ec8df75c562d2a4822cea84d284cf8e72c2e14 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 911d70ba215..316b68410c0 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.01" +set(WarpX_amrex_branch "f1ec8df75c562d2a4822cea84d284cf8e72c2e14" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 7db3863d6ee..175d3cab79e 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.01" +set(WarpX_pyamrex_branch "0283db4c8825b5e094b9184fd42a267ca843c61c" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 252e1cb89ae..5dd813829c7 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.01 && cd - +cd amrex && git checkout --detach f1ec8df75c562d2a4822cea84d284cf8e72c2e14 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 82f7f1ee492f1d92e0d4e3ab142b9348454474b2 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Wed, 10 Jan 2024 09:07:36 -0800 Subject: [PATCH 102/176] Fix plasma injection from file for parallel runs (#4597) The use of amrex::ParallelDescriptor::ReduceBoolOr was incorrect. The logic is also simplified, even though it may not the most efficient way. --- Source/Initialization/PlasmaInjector.cpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index 3b2ad76513a..008709abe48 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -501,18 +501,15 @@ void PlasmaInjector::setupExternalFile (amrex::ParmParse const& pp_species) } // IOProcessor // Broadcast charge and mass to non-IO processors if read in from the file - if (!charge_is_specified && !species_is_specified) { - // Use ReduceBoolOr since Bcast(bool) doesn't compile - amrex::ParallelDescriptor::ReduceBoolOr(charge_from_source, amrex::ParallelDescriptor::IOProcessorNumber()); - if (charge_from_source) { - amrex::ParallelDescriptor::Bcast(&charge, 1, amrex::ParallelDescriptor::IOProcessorNumber()); - } + std::array flags{charge_from_source, mass_from_source}; + amrex::ParallelDescriptor::Bcast(flags.data(), flags.size(), amrex::ParallelDescriptor::IOProcessorNumber()); + charge_from_source = flags[0]; + mass_from_source = flags[1]; + if (charge_from_source) { + amrex::ParallelDescriptor::Bcast(&charge, 1, amrex::ParallelDescriptor::IOProcessorNumber()); } - if (!mass_is_specified && !species_is_specified) { - amrex::ParallelDescriptor::ReduceBoolOr(mass_from_source, amrex::ParallelDescriptor::IOProcessorNumber()); - if (mass_from_source) { - amrex::ParallelDescriptor::Bcast(&mass, 1, amrex::ParallelDescriptor::IOProcessorNumber()); - } + if (mass_from_source) { + amrex::ParallelDescriptor::Bcast(&mass, 1, amrex::ParallelDescriptor::IOProcessorNumber()); } #else WARPX_ABORT_WITH_MESSAGE( From 12a265c14efb881eebda29cd3ece9a2c688dd1e4 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 15 Jan 2024 07:53:34 -0800 Subject: [PATCH 103/176] Move input of galerkin_scheme upward to before the implicit check of it (#4607) --- Source/WarpX.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index f3008e6157d..21a4a2a5234 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1210,6 +1210,11 @@ WarpX::ReadParameters () // Use same shape factors in all directions, for gathering if (field_gathering_algo == GatheringAlgo::MomentumConserving) { galerkin_interpolation = false; } + { + const ParmParse pp_interpolation("interpolation"); + pp_interpolation.query("galerkin_scheme",galerkin_interpolation); + } + // With the PSATD solver, momentum-conserving field gathering // combined with mesh refinement does not seem to work correctly // TODO Needs debugging @@ -1377,12 +1382,6 @@ WarpX::ReadParameters () } - { - const ParmParse pp_interpolation("interpolation"); - - pp_interpolation.query("galerkin_scheme",galerkin_interpolation); - } - { const ParmParse pp_warpx("warpx"); From a3fd1427d3d46d766464a4d1b28623a49ee81d72 Mon Sep 17 00:00:00 2001 From: Edward Basso Date: Mon, 15 Jan 2024 18:29:53 -0800 Subject: [PATCH 104/176] Docs: spruce up parameters.rst (#4609) * Spruce up parameters.rst * Conform more to latest version of development * Fix author name --- Docs/source/refs.bib | 139 ++++++++++++++++++++++--------- Docs/source/usage/parameters.rst | 61 +++++++------- 2 files changed, 130 insertions(+), 70 deletions(-) diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index d94938775c6..895a6c1392b 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -278,57 +278,116 @@ @Inbook{VanLeerBookChapter1997 } @article{Yakimenko2019, - title = {Prospect of Studying Nonperturbative QED with Beam-Beam Collisions}, - author = {Yakimenko, V. and Meuren, S. and Del Gaudio, F. and Baumann, C. and Fedotov, A. and Fiuza, F. and Grismayer, T. and Hogan, M. J. and Pukhov, A. and Silva, L. O. and White, G.}, - journal = {Phys. Rev. Lett.}, - volume = {122}, - issue = {19}, - pages = {190404}, - numpages = {7}, - year = {2019}, - month = {May}, - publisher = {American Physical Society}, - doi = {10.1103/PhysRevLett.122.190404}, - url = {https://link.aps.org/doi/10.1103/PhysRevLett.122.190404} +author = {Yakimenko, V. and Meuren, S. and Del Gaudio, F. and Baumann, C. and Fedotov, A. and Fiuza, F. and Grismayer, T. and Hogan, M. J. and Pukhov, A. and Silva, L. O. and White, G.}, +doi = {10.1103/PhysRevLett.122.190404}, +issue = {19}, +journal = {Phys. Rev. Lett.}, +month = {May}, +numpages = {7}, +pages = {190404}, +publisher = {American Physical Society}, +title = {Prospect of Studying Nonperturbative QED with Beam-Beam Collisions}, +volume = {122}, +year = {2019}, } @article{Groenewald2023, author = {Groenewald, R. E. and Veksler, A. and Ceccherini, F. and Necas, A. and Nicks, B. S. and Barnes, D. C. and Tajima, T. and Dettrick, S. A.}, -title = "{Accelerated kinetic model for global macro stability studies of high-beta fusion reactors}", +doi = {10.1063/5.0178288}, +issn = {1070-664X}, journal = {Physics of Plasmas}, -volume = {30}, +month = {Dec}, number = {12}, pages = {122508}, +title = {{Accelerated kinetic model for global macro stability studies of high-beta fusion reactors}}, +volume = {30}, year = {2023}, -month = {12}, +} + +@article{PerezPOP2012, +author = {Pérez, F. and Gremillet, L. and Decoster, A. and Drouin, M. and Lefebvre, E.}, +doi = {10.1063/1.4742167}, issn = {1070-664X}, -doi = {10.1063/5.0178288}, +journal = {Physics of Plasmas}, +month = {Aug}, +number = {8}, +pages = {083104}, +title = {{Improved modeling of relativistic collisions and collisional ionization in particle-in-cell codes}}, +volume = {19}, +year = {2012} +} + +@article{HigginsonJCP2019, +author = {Drew Pitney Higginson and Anthony Link and Andrea Schmidt}, +doi = {10.1016/j.jcp.2019.03.020}, +issn = {0021-9991}, +journal = {Journal of Computational Physics}, +keywords = {Particle-in-cell, Nuclear Fusion, Monte Carlo methods, Inertial confinement fusion, Thermonuclear Fusion}, +month = {Jul}, +pages = {439--453}, +title = {{A pairwise nuclear fusion algorithm for weighted particle-in-cell plasma simulations}}, +volume = {388}, +year = {2019} } -@article{Perez2012, - author = {Pérez, F. and Gremillet, L. and Decoster, A. and Drouin, M. and Lefebvre, E.}, - title = "{Improved modeling of relativistic collisions and collisional ionization in particle-in-cell codes}", - journal = {Physics of Plasmas}, - volume = {19}, - number = {8}, - pages = {083104}, - year = {2012}, - month = {08}, - issn = {1070-664X}, - doi = {10.1063/1.4742167}, - url = {https://doi.org/10.1063/1.4742167}, - eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4742167/13891570/083104\_1\_online.pdf}, +@article{VerboncoeurJCP2001, +author = {J.P. Verboncoeur}, +doi = {10.1006/jcph.2001.6923}, +issn = {0021-9991}, +journal = {Journal of Computational Physics}, +number = {1}, +pages = {421-427}, +title = {{Symmetric Spline Weighting for Charge and Current Density in Particle Simulation}}, +volume = {174}, +year = {2001} +} + +@article{MuravievCPC2021, +author = {A. Muraviev and A. Bashinov and E. Efimenko and V. Volokitin and I. Meyerov and A. Gonoskov}, +doi = {10.1016/j.cpc.2021.107826}, +issn = {0010-4655}, +journal = {Computer Physics Communications}, +keywords = {Particle-in-cell, Resampling, Merging, Thinning, QED cascades}, +pages = {107826}, +title = {{Strategies for particle resampling in PIC simulations}}, +volume = {262}, +year = {2021} } -@article{Higginson2019, - doi = {10.1016/j.jcp.2019.03.020}, - url = {https://doi.org/10.1016/j.jcp.2019.03.020}, - year = {2019}, - month = jul, - publisher = {Elsevier {BV}}, - volume = {388}, - pages = {439--453}, - author = {Drew Pitney Higginson and Anthony Link and Andrea Schmidt}, - title = {A pairwise nuclear fusion algorithm for weighted particle-in-cell plasma simulations}, - journal = {Journal of Computational Physics} +@article{AkturkOE2004, +author = {Selcuk Akturk and Xun Gu and Erik Zeek and Rick Trebino}, +doi = {10.1364/OPEX.12.004399}, +journal = {Opt. Express}, +keywords = {Pulses; Ultrafast measurements; CCD cameras; Dispersion; Distortion; Fourier transforms; Gaussian beams; Ultrashort pulses}, +month = {Sep}, +number = {19}, +pages = {4399--4410}, +publisher = {Optica Publishing Group}, +title = {{Pulse-front tilt caused by spatial and temporal chirp}}, +volume = {12}, +year = {2004} +} + +@inproceedings{XiaoIEEE2005, +author = {Tian Xiao and Qing Huo Liu}, +booktitle = {2005 IEEE Antennas and Propagation Society International Symposium}, +doi = {10.1109/APS.2005.1551259}, +number = {}, +pages = {122-125 Vol. 1A}, +title = {{An enlarged cell technique for the conformal FDTD method to model perfectly conducting objects}}, +volume = {1A}, +year = {2005} +} + +@article{GrismayerNJP2021, +author = {T Grismayer and R Torres and P Carneiro and F Cruz and R A Fonseca and L O Silva}, +doi = {10.1088/1367-2630/ac2004}, +journal = {New Journal of Physics}, +month = {Sep}, +number = {9}, +pages = {095005}, +publisher = {IOP Publishing}, +title = {{Quantum Electrodynamics vacuum polarization solver}}, +volume = {23}, +year = {2021} } diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 674f86143e3..d9b82cf8fe4 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -155,7 +155,7 @@ Overall simulation parameters \qquad \boldsymbol{B} = -\frac{1}{c}\boldsymbol{\beta}\times\boldsymbol{\nabla}\phi where :math:`\boldsymbol{\beta}` is the average (normalized) velocity of the considered species (which can be relativistic). - See e.g. `Vay et al., Physics of Plasmas 15, 056701 (2008) `__ for more information. + See, e.g., :cite:t:`param-Vaypop2008` for more information. See the `AMReX documentation `_ for details of the MLMG solver (the default solver used with electrostatic @@ -387,7 +387,7 @@ Domain Boundary Conditions * ``Periodic``: This option can be used to set periodic domain boundaries. Note that if the fields for lo in a certain dimension are set to periodic, then the corresponding upper boundary must also be set to periodic. If particle boundaries are not specified in the input file, then particles boundaries by default will be set to periodic. If particles boundaries are specified, then they must be set to periodic corresponding to the periodic field boundaries. - * ``pml`` (default): This option can be used to add Perfectly Matched Layers (PML) around the simulation domain. See the :ref:`PML theory section ` for more details. + * ``pml`` (default): This option can be used to add Perfectly Matched Layers (PML) around the simulation domain. See the :ref:`PML theory section ` for more details. Additional pml algorithms can be explored using the parameters ``warpx.do_pml_in_domain``, ``warpx.pml_has_particles``, and ``warpx.do_pml_j_damping``. * ``absorbing_silver_mueller``: This option can be used to set the Silver-Mueller absorbing boundary conditions. These boundary conditions are simpler and less computationally expensive than the pml, but are also less effective at absorbing the field. They only work with the Yee Maxwell solver. @@ -417,7 +417,7 @@ Domain Boundary Conditions * ``boundary.verboncoeur_axis_correction`` (`bool`) optional (default `true`) Whether to apply the Verboncoeur correction on the charge and current density on axis when using RZ. For nodal values (rho and Jz), the cell volume for values on axis is calculated as :math:`\pi*\Delta r^2/4`. - In `Verboncoeur JCP 174, 421-427 (2001) `__, it is shown that using + In :cite:t:`param-VerboncoeurJCP2001`, it is shown that using :math:`\pi*\Delta r^2/3` instead will give a uniform density if the particle density is uniform. Additional PML parameters @@ -682,8 +682,8 @@ Particle initialization * ``particles.rigid_injected_species`` (`strings`, separated by spaces) List of species injected using the rigid injection method. The rigid injection - method is useful when injecting a relativistic particle beam, in boosted-frame - simulation ; see the :ref:`input-output section ` for more details. + method is useful when injecting a relativistic particle beam in boosted-frame + simulations; see the :ref:`input-output section ` for more details. For species injected using this method, particles are translated along the `+z` axis with constant velocity as long as their ``z`` coordinate verifies ``zzinject_plane``, @@ -1236,7 +1236,7 @@ Particle initialization The algorithm used for resampling. Currently there is only one option, which is already set by default: - * ``leveling_thinning`` This algorithm is defined in `Muraviev et al., arXiv:2006.08593 (2020) `_. + * ``leveling_thinning`` This algorithm is defined in :cite:t:`param-MuravievCPC2021`. It has two parameters: * ``.resampling_algorithm_target_ratio`` (`float`) optional (default `1.5`) @@ -1441,18 +1441,18 @@ Laser initialization * ``.stc_direction`` (`3 floats`) optional (default `1. 0. 0.`) Direction of laser spatio-temporal couplings. - See definition in Akturk et al., Opt Express, vol 12, no 19 (2004). + See definition in :cite:t:`param-AkturkOE2004`. * ``.zeta`` (`float`; in meters.seconds) optional (default `0.`) Spatial chirp at focus in direction ``.stc_direction``. See definition in - Akturk et al., Opt Express, vol 12, no 19 (2004). + :cite:t:`param-AkturkOE2004`. * ``.beta`` (`float`; in seconds) optional (default `0.`) Angular dispersion (or angular chirp) at focus in direction ``.stc_direction``. - See definition in Akturk et al., Opt Express, vol 12, no 19 (2004). + See definition in :cite:t:`param-AkturkOE2004`. * ``.phi2`` (`float`; in seconds**2) optional (default `0.`) - The amount of temporal chirp :math:`\phi^{(2)}`, at focus (in the lab frame). Namely, a wave packet + The amount of temporal chirp :math:`\phi^{(2)}` at focus (in the lab frame). Namely, a wave packet centered on the frequency :math:`(\omega_0 + \delta \omega)` will reach its peak intensity at :math:`z(\delta \omega) = z_0 - c \phi^{(2)} \, \delta \omega`. Thus, a positive :math:`\phi^{(2)}` corresponds to positive chirp, i.e. red part of the spectrum in the @@ -1470,6 +1470,8 @@ Laser initialization \tau' = \sqrt{ \tau^2 + 4 (\phi^{(2)})^2/\tau^2 } + See also the definition in :cite:t:`param-AkturkOE2004`. + * ``.do_continuous_injection`` (`0` or `1`) optional (default `0`). Whether or not to use continuous injection. If the antenna starts outside of the simulation domain but enters it @@ -1779,11 +1781,11 @@ Details about the collision models can be found in the :ref:`theory section .fusion_multiplier`` (`float`) optional. Only for ``nuclearfusion``. @@ -1833,7 +1835,7 @@ Details about the collision models can be found in the :ref:`theory section .fusion_probability_threshold`` (`float`) optional. @@ -1993,12 +1995,12 @@ Particle push, charge and current deposition, field gathering 2. ``esirkepov`` The current density is deposited as described in - `(Esirkepov, CPC, 2001) `_. + :cite:t:`param-Esirkepovcpc01`. This deposition scheme guarantees charge conservation for shape factors of arbitrary order. 3. ``vay`` - The current density is deposited as described in `(Vay et al, 2013) `_ (see section :ref:`current_deposition` for more details). + The current density is deposited as described in :cite:t:`param-VayJCP2013` (see section :ref:`current_deposition` for more details). This option guarantees charge conservation only when used in combination with ``psatd.periodic_single_box_fft=1``, that is, only for periodic single-box simulations with global FFTs without guard cells. The implementation for domain @@ -2025,8 +2027,8 @@ Particle push, charge and current deposition, field gathering The algorithm for the particle pusher. Available options are: - ``boris``: Boris pusher. - - ``vay``: Vay pusher (see `Vay, Phys. Plasmas (2008) `__) - - ``higuera``: Higuera-Cary pusher (see `Higuera and Cary, Phys. Plasmas (2017) `__) + - ``vay``: Vay pusher (see :cite:t:`param-Vaypop2008`) + - ``higuera``: Higuera-Cary pusher (see :cite:t:`param-HigueraPOP2017`) If ``algo.particle_pusher`` is not specified, ``boris`` is the default. @@ -2048,9 +2050,9 @@ Two families of Maxwell solvers are implemented in WarpX, based on the Finite-Di - ``yee``: Yee FDTD solver. - ``ckc``: (not available in ``RZ`` geometry) Cole-Karkkainen solver with Cowan - coefficients (see `Cowan, PRSTAB 16 (2013) `_). + coefficients (see :cite:t:`param-CowanPRSTAB13`). - ``psatd``: Pseudo-spectral solver (see :ref:`theory `). - - ``ect``: Enlarged cell technique (conformal finite difference solver. See `Xiao and Liu, IEEE Antennas and Propagation Society International Symposium (2005) `_). + - ``ect``: Enlarged cell technique (conformal finite difference solver. See :cite:t:`param-XiaoIEEE2005`). - ``hybrid``: The E-field will be solved using Ohm's law and a kinetic-fluid hybrid model (see :ref:`theory `). - ``none``: No field solve will be performed. @@ -2087,14 +2089,14 @@ Maxwell solver: PSATD method If true, a current correction scheme in Fourier space is applied in order to guarantee charge conservation. The default value is ``psatd.current_correction=1``, unless a charge-conserving current deposition scheme is used (by setting ``algo.current_deposition=esirkepov`` or ``algo.current_deposition=vay``) or unless the ``div(E)`` cleaning scheme is used (by setting ``warpx.do_dive_cleaning=1``). - If ``psatd.v_galilean`` is zero, the spectral solver used is the standard PSATD scheme described in (`Vay et al, JCP 243, 2013 `_) and the current correction reads + If ``psatd.v_galilean`` is zero, the spectral solver used is the standard PSATD scheme described in :cite:t:`param-VayJCP2013` and the current correction reads .. math:: \widehat{\boldsymbol{J}}^{\,n+1/2}_{\mathrm{correct}} = \widehat{\boldsymbol{J}}^{\,n+1/2} - \bigg(\boldsymbol{k}\cdot\widehat{\boldsymbol{J}}^{\,n+1/2} - i \frac{\widehat{\rho}^{n+1} - \widehat{\rho}^{n}}{\Delta{t}}\bigg) \frac{\boldsymbol{k}}{k^2} - If ``psatd.v_galilean`` is non-zero, the spectral solver used is the Galilean PSATD scheme described in (`Lehe et al, PRE 94, 2016 `_) and the current correction reads + If ``psatd.v_galilean`` is non-zero, the spectral solver used is the Galilean PSATD scheme described in :cite:t:`param-LehePRE2016` and the current correction reads .. math:: \widehat{\boldsymbol{J}}^{\,n+1/2}_{\mathrm{correct}} = \widehat{\boldsymbol{J}}^{\,n+1/2} @@ -2110,7 +2112,7 @@ Maxwell solver: PSATD method If false, instead, the update equation for the electric field is expressed in terms of the current density :math:`\widehat{\boldsymbol{J}}^{\,n+1/2}` only. If charge is expected to be conserved (by setting, for example, ``psatd.current_correction=1``), then the two formulations are expected to be equivalent. - If ``psatd.v_galilean`` is zero, the spectral solver used is the standard PSATD scheme described in (`Vay et al, JCP 243, 2013 `_): + If ``psatd.v_galilean`` is zero, the spectral solver used is the standard PSATD scheme described in :cite:t:`param-VayJCP2013`: 1. if ``psatd.update_with_rho=0``, the update equation for the electric field reads @@ -2136,9 +2138,9 @@ Maxwell solver: PSATD method \frac{1}{\Delta{t}}\right)\widehat{\rho}^{n+1} \boldsymbol{k} \end{split} - The coefficients :math:`C` and :math:`S` are defined in (`Vay et al, JCP 243, 2013 `_). + The coefficients :math:`C` and :math:`S` are defined in :cite:t:`param-VayJCP2013`. - If ``psatd.v_galilean`` is non-zero, the spectral solver used is the Galilean PSATD scheme described in (`Lehe et al, PRE 94, 2016 `_): + If ``psatd.v_galilean`` is non-zero, the spectral solver used is the Galilean PSATD scheme described in :cite:t:`param-LehePRE2016`: 1. if ``psatd.update_with_rho=0``, the update equation for the electric field reads @@ -2168,7 +2170,7 @@ Maxwell solver: PSATD method - i \, \frac{\chi_2}{\epsilon_0 k^{2}} \widehat{\rho}^{\,n+1} \boldsymbol{k} \end{split} - The coefficients :math:`C`, :math:`S`, :math:`\theta`, :math:`\nu`, :math:`\chi_1`, :math:`\chi_2`, and :math:`\chi_3` are defined in (`Lehe et al, PRE 94, 2016 `_). + The coefficients :math:`C`, :math:`S`, :math:`\theta`, :math:`\nu`, :math:`\chi_1`, :math:`\chi_2`, and :math:`\chi_3` are defined in :cite:t:`param-LehePRE2016`. The default value for ``psatd.update_with_rho`` is ``1`` if ``psatd.v_galilean`` is non-zero and ``0`` otherwise. The option ``psatd.update_with_rho=0`` is not implemented with the following algorithms: @@ -2279,7 +2281,7 @@ Grid types (collocated, staggered, hybrid) Whether to use a Galerkin scheme when gathering fields to particles. When set to ``1``, the interpolation orders used for field-gathering are reduced for certain field components along certain directions. For example, :math:`E_z` is gathered using ``algo.particle_shape`` along :math:`(x,y)` and ``algo.particle_shape - 1`` along :math:`z`. - See equations (21)-(23) of (`Godfrey and Vay, 2013 `_) and associated references for details. + See equations (21)-(23) of :cite:t:`param-Godfrey2013` and associated references for details. Default: ``interpolation.galerkin_scheme = 0`` with collocated grids and/or momentum-conserving field gathering, ``interpolation.galerkin_scheme = 1`` otherwise. @@ -2338,9 +2340,8 @@ Additional parameters is performed at every timestep regardless of this parameter. * ``warpx.use_hybrid_QED`` (`bool`; default: 0) - Will use the Hybird QED Maxwell solver when pushing fields: a QED correction is added to the - field solver to solve non-linear Maxwell's equations, according to [Quantum Electrodynamics - vacuum polarization solver, P. Carneiro et al., `ArXiv 2016 `__]. + Will use the Hybrid QED Maxwell solver when pushing fields: a QED correction is added to the + field solver to solve non-linear Maxwell's equations, according to :cite:t:`param-GrismayerNJP2021`. Note that this option can only be used with the PSATD build. Furthermore, one must set ``warpx.grid_type = collocated`` (which otherwise would be ``staggered`` by default). From a8f20e335a4935f41a592ff64627b54c3c4e1f5c Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Mon, 15 Jan 2024 20:57:38 -0800 Subject: [PATCH 105/176] atomicAdd -> lockAdd (#4608) For WarpX's Gordon Bell runs on Fugaku, Stephan Jaure of ATOS optimized atomicAdd using pthread spin locks. The optimized approach has been implemented in amrex::BaseFab::lockAdd, which is now used in WarpX. --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- Source/Particles/WarpXParticleContainer.cpp | 8 ++++---- Source/ablastr/particles/DepositCharge.H | 2 +- cmake/dependencies/AMReX.cmake | 2 +- run_test.sh | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index b50989f2f5c..ac80e8d602c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach f1ec8df75c562d2a4822cea84d284cf8e72c2e14 && cd - + cd ../amrex && git checkout --detach 255d30f387cf2c1a7eff5a31f703c94de803e8d8 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 2a98d7b3736..f6ff47ce23b 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = f1ec8df75c562d2a4822cea84d284cf8e72c2e14 +branch = 255d30f387cf2c1a7eff5a31f703c94de803e8d8 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index f5ef38e881b..703882ff2c7 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = f1ec8df75c562d2a4822cea84d284cf8e72c2e14 +branch = 255d30f387cf2c1a7eff5a31f703c94de803e8d8 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index f1b19dd8407..6b6d3ec52f2 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -701,9 +701,9 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, #ifndef AMREX_USE_GPU // CPU, tiling: atomicAdd local_j into j WARPX_PROFILE_VAR_START(blp_accumulate); - (*jx)[pti].atomicAdd(local_jx[thread_num], tbx, tbx, 0, 0, jx->nComp()); - (*jy)[pti].atomicAdd(local_jy[thread_num], tby, tby, 0, 0, jy->nComp()); - (*jz)[pti].atomicAdd(local_jz[thread_num], tbz, tbz, 0, 0, jz->nComp()); + (*jx)[pti].lockAdd(local_jx[thread_num], tbx, tbx, 0, 0, jx->nComp()); + (*jy)[pti].lockAdd(local_jy[thread_num], tby, tby, 0, 0, jy->nComp()); + (*jz)[pti].lockAdd(local_jz[thread_num], tbz, tbz, 0, 0, jz->nComp()); WARPX_PROFILE_VAR_STOP(blp_accumulate); #endif } @@ -996,7 +996,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, #ifndef AMREX_USE_GPU // CPU, tiling: atomicAdd local_rho into rho WARPX_PROFILE_VAR_START(blp_accumulate); - (*rho)[pti].atomicAdd(local_rho[thread_num], tb, tb, 0, icomp*nc, nc); + (*rho)[pti].lockAdd(local_rho[thread_num], tb, tb, 0, icomp*nc, nc); WARPX_PROFILE_VAR_STOP(blp_accumulate); #endif } else { diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index da24af94dfb..ad01ba4a213 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -201,7 +201,7 @@ deposit_charge (typename T_PC::ParIterType& pti, #ifndef AMREX_USE_GPU // CPU, tiling: atomicAdd local_rho into rho ABLASTR_PROFILE_VAR_START(blp_accumulate, do_device_synchronize); - (*rho)[pti].atomicAdd(local_rho, tb, tb, 0, icomp*nc, nc); + (*rho)[pti].lockAdd(local_rho, tb, tb, 0, icomp*nc, nc); ABLASTR_PROFILE_VAR_STOP(blp_accumulate, do_device_synchronize); #endif } diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 316b68410c0..080465083f9 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "f1ec8df75c562d2a4822cea84d284cf8e72c2e14" +set(WarpX_amrex_branch "255d30f387cf2c1a7eff5a31f703c94de803e8d8" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/run_test.sh b/run_test.sh index 5dd813829c7..dc0906cb4af 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach f1ec8df75c562d2a4822cea84d284cf8e72c2e14 && cd - +cd amrex && git checkout --detach 255d30f387cf2c1a7eff5a31f703c94de803e8d8 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From a3e04a73e9c42efc11ba35155b7b2c9ec4134e06 Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 16 Jan 2024 09:19:04 -0800 Subject: [PATCH 106/176] Disambiguate the routine name initialize_inputs in picmi.py (#4554) --- Python/pywarpx/picmi.py | 127 ++++++++++++++++++++-------------------- 1 file changed, 64 insertions(+), 63 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 7d7e150f111..b33a1a08899 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -219,10 +219,10 @@ def init(self, kw): self.resampling_trigger_intervals = kw.pop('warpx_resampling_trigger_intervals', None) self.resampling_triggering_max_avg_ppc = kw.pop('warpx_resampling_trigger_max_avg_ppc', None) - def initialize_inputs(self, layout, - initialize_self_fields = False, - injection_plane_position = None, - injection_plane_normal_vector = None): + def species_initialize_inputs(self, layout, + initialize_self_fields = False, + injection_plane_position = None, + injection_plane_normal_vector = None): self.species_number = len(pywarpx.particles.species_names) if self.name is None: @@ -274,7 +274,8 @@ def initialize_inputs(self, layout, distributions_is_list = np.iterable(self.initial_distribution) layout_is_list = np.iterable(layout) if not distributions_is_list and not layout_is_list: - self.initial_distribution.initialize_inputs(self.species_number, layout, self.species, self.density_scale, '') + self.initial_distribution.distribution_initialize_inputs(self.species_number, layout, self.species, + self.density_scale, '') elif distributions_is_list and (layout_is_list or layout is None): assert layout is None or (len(self.initial_distribution) == len(layout)),\ Exception('The initial distribution and layout lists must have the same lenth') @@ -282,8 +283,8 @@ def initialize_inputs(self, layout, self.species.injection_sources = source_names for i, dist in enumerate(self.initial_distribution): layout_i = layout[i] if layout is not None else None - dist.initialize_inputs(self.species_number, layout_i, self.species, - self.density_scale, source_names[i]) + dist.distribution_initialize_inputs(self.species_number, layout_i, self.species, + self.density_scale, source_names[i]) else: raise Exception('The initial distribution and layout must both be scalars or both be lists') @@ -298,15 +299,15 @@ def initialize_inputs(self, layout, picmistandard.PICMI_MultiSpecies.Species_class = Species class MultiSpecies(picmistandard.PICMI_MultiSpecies): - def initialize_inputs(self, layout, - initialize_self_fields = False, - injection_plane_position = None, - injection_plane_normal_vector = None): + def species_initialize_inputs(self, layout, + initialize_self_fields = False, + injection_plane_position = None, + injection_plane_normal_vector = None): for species in self.species_instances_list: - species.initialize_inputs(layout, - initialize_self_fields, - injection_plane_position, - injection_plane_normal_vector) + species.species_initialize_inputs(layout, + initialize_self_fields, + injection_plane_position, + injection_plane_normal_vector) class GaussianBunchDistribution(picmistandard.PICMI_GaussianBunchDistribution): @@ -314,7 +315,7 @@ def init(self, kw): self.do_symmetrize = kw.pop('warpx_do_symmetrize', None) self.symmetrization_order = kw.pop('warpx_symmetrization_order', None) - def initialize_inputs(self, species_number, layout, species, density_scale, source_name): + def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): species.add_new_group_attr(source_name, 'injection_style', "gaussian_beam") species.add_new_group_attr(source_name, 'x_m', self.centroid_position[0]) species.add_new_group_attr(source_name, 'y_m', self.centroid_position[1]) @@ -441,7 +442,7 @@ def setup_parse_momentum_functions(self, species, source_name, expressions, suff class UniformFluxDistribution(picmistandard.PICMI_UniformFluxDistribution, DensityDistributionBase): - def initialize_inputs(self, species_number, layout, species, density_scale, source_name): + def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.fill_in = False self.set_mangle_dict() @@ -465,7 +466,7 @@ def initialize_inputs(self, species_number, layout, species, density_scale, sour class UniformDistribution(picmistandard.PICMI_UniformDistribution, DensityDistributionBase): - def initialize_inputs(self, species_number, layout, species, density_scale, source_name): + def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.set_mangle_dict() self.set_species_attributes(species, layout, source_name) @@ -492,7 +493,7 @@ class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDist def init(self, kw): self.momentum_spread_expressions = kw.pop('warpx_momentum_spread_expressions', [None, None, None]) - def initialize_inputs(self, species_number, layout, species, density_scale, source_name): + def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.set_mangle_dict() self.set_species_attributes(species, layout, source_name) @@ -509,7 +510,7 @@ class ParticleListDistribution(picmistandard.PICMI_ParticleListDistribution): def init(self, kw): pass - def initialize_inputs(self, species_number, layout, species, density_scale, source_name): + def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): species.add_new_group_attr(source_name, 'injection_style', "multipleparticles") species.add_new_group_attr(source_name, 'multiple_particles_pos_x', self.x) @@ -539,7 +540,7 @@ def init(self, kw): class BinomialSmoother(picmistandard.PICMI_BinomialSmoother): - def initialize_inputs(self, solver): + def smoother_initialize_inputs(self, solver): pywarpx.warpx.use_filter = 1 pywarpx.warpx.use_filter_compensation = bool(np.all(self.compensation)) if self.n_pass is None: @@ -629,7 +630,7 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound - def initialize_inputs(self): + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells # Maximum allowable size of each subdomain in the problem domain; @@ -730,7 +731,7 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound - def initialize_inputs(self): + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells # Maximum allowable size of each subdomain in the problem domain; @@ -833,7 +834,7 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound - def initialize_inputs(self): + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells # Maximum allowable size of each subdomain in the problem domain; @@ -956,7 +957,7 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound - def initialize_inputs(self): + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells # Maximum allowable size of each subdomain in the problem domain; @@ -1061,9 +1062,9 @@ def init(self, kw): self.pml_has_particles = kw.pop('warpx_pml_has_particles', None) self.do_pml_j_damping = kw.pop('warpx_do_pml_j_damping', None) - def initialize_inputs(self): + def solver_initialize_inputs(self): - self.grid.initialize_inputs() + self.grid.grid_initialize_inputs() pywarpx.warpx.pml_ncell = self.pml_ncell @@ -1099,7 +1100,7 @@ def initialize_inputs(self): pywarpx.warpx.cfl = self.cfl if self.source_smoother is not None: - self.source_smoother.initialize_inputs(self) + self.source_smoother.smoother_initialize_inputs(self) pywarpx.warpx.do_dive_cleaning = self.divE_cleaning pywarpx.warpx.do_divb_cleaning = self.divB_cleaning @@ -1167,14 +1168,14 @@ def __init__(self, grid, Te=None, n0=None, gamma=None, self.handle_init(kw) - def initialize_inputs(self): + def solver_initialize_inputs(self): # Add the user defined keywords to my_constants # The keywords are mangled if there is a conflicting variable already # defined in my_constants with the same name but different value. self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) - self.grid.initialize_inputs() + self.grid.grid_initialize_inputs() pywarpx.algo.maxwell_solver = self.method @@ -1222,9 +1223,9 @@ def init(self, kw): self.self_fields_verbosity = kw.pop('warpx_self_fields_verbosity', None) self.magnetostatic = kw.pop('warpx_magnetostatic', False) - def initialize_inputs(self): + def solver_initialize_inputs(self): - self.grid.initialize_inputs() + self.grid.grid_initialize_inputs() if self.relativistic: pywarpx.warpx.do_electrostatic = 'relativistic' @@ -1246,7 +1247,7 @@ def initialize_inputs(self): class GaussianLaser(picmistandard.PICMI_GaussianLaser): - def initialize_inputs(self): + def laser_initialize_inputs(self): self.laser_number = len(pywarpx.lasers.names) + 1 if self.name is None: self.name = 'laser{}'.format(self.laser_number) @@ -1271,7 +1272,7 @@ class AnalyticLaser(picmistandard.PICMI_AnalyticLaser): def init(self, kw): self.mangle_dict = None - def initialize_inputs(self): + def laser_initialize_inputs(self): self.laser_number = len(pywarpx.lasers.names) + 1 if self.name is None: self.name = 'laser{}'.format(self.laser_number) @@ -1294,7 +1295,7 @@ def initialize_inputs(self): class LaserAntenna(picmistandard.PICMI_LaserAntenna): - def initialize_inputs(self, laser): + def laser_antenna_initialize_inputs(self, laser): laser.laser.position = self.position # This point is on the laser plane if ( self.normal_vector is not None @@ -1321,7 +1322,7 @@ def initialize_inputs(self, laser): class LoadInitialField(picmistandard.PICMI_LoadGriddedField): - def initialize_inputs(self): + def applied_field_initialize_inputs(self): pywarpx.warpx.read_fields_from_path = self.read_fields_from_path if self.load_E: pywarpx.warpx.E_ext_grid_init_style = 'read_from_file' @@ -1334,7 +1335,7 @@ def init(self, kw): self.mangle_dict = None self.maxlevel_extEMfield_init = kw.pop('warpx_maxlevel_extEMfield_init', None); - def initialize_inputs(self): + def applied_field_initialize_inputs(self): # Note that lower and upper_bound are not used by WarpX pywarpx.warpx.maxlevel_extEMfield_init = self.maxlevel_extEMfield_init; @@ -1361,7 +1362,7 @@ def initialize_inputs(self): class ConstantAppliedField(picmistandard.PICMI_ConstantAppliedField): - def initialize_inputs(self): + def applied_field_initialize_inputs(self): # Note that lower and upper_bound are not used by WarpX if (self.Ex is not None or @@ -1381,7 +1382,7 @@ class AnalyticAppliedField(picmistandard.PICMI_AnalyticAppliedField): def init(self, kw): self.mangle_dict = None - def initialize_inputs(self): + def applied_field_initialize_inputs(self): # Note that lower and upper_bound are not used by WarpX if self.mangle_dict is None: @@ -1407,7 +1408,7 @@ def initialize_inputs(self): class Mirror(picmistandard.PICMI_Mirror): - def initialize_inputs(self): + def applied_field_initialize_inputs(self): try: pywarpx.warpx.num_mirrors except AttributeError: @@ -1426,7 +1427,7 @@ class FieldIonization(picmistandard.PICMI_FieldIonization): """ WarpX only has ADK ionization model implemented. """ - def initialize_inputs(self): + def interaction_initialize_inputs(self): assert self.model == 'ADK', 'WarpX only has ADK ionization model implemented' self.ionized_species.species.do_field_ionization = 1 self.ionized_species.species.physical_element = self.ionized_species.particle_type @@ -1464,7 +1465,7 @@ def __init__(self, name, species, CoulombLog=None, ndt=None, **kw): self.handle_init(kw) - def initialize_inputs(self): + def collision_initialize_inputs(self): collision = pywarpx.Collisions.newcollision(self.name) collision.type = 'pairwisecoulomb' collision.species = [species.name for species in self.species] @@ -1521,7 +1522,7 @@ def __init__(self, name, species, background_density, self.handle_init(kw) - def initialize_inputs(self): + def collision_initialize_inputs(self): collision = pywarpx.Collisions.newcollision(self.name) collision.type = 'background_mcc' collision.species = self.species.name @@ -1656,7 +1657,7 @@ def __init__(self, implicit_function=None, stl_file=None, stl_scale=None, stl_ce self.handle_init(kw) - def initialize_inputs(self, solver): + def embedded_boundary_initialize_inputs(self, solver): # Add the user defined keywords to my_constants # The keywords are mangled if there is a conflicting variable already @@ -1728,7 +1729,7 @@ def __init__(self, period, starts, lengths, strengths_E=None, strengths_B=None, self.handle_init(kw) - def initialize_inputs(self): + def applied_field_initialize_inputs(self): pywarpx.particles.E_ext_particle_init_style = 'repeated_plasma_lens' pywarpx.particles.B_ext_particle_init_style = 'repeated_plasma_lens' @@ -2016,7 +2017,7 @@ def initialize_inputs(self): interpolation_order = particle_shape pywarpx.algo.particle_shape = interpolation_order - self.solver.initialize_inputs() + self.solver.solver_initialize_inputs() # Initialize warpx.field_centering_no and warpx.current_centering_no # if set by the user in the input (need to access grid info from solver attribute) @@ -2034,33 +2035,33 @@ def initialize_inputs(self): pywarpx.warpx.current_centering_noz = self.current_centering_order[-1] for i in range(len(self.species)): - self.species[i].initialize_inputs(self.layouts[i], - self.initialize_self_fields[i], - self.injection_plane_positions[i], - self.injection_plane_normal_vectors[i]) + self.species[i].species_initialize_inputs(self.layouts[i], + self.initialize_self_fields[i], + self.injection_plane_positions[i], + self.injection_plane_normal_vectors[i]) for interaction in self.interactions: assert(isinstance(interaction, FieldIonization)) - interaction.initialize_inputs() + interaction.interaction_initialize_inputs() if self.collisions is not None: pywarpx.collisions.collision_names = [] for collision in self.collisions: pywarpx.collisions.collision_names.append(collision.name) - collision.initialize_inputs() + collision.collision_initialize_inputs() if self.embedded_boundary is not None: - self.embedded_boundary.initialize_inputs(self.solver) + self.embedded_boundary.embedded_boundary_initialize_inputs(self.solver) for i in range(len(self.lasers)): - self.lasers[i].initialize_inputs() - self.laser_injection_methods[i].initialize_inputs(self.lasers[i]) + self.lasers[i].laser_initialize_inputs() + self.laser_injection_methods[i].laser_antenna_initialize_inputs(self.lasers[i]) for applied_field in self.applied_fields: - applied_field.initialize_inputs() + applied_field.applied_field_initialize_inputs() for diagnostic in self.diagnostics: - diagnostic.initialize_inputs() + diagnostic.diagnostic_initialize_inputs() if self.amr_restart: pywarpx.amr.restart = self.amr_restart @@ -2218,7 +2219,7 @@ def init(self, kw): self.particle_fields_to_plot = kw.pop('warpx_particle_fields_to_plot', []) self.particle_fields_species = kw.pop('warpx_particle_fields_species', None) - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() @@ -2356,7 +2357,7 @@ def __init__(self, period = 1, write_dir = None, name = None, **kw): self.handle_init(kw) - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() @@ -2416,7 +2417,7 @@ def init(self, kw): self.mangle_dict = None - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() @@ -2531,7 +2532,7 @@ def init(self, kw): self.lower_bound = kw.pop('warpx_lower_bound', None) self.upper_bound = kw.pop('warpx_upper_bound', None) - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() @@ -2630,7 +2631,7 @@ def init(self, kw): self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.buffer_size = kw.pop('warpx_buffer_size', None) - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() @@ -2922,7 +2923,7 @@ def _handle_charge_on_eb(self, **kw): return kw - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() From 11e2a1722aac8929db0ed677f634673d235c1396 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 16 Jan 2024 10:17:58 -0800 Subject: [PATCH 107/176] Fix undefined behavior in meanParticleVelocity (#4614) --- Source/Particles/WarpXParticleContainer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 6b6d3ec52f2..e94aae5d573 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1287,7 +1287,7 @@ std::array WarpXParticleContainer::meanParticleVelocity(bool lo ParallelDescriptor::ReduceLongSum(np_total); } - std::array mean_v; + std::array mean_v = {0,0,0}; if (np_total > 0) { mean_v[0] = vx_total / np_total; mean_v[1] = vy_total / np_total; From a2b9e68d8c7446e41a97bc5bd022e0df5d482c2a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 18 Jan 2024 17:10:09 -0800 Subject: [PATCH 108/176] Perlmutter (NERSC): January 2024 Update (#4620) Update modules of Perlmutter after the system upgrade. Since the default compilers changed, please run the install of dependencies scripts again and rebuild your WarpX executables, too. --- .../perlmutter_cpu_warpx.profile.example | 10 +++++----- .../perlmutter_gpu_warpx.profile.example | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 0150ded4839..67ae36c9f1a 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -7,14 +7,14 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo # required dependencies module load cpu -module load cmake/3.22.0 -module load cray-fftw/3.3.10.3 +module load cmake/3.24.3 +module load cray-fftw/3.3.10.6 # optional: for QED support with detailed tables export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 # optional: for openPMD and PSATD+RZ support -module load cray-hdf5-parallel/1.12.2.7 +module load cray-hdf5-parallel/1.12.2.9 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master:$CMAKE_PREFIX_PATH @@ -28,10 +28,10 @@ export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master/ export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH} # optional: CCache -export PATH=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8-eqk2d3bipbpkgwxq7ujlp6mckwal4dwz/bin:$PATH +export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8.2-cvooxdw5wgvv2g3vjxjkrpv6dopginv6/bin:$PATH # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load cray-python/3.11.5 if [ -d "${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx" ] then diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 54cd33fe476..7bf5dd13116 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -12,13 +12,13 @@ module load craype module load craype-x86-milan module load craype-accel-nvidia80 module load cudatoolkit -module load cmake/3.22.0 +module load cmake/3.24.3 # optional: for QED support with detailed tables export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 # optional: for openPMD and PSATD+RZ support -module load cray-hdf5-parallel/1.12.2.7 +module load cray-hdf5-parallel/1.12.2.9 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master:$CMAKE_PREFIX_PATH @@ -32,10 +32,10 @@ export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-mast export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH} # optional: CCache -export PATH=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8-eqk2d3bipbpkgwxq7ujlp6mckwal4dwz/bin:$PATH +export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8.2-cvooxdw5wgvv2g3vjxjkrpv6dopginv6/bin:$PATH # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load cray-python/3.11.5 if [ -d "${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx" ] then From 3b1dff5cdf9a5ad1181e4a2cc51373dab54f720b Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Fri, 19 Jan 2024 14:48:27 -0800 Subject: [PATCH 109/176] update cupy installation in Perlmutter dependencies (#4621) --- Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index 8dd64c56c58..3fe76359953 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -130,9 +130,9 @@ python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -python3 -m pip install cupy-cuda11x # CUDA 11.7 compatible wheel +python3 -m pip install cupy-cuda12x # CUDA 12 compatible wheel # optional: for libEnsemble python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch # CUDA 11.7 compatible wheel +python3 -m pip install --upgrade torch # CUDA 12 compatible wheel python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt From d2484901c9e9f4351117b196959f61c36f8858af Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:59:29 -0800 Subject: [PATCH 110/176] Use RK4 to integrate the B-field in time in the hybrid-PIC algorithm (#4461) * add external current support to the hybrid-PIC solver * add RZ support for `FiniteDifferenceSolver::CalculateCurrentAmpere` * allow an initial Bz field to be set in RZ * code cleanup and addition of CI test * revert unwanted changes * WIP transition to use RK4 in B-field time integration in hybrid-PIC algorithm Co-authored-by: Avigdor Veksler * fix some clang-tidy issues * Complete BfieldEvolveRK (#2) * continued transition to RK4 in B-field time integration in hybrid-PIC algorithm * RK-integration over all levels * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update some of the CI tests * more CI test updates * Various code cleanups * more updates to CI tests and checksum benchmarks * more updates to CI tests and checksum benchmarks; remove commented code * update documentation * fix 1 for RZ CI test after merging of #4464 * Avoid using `const` with `Real` passed by value Co-authored-by: Axel Huebl * reduce default number of substeps to 10 * formatting fix --------- Co-authored-by: Avigdor Veksler Co-authored-by: Avigdor Veksler <124003120+aveksler1@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- .../theory/kinetic_fluid_hybrid_model.rst | 12 +- Docs/source/usage/parameters.rst | 2 +- .../Tests/ohm_solver_EM_modes/PICMI_inputs.py | 3 +- .../ohm_solver_EM_modes/PICMI_inputs_rz.py | 2 +- .../Tests/ohm_solver_EM_modes/analysis_rz.py | 2 +- .../PICMI_inputs.py | 2 +- .../PICMI_inputs.py | 6 +- .../analysis.py | 10 +- .../PICMI_inputs.py | 2 +- .../Python_ohms_law_solver_EM_modes_1d.json | 18 +-- .../Python_ohms_law_solver_EM_modes_rz.json | 12 +- .../Python_ohms_law_solver_ion_beam_1d.json | 32 ++-- ...hon_ohms_law_solver_landau_damping_2d.json | 12 +- ...s_law_solver_magnetic_reconnection_2d.json | 20 +-- .../HybridPICModel/HybridPICModel.H | 29 +++- .../HybridPICModel/HybridPICModel.cpp | 153 ++++++++++++++++++ .../FieldSolver/WarpXPushFieldsHybridPIC.cpp | 30 ++-- 17 files changed, 258 insertions(+), 89 deletions(-) diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index 854c4d5ada1..3fb8320c531 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -116,8 +116,7 @@ be interpolated to the correct time, using :math:`\vec{J}_i^n = 1/2(\vec{J}_i^{n The electron pressure is simply calculated using :math:`\rho^n` and the B-field is also already known at the correct time since it was calculated for :math:`t=t_n` at the end of the last step. Once :math:`\vec{E}^n` is calculated, it is used to push :math:`\vec{B}^n` forward in time -(using the Maxwell-Faraday equation, i.e. the same as in the regular PIC routine with ``WarpX::EvolveB()``) -to :math:`\vec{B}^{n+1/2}`. +(using the Maxwell-Faraday equation) to :math:`\vec{B}^{n+1/2}`. Second half step """""""""""""""" @@ -147,10 +146,11 @@ Sub-stepping ^^^^^^^^^^^^ It is also well known that hybrid PIC routines require the B-field to be -updated with a smaller timestep than needed for the particles. The update steps -as outlined above are therefore wrapped in loops that enable the B-field to be -sub-stepped. The exact number of sub-steps used can be specified by the user -through a runtime simulation parameter (see :ref:`input parameters section `). +updated with a smaller timestep than needed for the particles. A 4th order +Runge-Kutta scheme is used to update the B-field. The RK scheme is repeated a +number of times during each half-step outlined above. The number of sub-steps +used can be specified by the user through a runtime simulation parameter +(see :ref:`input parameters section `). .. _theory-hybrid-model-elec-temp: diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index d9b82cf8fe4..493e8307037 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2256,7 +2256,7 @@ Maxwell solver: kinetic-fluid hybrid * ``hybrid_pic_model.n_floor`` (`float`) optional (default ``1``) If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma density floor, in :math:`m^{-3}`, which is useful since the generalized Ohm's law used to calculate the E-field includes a :math:`1/n` term. -* ``hybrid_pic_model.substeps`` (`int`) optional (default ``100``) +* ``hybrid_pic_model.substeps`` (`int`) optional (default ``10``) If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the number of sub-steps to take during the B-field update. .. note:: diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py index 9363bcebd18..4f3cd731323 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py @@ -58,7 +58,7 @@ class EMModes(object): # Plasma resistivity - used to dampen the mode excitation eta = [[1e-7, 1e-7], [1e-7, 1e-5], [1e-7, 1e-4]] # Number of substeps used to update B - substeps = [[250, 500], [250, 750], [250, 1000]] + substeps = 20 def __init__(self, test, dim, B_dir, verbose): """Get input parameters for the specific case desired.""" @@ -149,7 +149,6 @@ def get_simulation_parameters(self): self.NPPC = self.NPPC[self.dim-1] self.eta = self.eta[self.dim-1][idx] - self.substeps = self.substeps[self.dim-1][idx] def get_plasma_quantities(self): """Calculate various plasma parameters based on the simulation input.""" diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py index 21d8cafe750..81ba65e5b28 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py @@ -53,7 +53,7 @@ class CylindricalNormalModes(object): # Plasma resistivity - used to dampen the mode excitation eta = 5e-4 # Number of substeps used to update B - substeps = 250 + substeps = 20 def __init__(self, test, verbose): """Get input parameters for the specific case desired.""" diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py index 49746a73a77..9004c24a05c 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py @@ -154,7 +154,7 @@ def process(it): amps = np.abs(F_kw[2, 1, len(kz)//2-2:len(kz)//2+2]) print("Amplitude sample: ", amps) assert np.allclose( - amps, np.array([61.4170941, 19.39380715, 101.08640009, 11.09261815]) + amps, np.array([61.50945919, 19.74831134, 101.01820349, 10.8974811]) ) if sim.test: diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py index d18c2025e45..aa0cc96d97e 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py @@ -56,7 +56,7 @@ class IonLandauDamping(object): # Plasma resistivity - used to dampen the mode excitation eta = 1e-7 # Number of substeps used to update B - substeps = 100 + substeps = 10 def __init__(self, test, dim, m, T_ratio, verbose): diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py index 7a3b976b092..514f336c482 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py @@ -54,7 +54,7 @@ class HybridPICBeamInstability(object): # Plasma resistivity - used to dampen the mode excitation eta = 1e-7 # Number of substeps used to update B - substeps = 400 + substeps = 10 # Beam parameters n_beam = [0.02, 0.1] @@ -95,11 +95,9 @@ def __init__(self, test, dim, resonant, verbose): diag_period = 1 / 4.0 # Output interval (ion cyclotron periods) self.diag_steps = int(diag_period / self.DT) - # if this is a test case run for only 25 cyclotron periods and use - # fewer substeps to speed up the simulation + # if this is a test case run for only 25 cyclotron periods if self.test: self.LT = 25.0 - self.substeps = 100 self.total_steps = int(np.ceil(self.LT / self.DT)) diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py index 8c5a819a9e0..6a57b1c1046 100755 --- a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py @@ -190,11 +190,11 @@ # assert's fail, the full benchmark should be rerun (same as the test but # without the `--test` argument) and the growth rates (up to saturation) # compared to the theoretical ones to determine if the physics test passes. - # At creation, the full test had the following errors when ran on 8 procs: - # m4_rms_error = 4.476; m5_rms_error = 9.211; m6_rms_error = 3.252 - assert m4_rms_error < 1.55 - assert m5_rms_error < 0.75 - assert m6_rms_error < 0.40 + # At creation, the full test (3d) had the following errors (ran on 1 V100): + # m4_rms_error = 3.329; m5_rms_error = 1.052; m6_rms_error = 2.583 + assert np.isclose(m4_rms_error, 1.515, atol=0.01) + assert np.isclose(m5_rms_error, 0.718, atol=0.01) + assert np.isclose(m6_rms_error, 0.357, atol=0.01) # checksum check import os diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py index 143f1f3e82e..9bcd0584e0f 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py @@ -59,7 +59,7 @@ class ForceFreeSheetReconnection(object): # Plasma resistivity - used to dampen the mode excitation eta = 6e-3 # normalized resistivity # Number of substeps used to update B - substeps = 750 + substeps = 20 def __init__(self, test, verbose): diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json index 5e5369ed9c2..f5606cfee2f 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json @@ -1,17 +1,17 @@ { "lev=0": { - "Bx": 0.0846232979356012, - "By": 0.0748927888009307, + "Bx": 0.08460920626188952, + "By": 0.07485331147996604, "Bz": 256.0, - "Ex": 1960.2420822749025, - "Ey": 2368.794327568534, - "Ez": 4873.496044339846 + "Ex": 1954.3267519602405, + "Ey": 2363.4281756166347, + "Ez": 4873.508158589938 }, "ions": { - "particle_momentum_x": 1.615113301362171e-19, - "particle_momentum_y": 1.6152332353795267e-19, - "particle_momentum_z": 1.6134581585733078e-19, - "particle_position_x": 3678.484650863704, + "particle_momentum_x": 1.6151135948675135e-19, + "particle_momentum_y": 1.6152336151551518e-19, + "particle_momentum_z": 1.6134581268392543e-19, + "particle_position_x": 3678.484650899751, "particle_weight": 4.220251350277737e+21 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json index 870e4a82180..ca50554c394 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json @@ -1,12 +1,12 @@ { "lev=0": {}, "ions": { - "particle_momentum_x": 5.043873837579827e-17, - "particle_momentum_y": 5.04445113937769e-17, - "particle_momentum_z": 5.051930416996439e-17, - "particle_position_x": 143164.4268324243, - "particle_position_y": 143166.5184643828, - "particle_theta": 2573261.7450408577, + "particle_momentum_x": 5.043858996017125e-17, + "particle_momentum_y": 5.0444743275098145e-17, + "particle_momentum_z": 5.05192925609973e-17, + "particle_position_x": 143164.42694236583, + "particle_position_y": 143166.51848290052, + "particle_theta": 2573261.754119082, "particle_weight": 8.12868064536689e+18 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_ion_beam_1d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_ion_beam_1d.json index b0897c7372f..a602eaa68ee 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_ion_beam_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_ion_beam_1d.json @@ -1,27 +1,27 @@ { "lev=0": { - "Bx": 5.334191428317849, - "By": 5.304367780440314, + "Bx": 5.334151697582419, + "By": 5.304303632258862, "Bz": 256.0, - "Ex": 4.023743699806469e+05, - "Ey": 3.989324532762268e+05, - "Ez": 2.281344128344855e+04, - "jx": 1.525180463636894e+11, - "jy": 1.541352360760910e+11, - "jz": 3.182915979017075e+10 + "Ex": 402369.6682213072, + "Ey": 398930.6880468441, + "Ez": 22813.908607048776, + "jx": 152516944704.4641, + "jy": 154134670950.55667, + "jz": 31828054802.990257 }, "beam_ions": { - "particle_momentum_x": 1.267740609380299e-18, - "particle_momentum_y": 1.274270632846882e-18, - "particle_momentum_z": 1.391082122553233e-17, - "particle_position_x": 4.598149111137194e+03, + "particle_momentum_x": 1.2677390613057533e-18, + "particle_momentum_y": 1.2742687194492975e-18, + "particle_momentum_z": 1.3910821615724517e-17, + "particle_position_x": 4598.14909517292, "particle_weight": 2.1101256751388695e+20 }, "ions": { - "particle_momentum_x": 1.619717848597040e-18, - "particle_momentum_y": 1.619530182887040e-18, - "particle_momentum_z": 1.673190364768869e-18, - "particle_position_x": 9.193260427295945e+03, + "particle_momentum_x": 1.6197180437411583e-18, + "particle_momentum_y": 1.6195304119412495e-18, + "particle_momentum_z": 1.6731897345410398e-18, + "particle_position_x": 9193.260490317181, "particle_weight": 1.0550628375694317e+22 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_landau_damping_2d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_landau_damping_2d.json index f16c1a7a96a..e61206d19e3 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_landau_damping_2d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_landau_damping_2d.json @@ -1,19 +1,19 @@ { "lev=0": { "Bx": 0.0, - "By": 7.079680412579356e-06, + "By": 7.079679609748623e-06, "Bz": 0.0, - "Ex": 2726044.053666623, + "Ex": 2726044.0536666242, "Ey": 0.0, - "Ez": 4060168.641409589, - "jx": 177543428.8941277, + "Ez": 4060168.6414095857, + "jx": 177543428.8941278, "jy": 187432087.03814715, - "jz": 594259755.4658134 + "jz": 594259755.4658135 }, "ions": { "particle_momentum_x": 9.141594694084731e-17, "particle_momentum_y": 9.135546407258978e-17, - "particle_momentum_z": 9.137866220861256e-17, + "particle_momentum_z": 9.137866220861254e-17, "particle_position_x": 1197.3344862524336, "particle_position_y": 153269.17690371818, "particle_weight": 8.032598963696067e+16 diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json index c1788127e04..c7b467ad5f4 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json @@ -1,18 +1,18 @@ { "lev=0": { - "Bx": 1524.8789585917777, + "Bx": 1524.8789585554075, "By": 639.8314135126764, - "Bz": 7.47620755452188, - "Ex": 56501626.46458994, - "Ey": 75077196.84777552, - "Ez": 309549625.87520117 + "Bz": 7.476000395458839, + "Ex": 56499935.20497879, + "Ey": 75068107.08471295, + "Ez": 309535670.18599623 }, "ions": { - "particle_momentum_x": 7.14295522521196e-15, - "particle_momentum_y": 7.138078058060595e-15, - "particle_momentum_z": 7.14113444229561e-15, - "particle_position_x": 11170689.203149928, - "particle_position_y": 5585328.083195208, + "particle_momentum_x": 7.14295522755638e-15, + "particle_momentum_y": 7.1380780610425e-15, + "particle_momentum_z": 7.141134469227045e-15, + "particle_position_x": 11170689.203149632, + "particle_position_y": 5585328.083196239, "particle_weight": 9.036667901693183e+18 } } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 750a5e06933..d1931a71765 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -107,6 +107,33 @@ public: std::array< std::unique_ptr, 3> const& edge_lengths, int lev, PatchType patch_type, bool include_resistivity_term); + void BfieldEvolveRK ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, DtType a_dt_type, + amrex::IntVect ng, std::optional nodal_sync); + + void BfieldEvolveRK ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, int lev, DtType dt_type, + amrex::IntVect ng, std::optional nodal_sync); + + void FieldPush ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, DtType dt_type, + amrex::IntVect ng, std::optional nodal_sync); + /** * \brief * Function to calculate the electron pressure at a given timestep type @@ -130,7 +157,7 @@ public: // Declare variables to hold hybrid-PIC model parameters /** Number of substeps to take when evolving B */ - int m_substeps = 100; + int m_substeps = 10; /** Electron temperature in eV */ amrex::Real m_elec_temp; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 2cc6a8ec437..fb7e90f21a1 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -534,3 +534,156 @@ void HybridPICModel::FillElectronPressureMF ( }); } } + +void HybridPICModel::BfieldEvolveRK ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, DtType dt_type, + IntVect ng, std::optional nodal_sync ) +{ + auto& warpx = WarpX::GetInstance(); + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) + { + BfieldEvolveRK( + Bfield, Efield, Jfield, rhofield, edge_lengths, dt, lev, dt_type, + ng, nodal_sync + ); + } +} + +void HybridPICModel::BfieldEvolveRK ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, int lev, DtType dt_type, + IntVect ng, std::optional nodal_sync ) +{ + // Make copies of the B-field multifabs at t = n and create multifabs for + // each direction to store the Runge-Kutta intermediate terms. Each + // multifab has 2 components for the different terms that need to be stored. + std::array< MultiFab, 3 > B_old; + std::array< MultiFab, 3 > K; + for (int ii = 0; ii < 3; ii++) + { + B_old[ii] = MultiFab( + Bfield[lev][ii]->boxArray(), Bfield[lev][ii]->DistributionMap(), 1, + Bfield[lev][ii]->nGrowVect() + ); + MultiFab::Copy(B_old[ii], *Bfield[lev][ii], 0, 0, 1, ng); + + K[ii] = MultiFab( + Bfield[lev][ii]->boxArray(), Bfield[lev][ii]->DistributionMap(), 2, + Bfield[lev][ii]->nGrowVect() + ); + K[ii].setVal(0.0); + } + + // The Runge-Kutta scheme begins here. + // Step 1: + FieldPush( + Bfield, Efield, Jfield, rhofield, edge_lengths, + 0.5_rt*dt, dt_type, ng, nodal_sync + ); + + // The Bfield is now given by: + // B_new = B_old + 0.5 * dt * [-curl x E(B_old)] = B_old + 0.5 * dt * K0. + for (int ii = 0; ii < 3; ii++) + { + // Extract 0.5 * dt * K0 for each direction into index 0 of K. + MultiFab::LinComb( + K[ii], 1._rt, *Bfield[lev][ii], 0, -1._rt, B_old[ii], 0, 0, 1, ng + ); + } + + // Step 2: + FieldPush( + Bfield, Efield, Jfield, rhofield, edge_lengths, + 0.5_rt*dt, dt_type, ng, nodal_sync + ); + + // The Bfield is now given by: + // B_new = B_old + 0.5 * dt * K0 + 0.5 * dt * [-curl x E(B_old + 0.5 * dt * K1)] + // = B_old + 0.5 * dt * K0 + 0.5 * dt * K1 + for (int ii = 0; ii < 3; ii++) + { + // Subtract 0.5 * dt * K0 from the Bfield for each direction, to get + // B_new = B_old + 0.5 * dt * K1. + MultiFab::Subtract(*Bfield[lev][ii], K[ii], 0, 0, 1, ng); + // Extract 0.5 * dt * K1 for each direction into index 1 of K. + MultiFab::LinComb( + K[ii], 1._rt, *Bfield[lev][ii], 0, -1._rt, B_old[ii], 0, 1, 1, ng + ); + } + + // Step 3: + FieldPush( + Bfield, Efield, Jfield, rhofield, edge_lengths, + dt, dt_type, ng, nodal_sync + ); + + // The Bfield is now given by: + // B_new = B_old + 0.5 * dt * K1 + dt * [-curl x E(B_old + 0.5 * dt * K1)] + // = B_old + 0.5 * dt * K1 + dt * K2 + for (int ii = 0; ii < 3; ii++) + { + // Subtract 0.5 * dt * K1 from the Bfield for each direction to get + // B_new = B_old + dt * K2. + MultiFab::Subtract(*Bfield[lev][ii], K[ii], 1, 0, 1, ng); + } + + // Step 4: + FieldPush( + Bfield, Efield, Jfield, rhofield, edge_lengths, + 0.5_rt*dt, dt_type, ng, nodal_sync + ); + + // The Bfield is now given by: + // B_new = B_old + dt * K2 + 0.5 * dt * [-curl x E(B_old + dt * K2)] + // = B_old + dt * K2 + 0.5 * dt * K3 + for (int ii = 0; ii < 3; ii++) + { + // Subtract B_old from the Bfield for each direction, to get + // B = dt * K2 + 0.5 * dt * K3. + MultiFab::Subtract(*Bfield[lev][ii], B_old[ii], 0, 0, 1, ng); + + // Add dt * K2 + 0.5 * dt * K3 to index 0 of K (= 0.5 * dt * K0). + MultiFab::Add(K[ii], *Bfield[lev][ii], 0, 0, 1, ng); + + // Add 2 * 0.5 * dt * K1 to index 0 of K. + MultiFab::LinComb( + K[ii], 1.0, K[ii], 0, 2.0, K[ii], 1, 0, 1, ng + ); + + // Overwrite the Bfield with the Runge-Kutta sum: + // B_new = B_old + 1/3 * dt * (0.5 * K0 + K1 + K2 + 0.5 * K3). + MultiFab::LinComb( + *Bfield[lev][ii], 1.0, B_old[ii], 0, 1.0/3.0, K[ii], 0, 0, 1, ng + ); + } +} + +void HybridPICModel::FieldPush ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, DtType dt_type, + IntVect ng, std::optional nodal_sync ) +{ + auto& warpx = WarpX::GetInstance(); + + // Calculate J = curl x B / mu0 + CalculateCurrentAmpere(Bfield, edge_lengths); + // Calculate the E-field from Ohm's law + HybridPICSolveE(Efield, Jfield, Bfield, rhofield, edge_lengths, true); + warpx.FillBoundaryE(ng, nodal_sync); + // Push forward the B-field using Faraday's law + warpx.EvolveB(dt, dt_type); + warpx.FillBoundaryB(ng, nodal_sync); +} diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 9252671d1ff..50a85a8cdee 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -54,7 +54,7 @@ void WarpX::HybridPICEvolveFields () } // Get requested number of substeps to use - int sub_steps = m_hybrid_pic_model->m_substeps / 2; + int sub_steps = m_hybrid_pic_model->m_substeps; // Get the external current m_hybrid_pic_model->GetCurrentExternal(m_edge_lengths); @@ -68,10 +68,6 @@ void WarpX::HybridPICEvolveFields () // 0'th index of `rho_fp`, J_i^{n-1/2} in `current_fp_temp` and J_i^{n+1/2} // in `current_fp`. - // TODO: To speed up the algorithm insert Runge-Kutta integration logic - // for B update instead of the substep update used here - can test with - // small timestep using this simpler implementation - // Note: E^{n} is recalculated with the accurate J_i^{n} since at the end // of the last step we had to "guess" it. It also needs to be // recalculated to include the resistivity before evolving B. @@ -100,14 +96,12 @@ void WarpX::HybridPICEvolveFields () // momentum equation for (int sub_step = 0; sub_step < sub_steps; sub_step++) { - m_hybrid_pic_model->CalculateCurrentAmpere(Bfield_fp, m_edge_lengths); - m_hybrid_pic_model->HybridPICSolveE( - Efield_fp, current_fp_temp, Bfield_fp, rho_fp_temp, m_edge_lengths, - true + m_hybrid_pic_model->BfieldEvolveRK( + Bfield_fp, Efield_fp, current_fp_temp, rho_fp_temp, + m_edge_lengths, 0.5_rt/sub_steps*dt[0], + DtType::FirstHalf, guard_cells.ng_FieldSolver, + WarpX::sync_nodal_points ); - FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); - EvolveB(0.5_rt / sub_steps * dt[0], DtType::FirstHalf); - FillBoundaryB(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); } // Average rho^{n} and rho^{n+1} to get rho^{n+1/2} in rho_fp_temp @@ -128,14 +122,12 @@ void WarpX::HybridPICEvolveFields () // Now push the B field from t=n+1/2 to t=n+1 using the n+1/2 quantities for (int sub_step = 0; sub_step < sub_steps; sub_step++) { - m_hybrid_pic_model->CalculateCurrentAmpere(Bfield_fp, m_edge_lengths); - m_hybrid_pic_model->HybridPICSolveE( - Efield_fp, current_fp, Bfield_fp, rho_fp_temp, m_edge_lengths, - true + m_hybrid_pic_model->BfieldEvolveRK( + Bfield_fp, Efield_fp, current_fp, rho_fp_temp, + m_edge_lengths, 0.5_rt/sub_steps*dt[0], + DtType::SecondHalf, guard_cells.ng_FieldSolver, + WarpX::sync_nodal_points ); - FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); - EvolveB(0.5_rt / sub_steps * dt[0], DtType::SecondHalf); - FillBoundaryB(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); } // Extrapolate the ion current density to t=n+1 using From c5377b705e5d1caeb2ca37597c7452af88ced059 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:09:03 -0800 Subject: [PATCH 111/176] Update `picmi.ElectrostaticSolver` child class in external Poisson solver test (#4622) * update `picmi.ElectrostaticSolver` child class in external Poisson solver test * add checks that the external solvers were used --- .../capacitive_discharge/PICMI_inputs_1d.py | 8 ++++++-- .../capacitive_discharge/PICMI_inputs_2d.py | 7 +++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index 38b83e46948..01406b7a0fb 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -38,7 +38,7 @@ def __init__(self, grid, **kwargs): required_precision=1, **kwargs ) - def initialize_inputs(self): + def solver_initialize_inputs(self): """Grab geometrical quantities from the grid. The boundary potentials are also obtained from the grid using 'warpx_potential_zmin' for the left_voltage and 'warpx_potential_zmax' for the right_voltage. @@ -57,7 +57,7 @@ def initialize_inputs(self): self.grid.potential_zmin = None self.grid.potential_zmax = None - super(PoissonSolver1D, self).initialize_inputs() + super(PoissonSolver1D, self).solver_initialize_inputs() self.nz = self.grid.number_of_cells[0] self.dz = (self.grid.upper_bound[0] - self.grid.lower_bound[0]) / self.nz @@ -421,6 +421,10 @@ def run_sim(self): self.sim.step(self.diag_steps) + if self.pythonsolver: + # confirm that the external solver was run + assert hasattr(self.solver, 'phi') + if libwarpx.amr.ParallelDescriptor.MyProc() == 0: np.save(f'ion_density_case_{self.n+1}.npy', self.ion_density_array) diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py index 71e1070ef2f..65baabba605 100755 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py @@ -75,7 +75,7 @@ def __init__(self, grid, **kwargs): self.phi_wrapper = None self.time_sum = 0.0 - def initialize_inputs(self): + def solver_initialize_inputs(self): """Grab geometrical quantities from the grid. """ self.right_voltage = self.grid.potential_xmax @@ -89,7 +89,7 @@ def initialize_inputs(self): self.grid.potential_zmin = None self.grid.potential_zmax = None - super(PoissonSolverPseudo1D, self).initialize_inputs() + super(PoissonSolverPseudo1D, self).solver_initialize_inputs() self.nx = self.grid.number_of_cells[0] self.nz = self.grid.number_of_cells[1] @@ -359,3 +359,6 @@ def solve(self): ########################## sim.step(max_steps) + +# confirm that the external solver was run +assert hasattr(solver, 'phi') From 9d35f5e58c034e04f6aca4ef3773d7be27eda0c4 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 22 Jan 2024 16:10:29 -0800 Subject: [PATCH 112/176] Add warpx_used_inputs_file in Python input (#4618) --- Python/pywarpx/picmi.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index b33a1a08899..857f857a554 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1897,6 +1897,9 @@ class Simulation(picmistandard.PICMI_Simulation): warpx_sort_bin_size: list of int, optional (default 1 1 1) If `sort_intervals` is activated and `sort_particles_for_deposition` is false, particles are sorted in bins of `sort_bin_size` cells. In 2D, only the first two elements are read. + + warpx_used_inputs_file: string, optional + The name of the text file that the used input parameters is written to, """ # Set the C++ WarpX interface (see _libwarpx.LibWarpX) as an extension to @@ -1939,6 +1942,7 @@ def init(self, kw): self.sort_particles_for_deposition = kw.pop('warpx_sort_particles_for_deposition', None) self.sort_idx_type = kw.pop('warpx_sort_idx_type', None) self.sort_bin_size = kw.pop('warpx_sort_bin_size', None) + self.used_inputs_file = kw.pop('warpx_used_inputs_file', None) self.collisions = kw.pop('warpx_collisions', None) self.embedded_boundary = kw.pop('warpx_embedded_boundary', None) @@ -1991,6 +1995,7 @@ def initialize_inputs(self): pywarpx.warpx.do_multi_J_n_depositions = self.do_multi_J_n_depositions pywarpx.warpx.serialize_initial_conditions = self.serialize_initial_conditions pywarpx.warpx.random_seed = self.random_seed + pywarpx.warpx.used_inputs_file = self.used_inputs_file pywarpx.warpx.do_dynamic_scheduling = self.do_dynamic_scheduling From 4ab88c5909ac2a0dd300d8ed111bfa04280b3e76 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 23 Jan 2024 01:13:53 +0100 Subject: [PATCH 113/176] Fix undefined behavior issue in WarpX initialization (#4615) * fix out of bound issue found with sanitizer * fixed bug --- Source/WarpX.cpp | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 21a4a2a5234..92623bc2a06 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -288,26 +288,29 @@ WarpX::WarpX () t_old.resize(nlevs_max, std::numeric_limits::lowest()); dt.resize(nlevs_max, std::numeric_limits::max()); - // Loop over species (particles and lasers) - // and set current injection position per species mypc = std::make_unique(this); - const int n_containers = mypc->nContainers(); - for (int i=0; iGetParticleContainer(i); - // Storing injection position for all species, regardless of whether - // they are continuously injected, since it makes looping over the - // elements of current_injection_position easier elsewhere in the code. - if (moving_window_v > 0._rt) - { - // Inject particles continuously from the right end of the box - pc.m_current_injection_position = geom[0].ProbHi(moving_window_dir); - } - else if (moving_window_v < 0._rt) + // Loop over species (particles and lasers) + // and set current injection position per species + if (do_moving_window){ + const int n_containers = mypc->nContainers(); + for (int i=0; iGetParticleContainer(i); + + // Storing injection position for all species, regardless of whether + // they are continuously injected, since it makes looping over the + // elements of current_injection_position easier elsewhere in the code. + if (moving_window_v > 0._rt) + { + // Inject particles continuously from the right end of the box + pc.m_current_injection_position = geom[0].ProbHi(moving_window_dir); + } + else if (moving_window_v < 0._rt) + { + // Inject particles continuously from the left end of the box + pc.m_current_injection_position = geom[0].ProbLo(moving_window_dir); + } } } From 94babdaadab4c75fb6724120819f09d3d7e88f90 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 23 Jan 2024 09:14:45 -0800 Subject: [PATCH 114/176] Doc: Fast, Local Compilation (#4603) * Doc: Fast, Local Compilation Workflows for compiling without or with slow internet access or when working on WarpX and its dependencies at the same time. * Fix Typos Co-authored-by: David Grote Co-authored-by: Luca Fedeli Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * Doc: Rapid Python Dev (no LTO/IPO, no deps) * One more link on `CMAKE_PREFIX_PATH` * CCache Notes * Updates from Dave's Review Comments --------- Co-authored-by: David Grote Co-authored-by: Luca Fedeli Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- Docs/source/developers/local_compile.rst | 138 +++++++++++++++++++++++ Docs/source/developers/workflows.rst | 1 + Docs/source/install/cmake.rst | 3 + Docs/source/install/dependencies.rst | 5 + 4 files changed, 147 insertions(+) create mode 100644 Docs/source/developers/local_compile.rst diff --git a/Docs/source/developers/local_compile.rst b/Docs/source/developers/local_compile.rst new file mode 100644 index 00000000000..8bfa033a92d --- /dev/null +++ b/Docs/source/developers/local_compile.rst @@ -0,0 +1,138 @@ +.. _developers-local-compile: + +Fast, Local Compilation +======================= + +For simplicity, WarpX :ref:`compilation with CMake ` by default downloads, configures and compiles compatible versions of :ref:`central dependencies ` such as: + +* `AMReX `__ +* `PICSAR `__ +* `openPMD-api `__ +* `pyAMReX `__ +* `pybind11 `__ + +on-the-fly, which is called a *superbuild*. + +In some scenarios, e.g., when compiling without internet, with slow internet access, or when working on WarpX and its dependencies, modifications to the superbuild strategy might be preferable. +In the below workflows, you as the developer need to make sure to use compatible versions of the dependencies you provide. + + +.. _developers-local-compile-src: + +Compiling From Local Sources +---------------------------- + +This workflow is best for developers that make changes to WarpX, AMReX, PICSAR, openPMD-api and/or pyAMReX at the same time. +For instance, use this if you add a feature in AMReX and want to try it in WarpX before it is proposed as a pull request for inclusion in AMReX. + +Instead of downloading the source code of the above dependencies, one can also use an already cloned source copy. +For instance, clone these dependencies to ``$HOME/src``: + +.. code-block:: bash + + cd $HOME/src + + git clone https://github.com/ECP-WarpX/WarpX.git warpx + git clone https://github.com/AMReX-Codes/amrex.git + git clone https://github.com/openPMD/openPMD-api.git + git clone https://github.com/ECP-WarpX/picsar.git + git clone https://github.com/AMReX-Codes/pyamrex.git + git clone https://github.com/pybind/pybind11.git + +Now modify the dependencies as needed in their source locations, update sources if you cloned them earlier, etc. +When building WarpX, :ref:`the following CMake flags ` will use the respective local sources: + +.. code-block:: bash + + cd src/warpx + + rm -rf build + + cmake -S . -B build \ + -DWarpX_PYTHON=ON \ + -DWarpX_amrex_src=$HOME/src/amrex \ + -DWarpX_openpmd_src=$HOME/src/openPMD-api \ + -DWarpX_picsar_src=$HOME/src/picsar \ + -DWarpX_pyamrex_src=$HOME/src/pyamrex \ + -DWarpX_pybind11_src=$HOME/src/pybind11 + + cmake --build build -j 8 + cmake --build build -j 8 --target pip_install + + +.. _developers-local-compile-findpackage: + +Compiling With Pre-Compiled Dependencies +---------------------------------------- + +This workflow is the best and fastest to compile WarpX, when you just want to change code in WarpX and have the above central dependencies already made available *in the right configurations* (e.g., w/ or w/o MPI or GPU support) from a :ref:`module system ` or :ref:`package manager `. + +Instead of downloading the source code of the above central dependencies, or using a local copy of their source, we can compile and install those dependencies once. +By setting the `CMAKE_PREFIX_PATH `__ environment variable to the respective dependency install location prefixes, we can instruct CMake to `find their install locations and configurations `__. + +WarpX supports this with :ref:`the following CMake flags `: + +.. code-block:: bash + + cd src/warpx + + rm -rf build + + cmake -S . -B build \ + -DWarpX_PYTHON=ON \ + -DWarpX_amrex_internal=OFF \ + -DWarpX_openpmd_internal=OFF \ + -DWarpX_picsar_internal=OFF \ + -DWarpX_pyamrex_internal=OFF \ + -DWarpX_pybind11_internal=OFF + + cmake --build build -j 8 + cmake --build build -j 8 --target pip_install + +As a background, this is also the workflow how WarpX is built in :ref:`package managers such as Spack and Conda-Forge `. + + +.. _developers-local-compile-pylto: + +Faster Python Builds +-------------------- + +The Python bindings of WarpX and AMReX (pyAMReX) use `pybind11 `__. +Since pybind11 relies heavily on `C++ metaprogramming `__, speeding up the generated binding code requires that we perform a `link-time optimization (LTO) `__ step, also known as `interprocedural optimization (IPO) `__. + +For fast local development cycles, one can skip LTO/IPO with the following flags: + +.. code-block:: bash + + cd src/warpx + + cmake -S . -B build \ + -DWarpX_PYTHON=ON \ + -DWarpX_PYTHON_IPO=OFF \ + -DpyAMReX_IPO=OFF + + cmake --build build -j 8 --target pip_install + +.. note:: + + We might transition to `nanobind `__ in the future, which `does not rely on LTO/IPO `__ for optimal binaries. + You can contribute to `this pyAMReX pull request `__ to help exploring this library (and if it works for the HPC/GPU compilers that we need to support). + +For robustness, our ``pip_install`` target performs a regular ``wheel`` build and then installs it with ``pip``. +This step will check every time of WarpX dependencies are properly installed, to avoid broken installations. +When developing without internet or after the first ``pip_install`` succeeded in repeated installations in rapid development cycles, this check of ``pip`` can be skipped by using the ``pip_install_nodeps`` target instead: + +.. code-block:: bash + + cmake --build build -j 8 --target pip_install_nodeps + + +.. _developers-local-compile-ccache: + +CCache +------ + +WarpX builds will automatically search for `CCache `__ to speed up subsequent compilations in development cycles. +Make sure a :ref:`recent CCache version ` is installed to make use of this feature. + +For power developers that switch a lot between fundamentally different WarpX configurations (e.g., 1D to 3D, GPU and CPU builds, many branches with different bases, developing AMReX and WarpX at the same time), also consider increasing the `CCache cache size `__ and changing the `cache directory `__ if needed, e.g., due to storage quota constraints or to choose a fast(er) filesystem for the cache files. diff --git a/Docs/source/developers/workflows.rst b/Docs/source/developers/workflows.rst index d56974b4101..8b357f5d4da 100644 --- a/Docs/source/developers/workflows.rst +++ b/Docs/source/developers/workflows.rst @@ -10,3 +10,4 @@ Workflows testing documentation checksum + local_compile diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index e299a60c75b..d1dc25cf095 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -147,10 +147,12 @@ Relative paths are also supported, e.g. ``-DWarpX_amrex_src=../amrex``. Or build against an AMReX feature branch of a colleague. Assuming your colleague pushed AMReX to ``https://github.com/WeiqunZhang/amrex/`` in a branch ``new-feature`` then pass to ``cmake`` the arguments: ``-DWarpX_amrex_repo=https://github.com/WeiqunZhang/amrex.git -DWarpX_amrex_branch=new-feature``. +More details on this :ref:`workflow are described here `. You can speed up the install further if you pre-install these dependencies, e.g. with a package manager. Set ``-DWarpX__internal=OFF`` and add installation prefix of the dependency to the environment variable `CMAKE_PREFIX_PATH `__. Please see the :ref:`introduction to CMake ` if this sounds new to you. +More details on this :ref:`workflow are described here `. If you re-compile often, consider installing the `Ninja `__ build system. Pass ``-G Ninja`` to the CMake configuration call to speed up parallel compiles. @@ -327,6 +329,7 @@ This is the workflow most developers will prefer as it allows rapid re-compiles: # build & install Python only cmake --build build -j 4 --target pip_install +There is also a ``--target pip_install_nodeps`` option that :ref:`skips pip-based dependency checks `. WarpX release managers might also want to generate a self-contained source package that can be distributed to exotic architectures: diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index a939db840da..94b2be78bec 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -12,6 +12,11 @@ Please see installation instructions below. - `AMReX `__: we automatically download and compile a copy of AMReX - `PICSAR `__: we automatically download and compile a copy of PICSAR +and for Python bindings: + +- `pyAMReX `__: we automatically download and compile a copy of pyAMReX +- `pybind11 `__: we automatically download and compile a copy of pybind11 + Optional dependencies include: - `MPI 3.0+ `__: for multi-node and/or multi-GPU execution From 47e8ace4c35e204610c7afc0fc7ac83bd4775071 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 23 Jan 2024 17:06:46 -0800 Subject: [PATCH 115/176] AMReX/pyAMReX/PICSAR: Weekly Update (#4630) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index ac80e8d602c..09f7f9e80fa 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 255d30f387cf2c1a7eff5a31f703c94de803e8d8 && cd - + cd ../amrex && git checkout --detach 73b215557c0e842c3e829b683939bbb7a7e12373 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index f6ff47ce23b..f0557e05fc4 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 255d30f387cf2c1a7eff5a31f703c94de803e8d8 +branch = 73b215557c0e842c3e829b683939bbb7a7e12373 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 703882ff2c7..bce811cbfd0 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 255d30f387cf2c1a7eff5a31f703c94de803e8d8 +branch = 73b215557c0e842c3e829b683939bbb7a7e12373 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 080465083f9..31c8919e38f 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "255d30f387cf2c1a7eff5a31f703c94de803e8d8" +set(WarpX_amrex_branch "73b215557c0e842c3e829b683939bbb7a7e12373" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 175d3cab79e..b74b4a0bdf0 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "0283db4c8825b5e094b9184fd42a267ca843c61c" +set(WarpX_pyamrex_branch "47c4364845371cd4f529ebc1ae7120ab7849adb8" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index dc0906cb4af..77980bb00b0 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 255d30f387cf2c1a7eff5a31f703c94de803e8d8 && cd - +cd amrex && git checkout --detach 73b215557c0e842c3e829b683939bbb7a7e12373 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 77dbf6d3a12f0a5ef4c770ad160beb5445c0626b Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 24 Jan 2024 06:58:48 -0800 Subject: [PATCH 116/176] Temporary fix: disable hanging CI test (`Python_background_mcc_1d_tridiag`) (#4628) * Temporary fix: disable hanging CI test (`Python_background_mcc_1d_tridiag`) * only grab test names from lines where the *first* character is `[` * only grab test names from lines where the *first* character is `[` for all cases * also disable DSMC test for the time being --- .github/workflows/source/test_ci_matrix.sh | 16 ++--- Regression/WarpX-tests.ini | 72 +++++++++++----------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/source/test_ci_matrix.sh b/.github/workflows/source/test_ci_matrix.sh index e77d5ce2685..62903b66e45 100755 --- a/.github/workflows/source/test_ci_matrix.sh +++ b/.github/workflows/source/test_ci_matrix.sh @@ -7,26 +7,26 @@ cd Regression/ # Put the name of all CI tests into a text file python prepare_file_ci.py -grep "\[" ci-tests.ini > ci_all_tests.txt +grep "^\[" ci-tests.ini > ci_all_tests.txt export WARPX_CI_PSATD=TRUE # Concatenate the names of all elements in CI matrix into another test file WARPX_CI_REGULAR_CARTESIAN_1D=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini > ci_matrix_elements.txt +grep "^\[" ci-tests.ini > ci_matrix_elements.txt WARPX_CI_REGULAR_CARTESIAN_2D=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt WARPX_CI_REGULAR_CARTESIAN_3D=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt WARPX_CI_SINGLE_PRECISION=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt WARPX_CI_RZ_OR_NOMPI=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt WARPX_CI_QED=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt WARPX_CI_EB=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt # Check that the resulting lists are equal { diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index bce811cbfd0..75939c1fd12 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3021,24 +3021,24 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py -[Python_background_mcc_1d_tridiag] -buildDir = . -inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_1d.py --test -dim = 1 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py +# [Python_background_mcc_1d_tridiag] +# buildDir = . +# inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +# runtime_params = +# customRunCmd = python3 PICMI_inputs_1d.py --test +# dim = 1 +# addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE +# cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +# target = pip_install +# restartTest = 0 +# useMPI = 1 +# numprocs = 2 +# useOMP = 1 +# numthreads = 1 +# compileTest = 0 +# doVis = 0 +# compareParticles = 0 +# analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py [Python_collisionXZ] buildDir = . @@ -3079,24 +3079,24 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py -[Python_dsmc_1d] -buildDir = . -inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py -runtime_params = -customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc -dim = 1 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -target = pip_install -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py +# [Python_dsmc_1d] +# buildDir = . +# inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +# runtime_params = +# customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc +# dim = 1 +# addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE +# cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +# target = pip_install +# restartTest = 0 +# useMPI = 1 +# numprocs = 2 +# useOMP = 1 +# numthreads = 1 +# compileTest = 0 +# doVis = 0 +# compareParticles = 0 +# analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py [Python_ElectrostaticSphereEB] buildDir = . From c1533a967db8feb08fd142a53e270021877993c7 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 24 Jan 2024 07:19:25 -0800 Subject: [PATCH 117/176] Update pyAMReX (#4633) Update to include https://github.com/AMReX-Codes/pyamrex/pull/243 for #3850. --- cmake/dependencies/pyAMReX.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index b74b4a0bdf0..3c043f60226 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "47c4364845371cd4f529ebc1ae7120ab7849adb8" +set(WarpX_pyamrex_branch "cdf03496f6809527b97950e077508ca4b201fa9b" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") From 94ae11900131846e9f8b3704194673f7f02d8959 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 28 Jan 2024 10:37:19 -0800 Subject: [PATCH 118/176] Update Particle Container to Pure SoA (#3850) * Update Particle Container to Pure SoA Transition particle containers to pure SoA layouts. * Python: Pure SoA Particle * Update Particle Container to Pure SoA Transition particle containers to pure SoA layouts. * Python: Pure SoA Particle * fix a couple of places where we underflow the particle idcpu * fixed bad merge * fix bad merge again * Update Particle Container to Pure SoA Transition particle containers to pure SoA layouts. * Python: Pure SoA Particle * skip positions for pure SoA plotfiles * Update Particle Container to Pure SoA Transition particle containers to pure SoA layouts. Co-authored-by: Andrew Myers * Python: Pure SoA Particle * correctly account for positions in restart --------- Co-authored-by: Andrew Myers Co-authored-by: Andrew Myers --- Docs/source/usage/workflows/python_extend.rst | 3 +- .../particle_data_python/PICMI_inputs_2d.py | 6 +- .../PICMI_inputs_prev_pos_2d.py | 6 +- .../PICMI_inputs_runtime_component_analyze.py | 6 +- Python/pywarpx/particle_containers.py | 255 ++++++++---------- .../BackTransformParticleFunctor.H | 16 +- .../FlushFormats/FlushFormatAscent.cpp | 3 - .../FlushFormats/FlushFormatCheckpoint.cpp | 9 +- .../FlushFormats/FlushFormatPlotfile.cpp | 13 +- Source/Diagnostics/ParticleIO.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 6 +- .../FieldProbeParticleContainer.H | 20 +- .../FieldProbeParticleContainer.cpp | 36 +-- .../ReducedDiags/LoadBalanceCosts.cpp | 3 +- Source/Diagnostics/WarpXOpenPMD.H | 4 +- Source/Diagnostics/WarpXOpenPMD.cpp | 139 ++-------- .../ParticleBoundaryProcess.H | 5 +- Source/EmbeddedBoundary/ParticleScraper.H | 3 +- .../BinaryCollision/BinaryCollision.H | 16 +- .../Coulomb/PairWiseCoulombCollisionFunc.H | 7 +- .../Collision/BinaryCollision/DSMC/DSMC.H | 3 +- .../DSMC/SplitAndScatterFunc.H | 30 ++- .../NuclearFusion/NuclearFusionFunc.H | 14 +- .../ProtonBoronFusionInitializeMomentum.H | 10 +- .../TwoProductFusionInitializeMomentum.H | 4 +- .../BinaryCollision/ParticleCreationFunc.H | 60 +++-- .../BinaryCollision/ShuffleFisherYates.H | 2 +- .../Particles/Deposition/ChargeDeposition.H | 2 +- .../Particles/Deposition/CurrentDeposition.H | 2 +- .../ElementaryProcess/QEDPairGeneration.H | 2 +- .../ElementaryProcess/QEDPhotonEmission.H | 16 +- Source/Particles/LaserParticleContainer.H | 7 +- .../NamedComponentParticleContainer.H | 47 +++- Source/Particles/ParticleBoundaryBuffer.cpp | 4 +- Source/Particles/ParticleCreation/SmartCopy.H | 5 +- .../Particles/ParticleCreation/SmartCreate.H | 22 +- .../Particles/ParticleCreation/SmartUtils.H | 9 +- Source/Particles/PhysicalParticleContainer.H | 8 +- .../Particles/PhysicalParticleContainer.cpp | 129 ++++----- Source/Particles/Pusher/GetAndSetPosition.H | 152 +++++++---- .../Particles/Resampling/LevelingThinning.cpp | 5 +- Source/Particles/Sorting/Partition.cpp | 4 +- Source/Particles/Sorting/SortingUtils.H | 41 ++- Source/Particles/Sorting/SortingUtils.cpp | 2 +- Source/Particles/WarpXParticleContainer.H | 14 +- Source/Particles/WarpXParticleContainer.cpp | 89 +++--- .../Particles/ParticleBoundaryBuffer.cpp | 10 +- .../PinnedMemoryParticleContainer.cpp | 2 +- .../Particles/WarpXParticleContainer.cpp | 18 +- Source/Utils/ParticleUtils.H | 7 +- Source/Utils/ParticleUtils.cpp | 26 +- Source/ablastr/particles/IndexHandling.H | 41 --- Source/ablastr/particles/ParticleMoments.H | 25 +- 53 files changed, 661 insertions(+), 709 deletions(-) delete mode 100644 Source/ablastr/particles/IndexHandling.H diff --git a/Docs/source/usage/workflows/python_extend.rst b/Docs/source/usage/workflows/python_extend.rst index 6c9286c02ce..7a3ef0e7af9 100644 --- a/Docs/source/usage/workflows/python_extend.rst +++ b/Docs/source/usage/workflows/python_extend.rst @@ -252,7 +252,8 @@ Particles can be added to the simulation at specific positions and with specific .. autoclass:: pywarpx.particle_containers.ParticleContainerWrapper :members: -The ``get_particle_structs()`` and ``get_particle_arrays()`` functions are called +The ``get_particle_real_arrays()``, ``get_particle_int_arrays()`` and +``get_particle_idcpu_arrays()`` functions are called by several utility functions of the form ``get_particle_{comp_name}`` where ``comp_name`` is one of ``x``, ``y``, ``z``, ``r``, ``theta``, ``id``, ``cpu``, ``weight``, ``ux``, ``uy`` or ``uz``. diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py index a4b7d9e134e..572871b8ed5 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py @@ -153,10 +153,10 @@ def add_particles(): ########################## assert (elec_wrapper.nps == 270 / (2 - args.unique)) -assert (elec_wrapper.particle_container.get_comp_index('w') == 0) -assert (elec_wrapper.particle_container.get_comp_index('newPid') == 4) +assert (elec_wrapper.particle_container.get_comp_index('w') == 2) +assert (elec_wrapper.particle_container.get_comp_index('newPid') == 6) -new_pid_vals = elec_wrapper.get_particle_arrays('newPid', 0) +new_pid_vals = elec_wrapper.get_particle_real_arrays('newPid', 0) for vals in new_pid_vals: assert np.allclose(vals, 5) diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py index 1becd4464e7..5de9879f0f8 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py @@ -120,12 +120,12 @@ elec_count = elec_wrapper.nps # check that the runtime attributes have the right indices -assert (elec_wrapper.particle_container.get_comp_index('prev_x') == 4) -assert (elec_wrapper.particle_container.get_comp_index('prev_z') == 5) +assert (elec_wrapper.particle_container.get_comp_index('prev_x') == 6) +assert (elec_wrapper.particle_container.get_comp_index('prev_z') == 7) # sanity check that the prev_z values are reasonable and # that the correct number of values are returned -prev_z_vals = elec_wrapper.get_particle_arrays('prev_z', 0) +prev_z_vals = elec_wrapper.get_particle_real_arrays('prev_z', 0) running_count = 0 for z_vals in prev_z_vals: diff --git a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py index 706dedb6959..32c9f4e5808 100755 --- a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py +++ b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py @@ -158,10 +158,10 @@ def add_particles(): ########################## assert electron_wrapper.nps == 90 -assert electron_wrapper.particle_container.get_comp_index("w") == 0 -assert electron_wrapper.particle_container.get_comp_index("newPid") == 4 +assert electron_wrapper.particle_container.get_comp_index("w") == 2 +assert electron_wrapper.particle_container.get_comp_index("newPid") == 6 -new_pid_vals = electron_wrapper.get_particle_arrays("newPid", 0) +new_pid_vals = electron_wrapper.get_particle_real_arrays("newPid", 0) for vals in new_pid_vals: assert np.allclose(vals, 5) diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index 273a981f4bd..72a675ec43c 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -117,8 +117,10 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, kwargs[key] = np.full(maxlen, val) # --- The number of built in attributes + # --- The positions + built_in_attrs = libwarpx.dim # --- The three velocities - built_in_attrs = 3 + built_in_attrs += 3 if libwarpx.geometry_dim == 'rz': # --- With RZ, there is also theta built_in_attrs += 1 @@ -188,28 +190,25 @@ def add_real_comp(self, pid_name, comm=True): self.particle_container.add_real_comp(pid_name, comm) - def get_particle_structs(self, level, copy_to_host=False): + def get_particle_real_arrays(self, comp_name, level, copy_to_host=False): ''' - This returns a list of numpy or cupy arrays containing the particle struct data - on each tile for this process. The particle data is represented as a structured - array and contains the particle 'x', 'y', 'z', and 'idcpu'. + This returns a list of numpy or cupy arrays containing the particle real array data + on each tile for this process. - Unless copy_to_host is specified, the data for the structs are - not copied, but share the underlying memory buffer with WarpX. The + Unless copy_to_host is specified, the data for the arrays are not + copied, but share the underlying memory buffer with WarpX. The arrays are fully writeable. - Note that cupy does not support structs: - https://github.com/cupy/cupy/issues/2031 - and will return arrays of binary blobs for the AoS (DP: ``"|V24"``). If copied - to host via copy_to_host, we correct for the right numpy AoS type. - Parameters ---------- - level : int + comp_name : str + The component of the array data that will be returned + + level : int The refinement level to reference (default=0) - copy_to_host : bool + copy_to_host : bool For GPU-enabled runs, one can either return the GPU arrays (the default) or force a device-to-host copy. @@ -217,30 +216,29 @@ def get_particle_structs(self, level, copy_to_host=False): ------- List of arrays - The requested particle struct data + The requested particle array data ''' - particle_data = [] + comp_idx = self.particle_container.get_comp_index(comp_name) + + data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): + soa = pti.soa() + idx = soa.GetRealData(comp_idx) if copy_to_host: - particle_data.append(pti.aos().to_numpy(copy=True)) + data_array.append(idx.to_numpy(copy=True)) else: - if libwarpx.amr.Config.have_gpu: - libwarpx.amr.Print( - "get_particle_structs: cupy does not yet support structs. " - "https://github.com/cupy/cupy/issues/2031" - "Did you mean copy_to_host=True?" - ) xp, cupy_status = load_cupy() if cupy_status is not None: libwarpx.amr.Print(cupy_status) - aos_arr = xp.array(pti.aos(), copy=False) # void blobs for cupy - particle_data.append(aos_arr) - return particle_data + + data_array.append(xp.array(idx, copy=False)) + + return data_array - def get_particle_arrays(self, comp_name, level, copy_to_host=False): + def get_particle_int_arrays(self, comp_name, level, copy_to_host=False): ''' - This returns a list of numpy or cupy arrays containing the particle array data + This returns a list of numpy or cupy arrays containing the particle int array data on each tile for this process. Unless copy_to_host is specified, the data for the arrays are not @@ -266,12 +264,52 @@ def get_particle_arrays(self, comp_name, level, copy_to_host=False): List of arrays The requested particle array data ''' - comp_idx = self.particle_container.get_comp_index(comp_name) + comp_idx = self.particle_container.get_icomp_index(comp_name) data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): soa = pti.soa() - idx = soa.GetRealData(comp_idx) + idx = soa.GetIntData(comp_idx) + if copy_to_host: + data_array.append(idx.to_numpy(copy=True)) + else: + xp, cupy_status = load_cupy() + if cupy_status is not None: + libwarpx.amr.Print(cupy_status) + + data_array.append(xp.array(idx, copy=False)) + + return data_array + + + def get_particle_idcpu_arrays(self, level, copy_to_host=False): + ''' + This returns a list of numpy or cupy arrays containing the particle idcpu data + on each tile for this process. + + Unless copy_to_host is specified, the data for the arrays are not + copied, but share the underlying memory buffer with WarpX. The + arrays are fully writeable. + + Parameters + ---------- + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle array data + ''' + data_array = [] + for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): + soa = pti.soa() + idx = soa.GetIdCPUData() if copy_to_host: data_array.append(idx.to_numpy(copy=True)) else: @@ -284,6 +322,31 @@ def get_particle_arrays(self, comp_name, level, copy_to_host=False): return data_array + def get_particle_idcpu(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle 'idcpu' + numbers on each tile. + + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle idcpu + ''' + return self.get_particle_idcpu_arrays(level, copy_to_host=copy_to_host) + idcpu = property(get_particle_idcpu) + + def get_particle_id(self, level=0, copy_to_host=False): ''' Return a list of numpy or cupy arrays containing the particle 'id' @@ -305,8 +368,8 @@ def get_particle_id(self, level=0, copy_to_host=False): List of arrays The requested particle ids ''' - structs = self.get_particle_structs(level, copy_to_host) - return [libwarpx.amr.unpack_ids(struct['cpuid']) for struct in structs] + idcpu = self.get_particle_idcpu(level, copy_to_host) + return [libwarpx.amr.unpack_ids(tile) for tile in idcpu] def get_particle_cpu(self, level=0, copy_to_host=False): @@ -330,8 +393,8 @@ def get_particle_cpu(self, level=0, copy_to_host=False): List of arrays The requested particle cpus ''' - structs = self.get_particle_structs(level, copy_to_host) - return [libwarpx.amr.unpack_cpus(struct['cpuid']) for struct in structs] + idcpu = self.get_particle_idcpu(level, copy_to_host) + return [libwarpx.amr.unpack_cpus(tile) for tile in idcpu] def get_particle_x(self, level=0, copy_to_host=False): @@ -355,20 +418,7 @@ def get_particle_x(self, level=0, copy_to_host=False): List of arrays The requested particle x position ''' - if copy_to_host: - # Use the numpy version of cosine - xp = np - else: - xp, cupy_status = load_cupy() - - structs = self.get_particle_structs(level, copy_to_host) - if libwarpx.geometry_dim == '3d' or libwarpx.geometry_dim == '2d': - return [struct['x'] for struct in structs] - elif libwarpx.geometry_dim == 'rz': - theta = self.get_particle_theta(level, copy_to_host) - return [struct['x']*xp.cos(theta) for struct, theta in zip(structs, theta)] - elif libwarpx.geometry_dim == '1d': - raise Exception('get_particle_x: There is no x coordinate with 1D Cartesian') + return self.get_particle_real_arrays('x', level, copy_to_host=copy_to_host) xp = property(get_particle_x) @@ -393,20 +443,7 @@ def get_particle_y(self, level=0, copy_to_host=False): List of arrays The requested particle y position ''' - if copy_to_host: - # Use the numpy version of sine - xp = np - else: - xp, cupy_status = load_cupy() - - structs = self.get_particle_structs(level, copy_to_host) - if libwarpx.geometry_dim == '3d': - return [struct['y'] for struct in structs] - elif libwarpx.geometry_dim == 'rz': - theta = self.get_particle_theta(level, copy_to_host) - return [struct['x']*xp.sin(theta) for struct, theta in zip(structs, theta)] - elif libwarpx.geometry_dim == '1d' or libwarpx.geometry_dim == '2d': - raise Exception('get_particle_y: There is no y coordinate with 1D or 2D Cartesian') + return self.get_particle_real_arrays('y', level, copy_to_host=copy_to_host) yp = property(get_particle_y) @@ -433,11 +470,12 @@ def get_particle_r(self, level=0, copy_to_host=False): ''' xp, cupy_status = load_cupy() - structs = self.get_particle_structs(level, copy_to_host) if libwarpx.geometry_dim == 'rz': - return [struct['x'] for struct in structs] + return self.get_particle_x(level, copy_to_host) elif libwarpx.geometry_dim == '3d': - return [xp.sqrt(struct['x']**2 + struct['y']**2) for struct in structs] + x = self.get_particle_x(level, copy_to_host) + y = self.get_particle_y(level, copy_to_host) + return xp.sqrt(x**2 + y**2) elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': raise Exception('get_particle_r: There is no r coordinate with 1D or 2D Cartesian') rp = property(get_particle_r) @@ -467,10 +505,11 @@ def get_particle_theta(self, level=0, copy_to_host=False): xp, cupy_status = load_cupy() if libwarpx.geometry_dim == 'rz': - return self.get_particle_arrays('theta', level, copy_to_host) + return self.get_particle_real_arrays('theta', level, copy_to_host) elif libwarpx.geometry_dim == '3d': - structs = self.get_particle_structs(level, copy_to_host) - return [xp.arctan2(struct['y'], struct['x']) for struct in structs] + x = self.get_particle_x(level, copy_to_host) + y = self.get_particle_y(level, copy_to_host) + return xp.arctan2(y, x) elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': raise Exception('get_particle_theta: There is no theta coordinate with 1D or 2D Cartesian') thetap = property(get_particle_theta) @@ -497,13 +536,7 @@ def get_particle_z(self, level=0, copy_to_host=False): List of arrays The requested particle z position ''' - structs = self.get_particle_structs(level, copy_to_host) - if libwarpx.geometry_dim == '3d': - return [struct['z'] for struct in structs] - elif libwarpx.geometry_dim == 'rz' or libwarpx.geometry_dim == '2d': - return [struct['y'] for struct in structs] - elif libwarpx.geometry_dim == '1d': - return [struct['x'] for struct in structs] + return self.get_particle_real_arrays('z', level, copy_to_host=copy_to_host) zp = property(get_particle_z) @@ -528,7 +561,7 @@ def get_particle_weight(self, level=0, copy_to_host=False): List of arrays The requested particle weight ''' - return self.get_particle_arrays('w', level, copy_to_host=copy_to_host) + return self.get_particle_real_arrays('w', level, copy_to_host=copy_to_host) wp = property(get_particle_weight) @@ -553,7 +586,7 @@ def get_particle_ux(self, level=0, copy_to_host=False): List of arrays The requested particle x momentum ''' - return self.get_particle_arrays('ux', level, copy_to_host=copy_to_host) + return self.get_particle_real_arrays('ux', level, copy_to_host=copy_to_host) uxp = property(get_particle_ux) @@ -578,7 +611,7 @@ def get_particle_uy(self, level=0, copy_to_host=False): List of arrays The requested particle y momentum ''' - return self.get_particle_arrays('uy', level, copy_to_host=copy_to_host) + return self.get_particle_real_arrays('uy', level, copy_to_host=copy_to_host) uyp = property(get_particle_uy) @@ -604,7 +637,7 @@ def get_particle_uz(self, level=0, copy_to_host=False): The requested particle z momentum ''' - return self.get_particle_arrays('uz', level, copy_to_host=copy_to_host) + return self.get_particle_real_arrays('uz', level, copy_to_host=copy_to_host) uzp = property(get_particle_uz) @@ -720,70 +753,6 @@ def get_particle_boundary_buffer_size(self, species_name, boundary, local=False) ) - def get_particle_boundary_buffer_structs( - self, species_name, boundary, level, copy_to_host=False - ): - ''' - This returns a list of numpy or cupy arrays containing the particle struct data - for a species that has been scraped by a specific simulation boundary. The - particle data is represented as a structured array and contains the - particle 'x', 'y', 'z', and 'idcpu'. - - Unless copy_to_host is specified, the data for the structs are - not copied, but share the underlying memory buffer with WarpX. The - arrays are fully writeable. - - Note that cupy does not support structs: - https://github.com/cupy/cupy/issues/2031 - and will return arrays of binary blobs for the AoS (DP: ``"|V24"``). If copied - to host via copy_to_host, we correct for the right numpy AoS type. - - Parameters - ---------- - - species_name : str - The species name that the data will be returned for - - boundary : str - The boundary from which to get the scraped particle data in the - form x/y/z_hi/lo or eb. - - level : int - The refinement level to reference (default=0) - - copy_to_host : bool - For GPU-enabled runs, one can either return the GPU - arrays (the default) or force a device-to-host copy. - - Returns - ------- - - List of arrays - The requested particle struct data - ''' - particle_container = self.particle_buffer.get_particle_container( - species_name, self._get_boundary_number(boundary) - ) - - particle_data = [] - for pti in libwarpx.libwarpx_so.BoundaryBufferParIter(particle_container, level): - if copy_to_host: - particle_data.append(pti.aos().to_numpy(copy=True)) - else: - if libwarpx.amr.Config.have_gpu: - libwarpx.amr.Print( - "get_particle_structs: cupy does not yet support structs. " - "https://github.com/cupy/cupy/issues/2031" - "Did you mean copy_to_host=True?" - ) - xp, cupy_status = load_cupy() - if cupy_status is not None: - libwarpx.amr.Print(cupy_status) - aos_arr = xp.array(pti.aos(), copy=False) # void blobs for cupy - particle_data.append(aos_arr) - return particle_data - - def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level): ''' This returns a list of numpy or cupy arrays containing the particle array data diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H index e88b522f148..fa00f6288f9 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H @@ -143,19 +143,19 @@ struct LorentzTransformParticles const amrex::ParticleReal uzp = uz_old_p * weight_old + uz_new_p * weight_new; #if defined (WARPX_DIM_3D) - dst.m_aos[i_dst].pos(0) = xp; - dst.m_aos[i_dst].pos(1) = yp; - dst.m_aos[i_dst].pos(2) = zp; + dst.m_rdata[PIdx::x][i_dst] = xp; + dst.m_rdata[PIdx::y][i_dst] = yp; + dst.m_rdata[PIdx::z][i_dst] = zp; #elif defined (WARPX_DIM_RZ) - dst.m_aos[i_dst].pos(0) = std::sqrt(xp*xp + yp*yp); - dst.m_aos[i_dst].pos(1) = zp; + dst.m_rdata[PIdx::x][i_dst] = std::sqrt(xp*xp + yp*yp); + dst.m_rdata[PIdx::z][i_dst] = zp; dst.m_rdata[PIdx::theta][i_dst] = std::atan2(yp, xp); #elif defined (WARPX_DIM_XZ) - dst.m_aos[i_dst].pos(0) = xp; - dst.m_aos[i_dst].pos(1) = zp; + dst.m_rdata[PIdx::x][i_dst] = xp; + dst.m_rdata[PIdx::z][i_dst] = zp; amrex::ignore_unused(yp); #elif defined (WARPX_DIM_1D_Z) - dst.m_aos[i_dst].pos(0) = zp; + dst.m_rdata[PIdx::z][i_dst] = zp; amrex::ignore_unused(xp, yp); #else amrex::ignore_unused(xp, yp, zp); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp index 980047e3b46..c224bc4f871 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp @@ -94,9 +94,6 @@ FlushFormatAscent::WriteParticles(const amrex::Vector& particle_di // get names of real comps std::map real_comps_map = pc->getParticleComps(); - // WarpXParticleContainer compile-time extra AoS attributes (Real): 0 - // WarpXParticleContainer compile-time extra AoS attributes (int): 0 - // WarpXParticleContainer compile-time extra SoA attributes (Real): PIdx::nattribs // not an efficient search, but N is small... for(int j = 0; j < PIdx::nattribs; ++j) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 5f59cd723da..00c28eead51 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -178,8 +178,8 @@ FlushFormatCheckpoint::CheckpointParticles ( Vector real_names; Vector int_names; + // note: positions skipped here, since we reconstruct a plotfile SoA from them real_names.push_back("weight"); - real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); @@ -189,9 +189,12 @@ FlushFormatCheckpoint::CheckpointParticles ( #endif // get the names of the real comps - real_names.resize(pc->NumRealComps()); + // note: skips the mandatory AMREX_SPACEDIM positions for pure SoA + real_names.resize(pc->NumRealComps() - AMREX_SPACEDIM); auto runtime_rnames = pc->getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) { real_names[x.second+PIdx::nattribs] = x.first; } + for (auto const& x : runtime_rnames) { + real_names[x.second + PIdx::nattribs - AMREX_SPACEDIM] = x.first; + } // and the int comps int_names.resize(pc->NumIntComps()); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index df73ed34c94..5ec067a6eac 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -355,8 +355,8 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, Vector int_flags; Vector real_flags; + // note: positions skipped here, since we reconstruct a plotfile SoA from them real_names.push_back("weight"); - real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); @@ -366,14 +366,21 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, #endif // get the names of the real comps - real_names.resize(tmp.NumRealComps()); + + // note: skips the mandatory AMREX_SPACEDIM positions for pure SoA + real_names.resize(tmp.NumRealComps() - AMREX_SPACEDIM); auto runtime_rnames = tmp.getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) { real_names[x.second+PIdx::nattribs] = x.first; } + for (auto const& x : runtime_rnames) { + real_names[x.second + PIdx::nattribs - AMREX_SPACEDIM] = x.first; + } // plot any "extra" fields by default real_flags = part_diag.m_plot_flags; real_flags.resize(tmp.NumRealComps(), 1); + // note: skip the mandatory AMREX_SPACEDIM positions for pure SoA + real_flags.erase(real_flags.begin(), real_flags.begin() + AMREX_SPACEDIM); + // and the names int_names.resize(tmp.NumIntComps()); auto runtime_inames = tmp.getParticleRuntimeiComps(); diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index 7ca5e6541d7..a8bb9303fe1 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -160,7 +160,7 @@ MultiParticleContainer::Restart (const std::string& dir) ); } - for (int j = PIdx::nattribs; j < nr; ++j) { + for (int j = PIdx::nattribs-AMREX_SPACEDIM; j < nr; ++j) { const auto& comp_name = real_comp_names[j]; auto current_comp_names = pc->getParticleComps(); auto search = current_comp_names.find(comp_name); diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 9f45392bb0a..24ad0e64ea8 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -431,8 +431,6 @@ void FieldProbe::ComputeDiags (int step) { const auto getPosition = GetParticlePosition(pti); auto setPosition = SetParticlePosition(pti); - const auto& aos = pti.GetArrayOfStructs(); - const auto* AMREX_RESTRICT m_structs = aos().dataPtr(); auto const np = pti.numParticles(); if (update_particles_moving_window) @@ -482,6 +480,8 @@ void FieldProbe::ComputeDiags (int step) ParticleReal* const AMREX_RESTRICT part_Bz = attribs[FieldProbePIdx::Bz].dataPtr(); ParticleReal* const AMREX_RESTRICT part_S = attribs[FieldProbePIdx::S].dataPtr(); + auto * const AMREX_RESTRICT idcpu = pti.GetStructOfArrays().GetIdCPUData().data(); + const auto &xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const std::array &dx = WarpX::CellSize(lev); @@ -556,7 +556,7 @@ void FieldProbe::ComputeDiags (int step) amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); long idx = ip*noutputs; - dvp[idx++] = m_structs[ip].id(); + dvp[idx++] = amrex::ParticleIDWrapper{idcpu[ip]}; // all particles created on IO cpu dvp[idx++] = xp; dvp[idx++] = yp; dvp[idx++] = zp; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H index c85bf8fd541..7d59ade5dc6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H @@ -24,7 +24,14 @@ struct FieldProbePIdx { enum { - Ex = 0, Ey, Ez, +#if !defined (WARPX_DIM_1D_Z) + x, +#endif +#if defined (WARPX_DIM_3D) + y, +#endif + z, + Ex, Ey, Ez, Bx, By, Bz, S, //!< the Poynting vector #ifdef WARPX_DIM_RZ @@ -40,9 +47,14 @@ struct FieldProbePIdx * nattribs tells the particle container to allot 7 SOA values. */ class FieldProbeParticleContainer - : public amrex::ParticleContainer<0, 0, FieldProbePIdx::nattribs> + : public amrex::ParticleContainerPureSoA { public: + static constexpr int NStructReal = 0; + static constexpr int NStructInt = 0; + static constexpr int NReal = FieldProbePIdx::nattribs; + static constexpr int NInt = 0; + FieldProbeParticleContainer (amrex::AmrCore* amr_core); ~FieldProbeParticleContainer() override = default; @@ -52,9 +64,9 @@ public: FieldProbeParticleContainer& operator= ( FieldProbeParticleContainer&& ) = default; //! amrex iterator for our number of attributes - using iterator = amrex::ParIter<0, 0, FieldProbePIdx::nattribs, 0>; + using iterator = amrex::ParIterSoA; //! amrex iterator for our number of attributes (read-only) - using const_iterator = amrex::ParConstIter<0, 0, FieldProbePIdx::nattribs, 0>; + using const_iterator = amrex::ParConstIterSoA; //! similar to WarpXParticleContainer::AddNParticles but does not include u(x,y,z) void AddNParticles (int lev, amrex::Vector const & x, amrex::Vector const & y, amrex::Vector const & z); diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp index 1fd741ddc47..18137fe9b2c 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp @@ -59,7 +59,7 @@ using namespace amrex; FieldProbeParticleContainer::FieldProbeParticleContainer (AmrCore* amr_core) - : ParticleContainer<0, 0, FieldProbePIdx::nattribs>(amr_core->GetParGDB()) + : ParticleContainerPureSoA(amr_core->GetParGDB()) { SetParticleSize(); } @@ -89,33 +89,17 @@ FieldProbeParticleContainer::AddNParticles (int lev, * is then coppied to the permament tile which is stored on the particle * (particle_tile). */ + using PinnedTile = typename ContainerLike::ParticleTileType; - using PinnedTile = ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>; PinnedTile pinned_tile; pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps()); for (int i = 0; i < np; i++) { - ParticleType p; - p.id() = ParticleType::NextID(); - p.cpu() = ParallelDescriptor::MyProc(); -#if defined(WARPX_DIM_3D) - p.pos(0) = x[i]; - p.pos(1) = y[i]; - p.pos(2) = z[i]; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(y); - p.pos(0) = x[i]; - p.pos(1) = z[i]; -#elif defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(x, y); - p.pos(0) = z[i]; -#endif - - // write position, cpu id, and particle id to particle - pinned_tile.push_back(p); + auto & idcpu_data = pinned_tile.GetStructOfArrays().GetIdCPUData(); + idcpu_data.push_back(0); + amrex::ParticleIDWrapper{idcpu_data.back()} = ParticleType::NextID(); + amrex::ParticleCPUWrapper(idcpu_data.back()) = ParallelDescriptor::MyProc(); } // write Real attributes (SoA) to particle initialized zero @@ -125,7 +109,13 @@ FieldProbeParticleContainer::AddNParticles (int lev, #ifdef WARPX_DIM_RZ pinned_tile.push_back_real(FieldProbePIdx::theta, np, 0.0); #endif - +#if !defined (WARPX_DIM_1D_Z) + pinned_tile.push_back_real(FieldProbePIdx::x, x); +#endif +#if defined (WARPX_DIM_3D) + pinned_tile.push_back_real(FieldProbePIdx::y, y); +#endif + pinned_tile.push_back_real(FieldProbePIdx::z, z); pinned_tile.push_back_real(FieldProbePIdx::Ex, np, 0.0); pinned_tile.push_back_real(FieldProbePIdx::Ey, np, 0.0); pinned_tile.push_back_real(FieldProbePIdx::Ez, np, 0.0); diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 893b00a5f00..b4e07b51982 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -56,8 +56,7 @@ namespace auto const & plev = pc.GetParticles(lev); auto const & ptile = plev.at(box_index); - auto const & aos = ptile.GetArrayOfStructs(); - auto const np = aos.numParticles(); + auto const np = ptile.numParticles(); num_macro_particles += np; } diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index e3b7b893d0a..110f1ada649 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -41,7 +41,7 @@ class WarpXParticleCounter { public: using ParticleContainer = typename WarpXParticleContainer::ContainerLike; - using ParticleIter = typename amrex::ParIter<0, 0, PIdx::nattribs, 0, amrex::PinnedArenaAllocator>; + using ParticleIter = typename amrex::ParIterSoA; WarpXParticleCounter (ParticleContainer* pc); [[nodiscard]] unsigned long GetTotalNumParticles () const {return m_Total;} @@ -77,7 +77,7 @@ class WarpXOpenPMDPlot { public: using ParticleContainer = typename WarpXParticleContainer::ContainerLike; - using ParticleIter = typename amrex::ParConstIter<0, 0, PIdx::nattribs, 0, amrex::PinnedArenaAllocator>; + using ParticleIter = typename amrex::ParConstIterSoA; /** Initialize openPMD I/O routines * diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 71d96a47927..d9b70a2e18d 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -18,11 +18,9 @@ #include "WarpX.H" #include "OpenPMDHelpFunction.H" -#include #include #include -#include #include #include #include @@ -548,6 +546,13 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { // see openPMD ED-PIC extension for namings // note: an underscore separates the record name from its component // for non-scalar records +#if !defined (WARPX_DIM_1D_Z) + real_names.push_back("position_x"); +#endif +#if defined (WARPX_DIM_3D) + real_names.push_back("position_y"); +#endif + real_names.push_back("position_z"); real_names.push_back("weighting"); real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); @@ -732,77 +737,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, contributed_particles = true; - // get position and particle ID from aos - // note: this implementation iterates the AoS 4x... - // if we flush late as we do now, we can also copy out the data in one go - const auto &aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile - { - // Save positions -#if defined(WARPX_DIM_RZ) - { - const std::shared_ptr z( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (auto i = 0; i < numParticleOnTile; i++) { - z.get()[i] = aos[i].pos(1); // {0: "r", 1: "z"} - } - std::string const positionComponent = "z"; - currSpecies["position"]["z"].storeChunk(z, {offset}, {numParticleOnTile64}); - } - - // reconstruct x and y from polar coordinates r, theta - auto const& soa = pti.GetStructOfArrays(); - amrex::ParticleReal const* theta = soa.GetRealData(PIdx::theta).dataPtr(); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(theta != nullptr, "openPMD: invalid theta pointer."); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(int(soa.GetRealData(PIdx::theta).size()) == numParticleOnTile, - "openPMD: theta and tile size do not match"); - { - const std::shared_ptr< amrex::ParticleReal > x( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - const std::shared_ptr< amrex::ParticleReal > y( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - for (auto i=0; i curr( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (auto i = 0; i < numParticleOnTile; i++) { - curr.get()[i] = aos[i].pos(currDim); - } - std::string const positionComponent = positionComponents[currDim]; - currSpecies["position"][positionComponent].storeChunk(curr, {offset}, - {numParticleOnTile64}); - } -#endif - - // save particle ID after converting it to a globally unique ID - const std::shared_ptr ids( - new uint64_t[numParticleOnTile], - [](uint64_t const *p) { delete[] p; } - ); - for (auto i = 0; i < numParticleOnTile; i++) { - ids.get()[i] = ablastr::particles::localIDtoGlobal(static_cast(aos[i].id()), static_cast(aos[i].cpu())); - } - const auto *const scalar = openPMD::RecordComponent::SCALAR; - currSpecies["id"][scalar].storeChunk(ids, {offset}, {numParticleOnTile64}); - - } - // save "extra" particle properties in AoS and SoA + // save particle properties SaveRealProperty(pti, currSpecies, offset, @@ -903,10 +838,9 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, std::set< std::string > addedRecords; // add meta-data per record only once for (auto idx=0; idxNumRealComps(); idx++) { - auto ii = ParticleContainer::NStructReal + idx; // jump over extra AoS names - if (write_real_comp[ii]) { + if (write_real_comp[idx]) { // handle scalar and non-scalar records by name - const auto [record_name, component_name] = detail::name2openPMD(real_comp_names[ii]); + const auto [record_name, component_name] = detail::name2openPMD(real_comp_names[idx]); auto currRecord = currSpecies[record_name]; // meta data for ED-PIC extension @@ -927,10 +861,9 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, } } for (auto idx=0; idx( numParticleOnTile ); - auto const& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile + auto const numParticleOnTile64 = static_cast(numParticleOnTile); auto const& soa = pti.GetStructOfArrays(); - // first we concatenate the AoS into contiguous arrays - { - // note: WarpX does not yet use extra AoS Real attributes - for( auto idx=0; idx d( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - - for( auto kk=0; kk AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator() (const PData& ptd, int i, + void operator() (PData& ptd, int i, const amrex::RealVect& /*pos*/, const amrex::RealVect& /*normal*/, amrex::RandomEngine const& /*engine*/) const noexcept { - auto& p = ptd.m_aos[i]; - p.id() = -p.id(); + ptd.id(i) = -ptd.id(i); } }; } diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index d6196c35f44..312b3762a7e 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -170,13 +170,12 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t auto& tile = pti.GetParticleTile(); auto ptd = tile.getParticleTileData(); const auto np = tile.numParticles(); - amrex::Particle<0,0> * const particles = tile.GetArrayOfStructs()().data(); auto phi = (*distance_to_eb[lev])[pti].array(); // signed distance function amrex::ParallelForRNG( np, [=] AMREX_GPU_DEVICE (const int ip, amrex::RandomEngine const& engine) noexcept { // skip particles that are already flagged for removal - if (particles[ip].id() < 0) return; + if (ptd.id(ip) < 0) return; amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 5c90dab25e6..c69f07acdb2 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -72,7 +72,8 @@ class BinaryCollision final // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = amrex::DenseBins; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using index_type = ParticleBins::index_type; @@ -261,9 +262,6 @@ public: const amrex::ParticleReal q1 = species_1.getCharge(); const amrex::ParticleReal m1 = species_1.getMass(); auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); - // Needed to access the particle id - ParticleType * AMREX_RESTRICT - particle_ptr_1 = ptile_1.GetArrayOfStructs()().data(); amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); #if defined WARPX_DIM_1D_Z @@ -371,7 +369,7 @@ public: soa_1, soa_1, product_species_vector, tile_products_data, - particle_ptr_1, particle_ptr_1, m1, m1, + m1, m1, products_mass, p_mask, products_np, copy_species1, copy_species2, p_pair_indices_1, p_pair_indices_2, @@ -403,9 +401,6 @@ public: const amrex::ParticleReal q1 = species_1.getCharge(); const amrex::ParticleReal m1 = species_1.getMass(); auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); - // Needed to access the particle id - ParticleType * AMREX_RESTRICT - particle_ptr_1 = ptile_1.GetArrayOfStructs()().data(); // - Species 2 const auto soa_2 = ptile_2.getParticleTileData(); index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); @@ -413,9 +408,6 @@ public: const amrex::ParticleReal q2 = species_2.getCharge(); const amrex::ParticleReal m2 = species_2.getMass(); auto get_position_2 = GetParticlePosition(ptile_2, getpos_offset); - // Needed to access the particle id - ParticleType * AMREX_RESTRICT - particle_ptr_2 = ptile_2.GetArrayOfStructs()().data(); amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); #if defined WARPX_DIM_1D_Z @@ -535,7 +527,7 @@ public: soa_1, soa_2, product_species_vector, tile_products_data, - particle_ptr_1, particle_ptr_2, m1, m2, + m1, m2, products_mass, p_mask, products_np, copy_species1, copy_species2, p_pair_indices_1, p_pair_indices_2, diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index feb7acf81d3..cfdc36d3c50 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -23,10 +23,13 @@ * \brief This functor performs pairwise Coulomb collision on a single cell by calling the function * ElasticCollisionPerez. It also reads and contains the Coulomb logarithm. */ -class PairWiseCoulombCollisionFunc{ +class PairWiseCoulombCollisionFunc +{ // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H index c1be307b811..ab01eba2c81 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H @@ -38,7 +38,8 @@ class DSMC final // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = amrex::DenseBins; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using index_type = ParticleBins::index_type; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index c1fb7ee7e38..5a7a4f3237a 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -10,6 +10,9 @@ #define SPLIT_AND_SCATTER_FUNC_H_ #include "Particles/Collision/ScatteringProcess.H" +#include "Particles/NamedComponentParticleContainer.H" + +#include /** * \brief Function that performs the particle scattering and injection due @@ -55,8 +58,6 @@ int splitScatteringParticles ( const auto ptile1_data = ptile1.getParticleTileData(); const auto ptile2_data = ptile2.getParticleTileData(); - const Long minus_one_long = -1; - ParallelForRNG(n_total_pairs, [=] AMREX_GPU_DEVICE (int i, RandomEngine const& engine) noexcept { @@ -70,20 +71,37 @@ int splitScatteringParticles ( // starting with the parent particles auto& w1 = ptile1_data.m_rdata[PIdx::w][p_pair_indices_1[i]]; auto& w2 = ptile2_data.m_rdata[PIdx::w][p_pair_indices_2[i]]; + uint64_t* AMREX_RESTRICT idcpu1 = ptile1_data.m_idcpu; + uint64_t* AMREX_RESTRICT idcpu2 = ptile2_data.m_idcpu; + + // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX + // to replace the following lambda. + auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + { + constexpr amrex::Long minus_one_long = -1; + uint64_t tmp = 0; + amrex::ParticleIDWrapper wrapper(tmp); + wrapper = minus_one_long; +#if defined(AMREX_USE_OMP) +#pragma omp atomic write + idcpu = wrapper.m_idata; +#else + auto *old_ptr = reinterpret_cast(&idcpu); + amrex::Gpu::Atomic::Exch(old_ptr, (unsigned long long) wrapper.m_idata); +#endif + }; // Remove p_pair_reaction_weight[i] from the colliding particles' weights. // If the colliding particle weight decreases to zero, remove particle by // setting its id to -1. Gpu::Atomic::AddNoRet(&w1, -p_pair_reaction_weight[i]); if (w1 <= 0._prt) { - auto& p = ptile1_data.m_aos[p_pair_indices_1[i]]; - p.atomicSetID(minus_one_long); + atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); } Gpu::Atomic::AddNoRet(&w2, -p_pair_reaction_weight[i]); if (w2 <= 0._prt) { - auto& p = ptile2_data.m_aos[p_pair_indices_2[i]]; - p.atomicSetID(minus_one_long); + atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); } // Set the child particle properties appropriately diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index 397536b67bf..b2a2112ca68 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -33,10 +33,13 @@ * creation functor. * This functor also reads and contains the fusion multiplier. */ -class NuclearFusionFunc{ +class NuclearFusionFunc +{ // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; @@ -154,12 +157,13 @@ public: // other species and we need to decrease their weight accordingly. // c1 corresponds to the minimum number of times a particle of species 1 will be paired // with a particle of species 2. Same for c2. - const index_type c1 = amrex::max(NI2/NI1,1u); - const index_type c2 = amrex::max(NI1/NI2,1u); + // index_type(1): https://github.com/AMReX-Codes/amrex/pull/3684 + const index_type c1 = amrex::max(NI2/NI1, index_type(1)); + const index_type c2 = amrex::max(NI1/NI2, index_type(1)); // multiplier ratio to take into account unsampled pairs const auto multiplier_ratio = static_cast( - (m_isSameSpecies)?(2u*max_N - 1):(max_N)); + m_isSameSpecies ? 2*max_N - 1 : max_N); #if (defined WARPX_DIM_RZ) amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 7b29267ec32..0b51d6b4b61 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -22,10 +22,12 @@ namespace { // Define shortcuts for frequently-used type names - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; - using index_type = ParticleBins::index_type; + using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; /** * \brief This function initializes the momentum of the alpha particles produced from diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H index be3f5b2d957..52e9db8aa94 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H @@ -24,7 +24,9 @@ namespace { // Define shortcuts for frequently-used type names using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; /** diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index dc830b477df..3928c74223d 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -30,13 +30,15 @@ * \brief This functor creates particles produced from a binary collision and sets their initial * properties (position, momentum, weight). */ -class ParticleCreationFunc{ +class ParticleCreationFunc +{ // Define shortcuts for frequently-used type names - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = amrex::DenseBins; - using index_type = ParticleBins::index_type; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; + using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; public: /** @@ -69,12 +71,6 @@ public: * @param[in, out] soa_1 struct of array data of the first colliding particle species * @param[in, out] soa_2 struct of array data of the second colliding particle species * @param[out] tile_products array containing tile data of the product particles. - * @param[out] particle_ptr_1 pointer to data of the first colliding particle species. Is - * needed to set the id of a particle to -1 in order to delete it when its weight - * reaches 0. - * @param[out] particle_ptr_2 pointer to data of the second colliding particle species. Is - * needed to set the id of a particle to -1 in order to delete it when its weight - * reaches 0. * @param[in] m1 mass of the first colliding particle species * @param[in] m2 mass of the second colliding particle species * @param[in] products_mass array storing the mass of product particles @@ -102,7 +98,6 @@ public: const SoaData_type& soa_1, const SoaData_type& soa_2, const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, - ParticleType* particle_ptr_1, ParticleType* particle_ptr_2, const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, const amrex::Vector& products_mass, const index_type* AMREX_RESTRICT p_mask, @@ -137,6 +132,8 @@ public: amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; + uint64_t* AMREX_RESTRICT idcpu2 = soa_2.m_idcpu; // Create necessary GPU vectors, that will be used in the kernel below amrex::Vector soa_products; @@ -205,16 +202,32 @@ public: amrex::Gpu::Atomic::AddNoRet(&w2[p_pair_indices_2[i]], -p_pair_reaction_weight[i]); + // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX + // to replace the following lambda. + auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + { + constexpr amrex::Long minus_one_long = -1; + uint64_t tmp = 0; + amrex::ParticleIDWrapper wrapper(tmp); + wrapper = minus_one_long; +#if defined(AMREX_USE_OMP) +#pragma omp atomic write + idcpu = wrapper.m_idata; +#else + auto *old_ptr = reinterpret_cast(&idcpu); + amrex::Gpu::Atomic::Exch(old_ptr, (unsigned long long) wrapper.m_idata); +#endif + }; + // If the colliding particle weight decreases to zero, remove particle by // setting its id to -1 - constexpr amrex::Long minus_one_long = -1; if (w1[p_pair_indices_1[i]] <= amrex::ParticleReal(0.)) { - particle_ptr_1[p_pair_indices_1[i]].atomicSetID(minus_one_long); + atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); } if (w2[p_pair_indices_2[i]] <= amrex::ParticleReal(0.)) { - particle_ptr_2[p_pair_indices_2[i]].atomicSetID(minus_one_long); + atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); } // Initialize the product particles' momentum, using a function depending on the @@ -294,12 +307,14 @@ private: * \brief This class does nothing and is used as second template parameter for binary collisions * that do not create particles. */ -class NoParticleCreationFunc{ - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = amrex::DenseBins; - using index_type = ParticleBins::index_type; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; +class NoParticleCreationFunc +{ + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; + using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; public: NoParticleCreationFunc () = default; @@ -313,7 +328,6 @@ public: const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, amrex::Vector& /*pc_products*/, ParticleTileType** /*tile_products*/, - ParticleType* /*particle_ptr_1*/, ParticleType* /*particle_ptr_2*/, const amrex::ParticleReal& /*m1*/, const amrex::ParticleReal& /*m2*/, const amrex::Vector& /*products_mass*/, const index_type* /*p_mask*/, const amrex::Vector& /*products_np*/, diff --git a/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H b/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H index 42259512b0d..3b8f72f4b84 100644 --- a/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H +++ b/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H @@ -12,7 +12,7 @@ /* \brief Shuffle array according to Fisher-Yates algorithm. * Only shuffle the part between is <= i < ie, n = ie-is. * T_index shall be - * amrex::DenseBins::index_type + * amrex::DenseBins::index_type */ template diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index d0db678dfda..d0822789015 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -252,7 +252,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio const int n_rz_azimuthal_modes, amrex::Real* cost, const long load_balance_costs_update_algo, - const amrex::DenseBins& a_bins, + const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, const amrex::IntVect& a_tbox_max_size, diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index efe6efcc788..7343a8c5626 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -592,7 +592,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, int n_rz_azimuthal_modes, amrex::Real* cost, long load_balance_costs_update_algo, - const amrex::DenseBins& a_bins, + const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, const amrex::IntVect& a_tbox_max_size) diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index 5abc9282d4f..ca00b56323a 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -167,7 +167,7 @@ public: p_ux, p_uy, p_uz, engine); - src.m_aos[i_src].id() = -1; //destroy photon after pair generation + amrex::ParticleIDWrapper{src.m_idcpu[i_src]} = -1; // destroy photon after pair generation } private: diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index 8ba5c63ad57..f3099bf997f 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -237,12 +238,11 @@ void cleanLowEnergyPhotons( const int old_size, const int num_added, const amrex::ParticleReal energy_threshold) { - auto pp = ptile.GetArrayOfStructs()().data() + old_size; - - const auto& soa = ptile.GetStructOfArrays(); + auto& soa = ptile.GetStructOfArrays(); + auto p_idcpu = soa.GetIdCPUData().data() + old_size; const auto p_ux = soa.GetRealData(PIdx::ux).data() + old_size; - const auto p_uy = soa.GetRealData(PIdx::uy).data() + old_size; - const auto p_uz = soa.GetRealData(PIdx::uz).data() + old_size; + const auto p_uy = soa.GetRealData(PIdx::uy).data() + old_size; + const auto p_uz = soa.GetRealData(PIdx::uz).data() + old_size; //The square of the energy threshold const auto energy_threshold2 = std::max( @@ -251,8 +251,6 @@ void cleanLowEnergyPhotons( amrex::ParallelFor(num_added, [=] AMREX_GPU_DEVICE (int ip) noexcept { - auto& p = pp[ip]; - const auto ux = p_ux[ip]; const auto uy = p_uy[ip]; const auto uz = p_uz[ip]; @@ -262,8 +260,8 @@ void cleanLowEnergyPhotons( constexpr amrex::ParticleReal me_c = PhysConst::m_e*PhysConst::c; const auto phot_energy2 = (ux*ux + uy*uy + uz*uz)*me_c*me_c; - if (phot_energy2 < energy_threshold2){ - p.id() = - 1; + if (phot_energy2 < energy_threshold2) { + amrex::ParticleIDWrapper{p_idcpu[ip]} = -1; } }); } diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index e6fa308431c..fac94ff20a3 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -56,10 +56,9 @@ public: * \brief Method to initialize runtime attributes. Does nothing for LaserParticleContainer. */ void DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& /*pinned_tile*/, - const int /*n_external_attr_real*/, - const int /*n_external_attr_int*/) final {} + typename ContainerLike::ParticleTileType& /*pinned_tile*/, + int /*n_external_attr_real*/, + int /*n_external_attr_int*/) final {} void ReadHeader (std::istream& is) final; diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index 3be0886425d..e04e7dba6df 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -18,12 +18,19 @@ #include -/** Particle Attributes stored in amrex::ParticleContainer's struct of array +/** Real Particle Attributes stored in amrex::ParticleContainer's struct of array */ struct PIdx { enum { - w = 0, ///< weight +#if !defined (WARPX_DIM_1D_Z) + x, +#endif +#if defined (WARPX_DIM_3D) + y, +#endif + z, + w, ///< weight ux, uy, uz, #ifdef WARPX_DIM_RZ theta, ///< RZ needs all three position components @@ -32,6 +39,15 @@ struct PIdx }; }; +/** Integer Particle Attributes stored in amrex::ParticleContainer's struct of array + */ +struct PIdxInt +{ + enum { + nattribs ///< number of attributes + }; +}; + /** Particle Container class that allows to add/access particle components * with a name (string) instead of doing so with an integer index. * (The "components" are all the particle quantities - except those @@ -45,11 +61,11 @@ struct PIdx */ template class T_Allocator=amrex::DefaultAllocator> class NamedComponentParticleContainer : -public amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator> +public amrex::ParticleContainerPureSoA { public: /** Construct an empty NamedComponentParticleContainer **/ - NamedComponentParticleContainer () : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>() {} + NamedComponentParticleContainer () : amrex::ParticleContainerPureSoA() {} /** Construct a NamedComponentParticleContainer from an AmrParGDB object * @@ -61,8 +77,15 @@ public: * AMR hierarchy. Usually, this is generated by an AmrCore or AmrLevel object. */ NamedComponentParticleContainer (amrex::AmrParGDB* amr_pgdb) - : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>(amr_pgdb) { + : amrex::ParticleContainerPureSoA(amr_pgdb) { // build up the map of string names to particle component numbers +#if !defined (WARPX_DIM_1D_Z) + particle_comps["x"] = PIdx::x; +#endif +#if defined (WARPX_DIM_3D) + particle_comps["y"] = PIdx::y; +#endif + particle_comps["z"] = PIdx::z; particle_comps["w"] = PIdx::w; particle_comps["ux"] = PIdx::ux; particle_comps["uy"] = PIdx::uy; @@ -85,12 +108,12 @@ public: * @param p_ricomps name-to-index map for run-time integer components */ NamedComponentParticleContainer( - amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator> && pc, + amrex::ParticleContainerPureSoA && pc, std::map p_comps, std::map p_icomps, std::map p_rcomps, std::map p_ricomps) - : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>(std::move(pc)), + : amrex::ParticleContainerPureSoA(std::move(pc)), particle_comps(std::move(p_comps)), particle_icomps(std::move(p_icomps)), particle_runtime_comps(std::move(p_rcomps)), @@ -118,7 +141,7 @@ public: NamedComponentParticleContainer make_alike () const { auto tmp = NamedComponentParticleContainer( - amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::template make_alike(), + amrex::ParticleContainerPureSoA::template make_alike(), particle_comps, particle_icomps, particle_runtime_comps, @@ -127,10 +150,10 @@ public: return tmp; } - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::NumRealComps; - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::NumIntComps; - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::AddRealComp; - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::AddIntComp; + using amrex::ParticleContainerPureSoA::NumRealComps; + using amrex::ParticleContainerPureSoA::NumIntComps; + using amrex::ParticleContainerPureSoA::AddRealComp; + using amrex::ParticleContainerPureSoA::AddIntComp; /** Allocate a new run-time real component * diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 54c4396379d..88304bd8a9c 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -50,7 +50,7 @@ struct CopyAndTimestamp { void operator() (const DstData& dst, const SrcData& src, int src_i, int dst_i) const noexcept { - dst.m_aos[dst_i] = src.m_aos[src_i]; + dst.m_idcpu[dst_i] = src.m_idcpu[src_i]; for (int j = 0; j < SrcData::NAR; ++j) { dst.m_rdata[j][dst_i] = src.m_rdata[j][src_i]; } @@ -222,7 +222,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles"); - using PIter = amrex::ParConstIter<0,0,PIdx::nattribs>; + using PIter = amrex::ParConstIterSoA; const auto& warpx_instance = WarpX::GetInstance(); const amrex::Geometry& geom = warpx_instance.Geom(0); auto plo = geom.ProbLoArray(); diff --git a/Source/Particles/ParticleCreation/SmartCopy.H b/Source/Particles/ParticleCreation/SmartCopy.H index 2c04baa18bb..6a6ceb3d290 100644 --- a/Source/Particles/ParticleCreation/SmartCopy.H +++ b/Source/Particles/ParticleCreation/SmartCopy.H @@ -26,7 +26,7 @@ * type. Second, if a given component name is found in both the src * and the dst, then the src value is copied. * - * Particle structs - positions and id numbers - are always copied. + * Particle positions and id numbers are always copied. * * You don't create this directly - use the SmartCopyFactory object below. */ @@ -48,9 +48,6 @@ struct SmartCopy void operator() (DstData& dst, const SrcData& src, int i_src, int i_dst, amrex::RandomEngine const& engine) const noexcept { - // the particle struct is always copied over - dst.m_aos[i_dst] = src.m_aos[i_src]; - // initialize the real components for (int j = 0; j < DstData::NAR; ++j) { dst.m_rdata[j][i_dst] = initializeRealValue(m_policy_real[j], engine); diff --git a/Source/Particles/ParticleCreation/SmartCreate.H b/Source/Particles/ParticleCreation/SmartCreate.H index 67d7767a5d3..7fb854ee083 100644 --- a/Source/Particles/ParticleCreation/SmartCreate.H +++ b/Source/Particles/ParticleCreation/SmartCreate.H @@ -14,6 +14,8 @@ #include #include #include +#include +#include /** * \brief This is a functor for performing a "smart create" that works @@ -47,23 +49,23 @@ struct SmartCreate const int id = 0) const noexcept { #if defined(WARPX_DIM_3D) - prt.m_aos[i_prt].pos(0) = x; - prt.m_aos[i_prt].pos(1) = y; - prt.m_aos[i_prt].pos(2) = z; + prt.m_rdata[PIdx::x][i_prt] = x; + prt.m_rdata[PIdx::y][i_prt] = y; + prt.m_rdata[PIdx::z][i_prt] = z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - prt.m_aos[i_prt].pos(0) = x; - prt.m_aos[i_prt].pos(1) = z; + prt.m_rdata[PIdx::x][i_prt] = x; + prt.m_rdata[PIdx::z][i_prt] = z; amrex::ignore_unused(y); #else - prt.m_aos[i_prt].pos(0) = z; + prt.m_rdata[PIdx::z][i_prt] = z; amrex::ignore_unused(x,y); #endif - prt.m_aos[i_prt].cpu() = cpu; - prt.m_aos[i_prt].id() = id; + amrex::ParticleIDWrapper{prt.m_idcpu[i_prt]} = id; + amrex::ParticleCPUWrapper{prt.m_idcpu[i_prt]} = cpu; - // initialize the real components - for (int j = 0; j < PartData::NAR; ++j) { + // initialize the real components after position + for (int j = AMREX_SPACEDIM; j < PartData::NAR; ++j) { prt.m_rdata[j][i_prt] = initializeRealValue(m_policy_real[j], engine); } for (int j = 0; j < prt.m_num_runtime_real; ++j) { diff --git a/Source/Particles/ParticleCreation/SmartUtils.H b/Source/Particles/ParticleCreation/SmartUtils.H index 732a12bb729..6b3d396900d 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.H +++ b/Source/Particles/ParticleCreation/SmartUtils.H @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -60,12 +61,12 @@ void setNewParticleIDs (PTile& ptile, int old_size, int num_added) } const int cpuid = amrex::ParallelDescriptor::MyProc(); - auto pp = ptile.GetArrayOfStructs()().data() + old_size; + auto ptd = ptile.getParticleTileData(); amrex::ParallelFor(num_added, [=] AMREX_GPU_DEVICE (int ip) noexcept { - auto& p = pp[ip]; - p.id() = pid+ip; - p.cpu() = cpuid; + auto const new_id = ip + old_size; + amrex::ParticleIDWrapper{ptd.m_idcpu[new_id]} = pid+ip; + amrex::ParticleCPUWrapper{ptd.m_idcpu[new_id]} = cpuid; }); } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index a12ae75f629..edf91a84526 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -268,11 +268,9 @@ public: * @param[in] engine the random engine, used in initialization of QED optical depths */ void DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>& pinned_tile, - int n_external_attr_real, - int n_external_attr_int) final; + typename ContainerLike::ParticleTileType& pinned_tile, + int n_external_attr_real, + int n_external_attr_int) final; /** * \brief Apply NCI Godfrey filter to all components of E and B before gather diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 0d9180c090e..fd55e369040 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -194,8 +194,8 @@ namespace * and avoid any possible undefined behavior before the next call to redistribute) and sets * the particle id to -1 so that it can be effectively deleted. * - * \param p particle aos data - * \param pa particle soa data + * \param idcpu particle id soa data + * \param pa particle real soa data * \param ip index for soa data * \param do_field_ionization whether species has ionization * \param pi ionization level data @@ -206,20 +206,21 @@ namespace */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void ZeroInitializeAndSetNegativeID ( - ParticleType& p, const GpuArray& pa, long& ip, + uint64_t * AMREX_RESTRICT idcpu, + const GpuArray& pa, long& ip, const bool& do_field_ionization, int* pi #ifdef WARPX_QED - ,const bool& has_quantum_sync, amrex::ParticleReal* p_optical_depth_QSR - ,const bool& has_breit_wheeler, amrex::ParticleReal* p_optical_depth_BW + ,const bool& has_quantum_sync, amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR + ,const bool& has_breit_wheeler, amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_BW #endif ) noexcept { - p.pos(0) = 0._rt; + pa[PIdx::z][ip] = 0._rt; #if (AMREX_SPACEDIM >= 2) - p.pos(1) = 0._rt; + pa[PIdx::x][ip] = 0._rt; #endif #if defined(WARPX_DIM_3D) - p.pos(2) = 0._rt; + pa[PIdx::y][ip] = 0._rt; #endif pa[PIdx::w ][ip] = 0._rt; pa[PIdx::ux][ip] = 0._rt; @@ -234,7 +235,7 @@ namespace if (has_breit_wheeler) {p_optical_depth_BW[ip] = 0._rt;} #endif - p.id() = -1; + amrex::ParticleIDWrapper{idcpu[ip]} = -1; } } @@ -776,11 +777,9 @@ PhysicalParticleContainer::AddPlasmaFromFile(PlasmaInjector & plasma_injector, void PhysicalParticleContainer::DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>& pinned_tile, - const int n_external_attr_real, - const int n_external_attr_int) + typename ContainerLike::ParticleTileType& pinned_tile, + int n_external_attr_real, + int n_external_attr_int) { ParticleCreation::DefaultInitializeRuntimeAttributes(pinned_tile, n_external_attr_real, n_external_attr_int, @@ -1080,7 +1079,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - Long pid; + int pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1089,7 +1088,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid + max_new_particles) < LastParticleID, + static_cast(pid) + static_cast(max_new_particles) < LongParticleIds::LastParticleID, "ERROR: overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); @@ -1100,16 +1099,16 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int DefineAndReturnParticleTile(lev, grid_id, tile_id); } - auto old_size = particle_tile.GetArrayOfStructs().size(); + auto old_size = particle_tile.size(); auto new_size = old_size + max_new_particles; particle_tile.resize(new_size); - ParticleType* pp = particle_tile.GetArrayOfStructs()().data() + old_size; auto& soa = particle_tile.GetStructOfArrays(); GpuArray pa; for (int ia = 0; ia < PIdx::nattribs; ++ia) { pa[ia] = soa.GetRealData(ia).data() + old_size; } + uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); @@ -1222,9 +1221,8 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int for (int i_part = 0; i_part < pcounts[index]; ++i_part) { long ip = poffset[index] + i_part; - ParticleType& p = pp[ip]; - p.id() = pid+ip; - p.cpu() = cpuid; + amrex::ParticleIDWrapper{pa_idcpu[ip]} = pid+ip; + amrex::ParticleCPUWrapper{pa_idcpu[ip]} = cpuid; const XDim3 r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? // In the refined injection region: use refinement ratio `lrrfac` inj_pos->getPositionUnitBox(i_part, lrrfac, engine) : @@ -1234,7 +1232,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int #if defined(WARPX_DIM_3D) if (!tile_realbox.contains(XDim3{pos.x,pos.y,pos.z})) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1245,7 +1243,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(k); if (!tile_realbox.contains(XDim3{pos.x,pos.z,0.0_rt})) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1256,7 +1254,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int #else amrex::ignore_unused(j,k); if (!tile_realbox.contains(XDim3{pos.z,0.0_rt,0.0_rt})) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1295,7 +1293,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const Real z0 = applyBallisticCorrection(pos, inj_mom, gamma_boost, beta_boost, t); if (!inj_pos->insideBounds(xb, yb, z0)) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1309,7 +1307,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // Remove particle if density below threshold if ( dens < density_min ){ - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1327,7 +1325,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // If the particle is not within the lab-frame zmin, zmax, etc. // go to the next generated particle. if (!inj_pos->insideBounds(xb, yb, z0_lab)) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1339,7 +1337,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int dens = inj_rho->getDensity(pos.x, pos.y, z0_lab); // Remove particle if density below threshold if ( dens < density_min ){ - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1406,17 +1404,17 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int pa[PIdx::uz][ip] = u.z; #if defined(WARPX_DIM_3D) - p.pos(0) = pos.x; - p.pos(1) = pos.y; - p.pos(2) = pos.z; + pa[PIdx::x][ip] = pos.x; + pa[PIdx::y][ip] = pos.y; + pa[PIdx::z][ip] = pos.z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) #ifdef WARPX_DIM_RZ pa[PIdx::theta][ip] = theta; #endif - p.pos(0) = xb; - p.pos(1) = pos.z; + pa[PIdx::x][ip] = xb; + pa[PIdx::z][ip] = pos.z; #else - p.pos(0) = pos.z; + pa[PIdx::z][ip] = pos.z; #endif } }); @@ -1635,7 +1633,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - Long pid; + int pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1644,23 +1642,23 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid + max_new_particles) < LastParticleID, + static_cast(pid) + static_cast(max_new_particles) < LongParticleIds::LastParticleID, "overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); auto& particle_tile = tmp_pc.DefineAndReturnParticleTile(0, grid_id, tile_id); - auto old_size = particle_tile.GetArrayOfStructs().size(); + auto old_size = particle_tile.size(); auto new_size = old_size + max_new_particles; particle_tile.resize(new_size); - ParticleType* pp = particle_tile.GetArrayOfStructs()().data() + old_size; auto& soa = particle_tile.GetStructOfArrays(); GpuArray pa; for (int ia = 0; ia < PIdx::nattribs; ++ia) { pa[ia] = soa.GetRealData(ia).data() + old_size; } + uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); @@ -1758,9 +1756,8 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, for (int i_part = 0; i_part < pcounts[index]; ++i_part) { const long ip = poffset[index] + i_part; - ParticleType& p = pp[ip]; - p.id() = pid+ip; - p.cpu() = cpuid; + amrex::ParticleIDWrapper{pa_idcpu[ip]} = pid+ip; + amrex::ParticleCPUWrapper{pa_idcpu[ip]} = cpuid; // This assumes the flux_pos is of type InjectorPositionRandomPlane const XDim3 r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? @@ -1785,19 +1782,19 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // the particles will be within the domain. #if defined(WARPX_DIM_3D) if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.x,ppos.y,ppos.z})) { - p.id() = -1; + amrex::ParticleIDWrapper{pa_idcpu[ip]} = -1; continue; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(k); if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.x,ppos.z,0.0_prt})) { - p.id() = -1; + amrex::ParticleIDWrapper{pa_idcpu[ip]} = -1; continue; } #else amrex::ignore_unused(j,k); if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.z,0.0_prt,0.0_prt})) { - p.id() = -1; + amrex::ParticleIDWrapper{pa_idcpu[ip]} = -1; continue; } #endif @@ -1805,7 +1802,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // If the particle's initial position is not within or on the species's // xmin, xmax, ymin, ymax, zmin, zmax, go to the next generated particle. if (!flux_pos->insideBoundsInclusive(ppos.x, ppos.y, ppos.z)) { - p.id() = -1; + amrex::ParticleIDWrapper{pa_idcpu[ip]} = -1; continue; } @@ -1839,7 +1836,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, Real flux = inj_flux->getFlux(ppos.x, ppos.y, ppos.z, t); // Remove particle if flux is negative or 0 if ( flux <=0 ){ - p.id() = -1; + amrex::ParticleIDWrapper{pa_idcpu[ip]} = -1; continue; } @@ -1898,18 +1895,18 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, UpdatePosition(ppos.x, ppos.y, ppos.z, pu.x, pu.y, pu.z, t_fract); #if defined(WARPX_DIM_3D) - p.pos(0) = ppos.x; - p.pos(1) = ppos.y; - p.pos(2) = ppos.z; + pa[PIdx::x][ip] = ppos.x; + pa[PIdx::y][ip] = ppos.y; + pa[PIdx::z][ip] = ppos.z; #elif defined(WARPX_DIM_RZ) pa[PIdx::theta][ip] = std::atan2(ppos.y, ppos.x); - p.pos(0) = std::sqrt(ppos.x*ppos.x + ppos.y*ppos.y); - p.pos(1) = ppos.z; + pa[PIdx::x][ip] = std::sqrt(ppos.x*ppos.x + ppos.y*ppos.y); + pa[PIdx::z][ip] = ppos.z; #elif defined(WARPX_DIM_XZ) - p.pos(0) = ppos.x; - p.pos(1) = ppos.z; + pa[PIdx::x][ip] = ppos.x; + pa[PIdx::z][ip] = ppos.z; #else - p.pos(0) = ppos.z; + pa[PIdx::z][ip] = ppos.z; #endif } }); @@ -2326,20 +2323,22 @@ PhysicalParticleContainer::SplitParticles (int lev) split_offset[1] /= ppc_nd[1]; split_offset[2] /= ppc_nd[2]; } - // particle Array Of Structs data - auto& particles = pti.GetArrayOfStructs(); // particle Struct Of Arrays data auto& attribs = pti.GetAttribs(); auto& wp = attribs[PIdx::w ]; auto& uxp = attribs[PIdx::ux]; auto& uyp = attribs[PIdx::uy]; auto& uzp = attribs[PIdx::uz]; + + ParticleTileType& ptile = ParticlesAt(lev, pti); + auto& soa = ptile.GetStructOfArrays(); + uint64_t * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); + const long np = pti.numParticles(); for(int i=0; i> attr_int; pctmp_split.AddNParticles(lev, np_split_to_add, - xp, yp, zp, uxp, uyp, uzp, - 1, attr, + xp, + yp, + zp, + uxp, + uyp, + uzp, + 1, + attr, 0, attr_int, - 1, NoSplitParticleID); + 1, LongParticleIds::NoSplitParticleID); // Copy particles from tmp to current particle container constexpr bool local_flag = true; addParticles(pctmp_split,local_flag); diff --git a/Source/Particles/Pusher/GetAndSetPosition.H b/Source/Particles/Pusher/GetAndSetPosition.H index e4477a2a60d..44641557756 100644 --- a/Source/Particles/Pusher/GetAndSetPosition.H +++ b/Source/Particles/Pusher/GetAndSetPosition.H @@ -30,24 +30,26 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, amrex::ParticleReal& y, amrex::ParticleReal& z) noexcept { -#ifdef WARPX_DIM_RZ - const amrex::ParticleReal theta = p.rdata(T_PIdx::theta); - const amrex::ParticleReal r = p.pos(0); + using namespace amrex::literals; + +#if defined(WARPX_DIM_RZ) + amrex::ParticleReal const theta = p.rdata(T_PIdx::theta); + amrex::ParticleReal const r = p.pos(T_PIdx::x); x = r*std::cos(theta); y = r*std::sin(theta); - z = p.pos(1); -#elif WARPX_DIM_3D - x = p.pos(0); - y = p.pos(1); - z = p.pos(2); -#elif WARPX_DIM_XZ - x = p.pos(0); - y = amrex::ParticleReal(0.0); - z = p.pos(1); + z = p.pos(PIdx::z); +#elif defined(WARPX_DIM_3D) + x = p.pos(PIdx::x); + y = p.pos(PIdx::y); + z = p.pos(PIdx::z); +#elif defined(WARPX_DIM_XZ) + x = p.pos(PIdx::x); + y = 0_prt; + z = p.pos(PIdx::z); #else - x = amrex::ParticleReal(0.0); - y = amrex::ParticleReal(0.0); - z = p.pos(0); + x = 0_prt; + y = 0_prt; + z = p.pos(PIdx::z); #endif } @@ -59,10 +61,19 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, template struct GetParticlePosition { - using PType = WarpXParticleContainer::ParticleType; using RType = amrex::ParticleReal; - const PType* AMREX_RESTRICT m_structs = nullptr; +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + const RType* AMREX_RESTRICT m_x = nullptr; + const RType* AMREX_RESTRICT m_z = nullptr; +#elif defined(WARPX_DIM_3D) + const RType* AMREX_RESTRICT m_x = nullptr; + const RType* AMREX_RESTRICT m_y = nullptr; + const RType* AMREX_RESTRICT m_z = nullptr; +#elif defined(WARPX_DIM_1D_Z) + const RType* AMREX_RESTRICT m_z = nullptr; +#endif + #if defined(WARPX_DIM_RZ) const RType* m_theta = nullptr; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) @@ -84,10 +95,19 @@ struct GetParticlePosition template GetParticlePosition (const ptiType& a_pti, long a_offset = 0) noexcept { - const auto& aos = a_pti.GetArrayOfStructs(); - m_structs = aos().dataPtr() + a_offset; -#if defined(WARPX_DIM_RZ) const auto& soa = a_pti.GetStructOfArrays(); + +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_3D) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_y = soa.GetRealData(PIdx::y).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_1D_Z) + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#endif +#if defined(WARPX_DIM_RZ) m_theta = soa.GetRealData(T_PIdx::theta).dataPtr() + a_offset; #endif } @@ -98,24 +118,23 @@ struct GetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void operator() (const long i, RType& x, RType& y, RType& z) const noexcept { - const PType& p = m_structs[i]; #ifdef WARPX_DIM_RZ - const RType r = p.pos(0); + RType const r = m_x[i]; x = r*std::cos(m_theta[i]); y = r*std::sin(m_theta[i]); - z = p.pos(1); + z = m_z[i]; #elif WARPX_DIM_3D - x = p.pos(0); - y = p.pos(1); - z = p.pos(2); + x = m_x[i]; + y = m_y[i]; + z = m_z[i]; #elif WARPX_DIM_XZ - x = p.pos(0); + x = m_x[i]; y = m_y_default; - z = p.pos(1); + z = m_z[i]; #else x = m_x_default; y = m_y_default; - z = p.pos(0); + z = m_z[i]; #endif } @@ -127,23 +146,22 @@ struct GetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void AsStored (const long i, RType& x, RType& y, RType& z) const noexcept { - const PType& p = m_structs[i]; #ifdef WARPX_DIM_RZ - x = p.pos(0); + x = m_x[i]; y = m_theta[i]; - z = p.pos(1); + z = m_z[i]; #elif WARPX_DIM_3D - x = p.pos(0); - y = p.pos(1); - z = p.pos(2); + x = m_x[i]; + y = m_y[i]; + z = m_z[i]; #elif WARPX_DIM_XZ - x = p.pos(0); + x = m_x[i]; y = m_y_default; - z = p.pos(1); + z = m_z[i]; #else x = m_x_default; y = m_y_default; - z = p.pos(0); + z = m_z[i]; #endif } }; @@ -158,10 +176,18 @@ struct GetParticlePosition template struct SetParticlePosition { - using PType = WarpXParticleContainer::ParticleType; using RType = amrex::ParticleReal; - PType* AMREX_RESTRICT m_structs; +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + RType* AMREX_RESTRICT m_x; + RType* AMREX_RESTRICT m_z; +#elif defined(WARPX_DIM_3D) + RType* AMREX_RESTRICT m_x; + RType* AMREX_RESTRICT m_y; + RType* AMREX_RESTRICT m_z; +#elif defined(WARPX_DIM_1D_Z) + RType* AMREX_RESTRICT m_z; +#endif #if defined(WARPX_DIM_RZ) RType* AMREX_RESTRICT m_theta; #endif @@ -169,10 +195,18 @@ struct SetParticlePosition template SetParticlePosition (const ptiType& a_pti, long a_offset = 0) noexcept { - auto& aos = a_pti.GetArrayOfStructs(); - m_structs = aos().dataPtr() + a_offset; -#if defined(WARPX_DIM_RZ) auto& soa = a_pti.GetStructOfArrays(); +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_3D) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_y = soa.GetRealData(PIdx::y).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_1D_Z) + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#endif +#if defined(WARPX_DIM_RZ) m_theta = soa.GetRealData(T_PIdx::theta).dataPtr() + a_offset; #endif } @@ -190,17 +224,17 @@ struct SetParticlePosition #endif #ifdef WARPX_DIM_RZ m_theta[i] = std::atan2(y, x); - m_structs[i].pos(0) = std::sqrt(x*x + y*y); - m_structs[i].pos(1) = z; + m_x[i] = std::sqrt(x*x + y*y); + m_z[i] = z; #elif WARPX_DIM_3D - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = y; - m_structs[i].pos(2) = z; + m_x[i] = x; + m_y[i] = y; + m_z[i] = z; #elif WARPX_DIM_XZ - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = z; + m_x[i] = x; + m_z[i] = z; #else - m_structs[i].pos(0) = z; + m_z[i] = z; #endif } @@ -218,18 +252,18 @@ struct SetParticlePosition amrex::ignore_unused(x,y); #endif #ifdef WARPX_DIM_RZ - m_structs[i].pos(0) = x; + m_x[i] = x; m_theta[i] = y; - m_structs[i].pos(1) = z; + m_z[i] = z; #elif WARPX_DIM_3D - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = y; - m_structs[i].pos(2) = z; + m_x[i] = x; + m_y[i] = y; + m_z[i] = z; #elif WARPX_DIM_XZ - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = z; + m_x[i] = x; + m_z[i] = z; #else - m_structs[i].pos(0) = z; + m_z[i] = z; #endif } }; diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index 680e33ebe6a..40f75c76eab 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -60,8 +60,7 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, auto& ptile = pc->ParticlesAt(lev, pti); auto& soa = ptile.GetStructOfArrays(); amrex::ParticleReal * const AMREX_RESTRICT w = soa.GetRealData(PIdx::w).data(); - WarpXParticleContainer::ParticleType * const AMREX_RESTRICT - particle_ptr = ptile.GetArrayOfStructs()().data(); + auto * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); // Using this function means that we must loop over the cells in the ParallelFor. In the case // of the leveling thinning algorithm, it would have possibly been more natural and more @@ -114,7 +113,7 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, // Remove particle with probability 1 - particle_weight/level_weight if (random_number > w[indices[i]]/level_weight) { - particle_ptr[indices[i]].id() = -1; + amrex::ParticleIDWrapper{idcpu[indices[i]]} = -1; } // Set particle weight to level weight otherwise else diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp index 58511cfd5e7..58e3450f47d 100644 --- a/Source/Particles/Sorting/Partition.cpp +++ b/Source/Particles/Sorting/Partition.cpp @@ -61,7 +61,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // Initialize temporary arrays Gpu::DeviceVector inexflag; inexflag.resize(np); - Gpu::DeviceVector pid; + Gpu::DeviceVector pid; pid.resize(np); // First, partition particles into the larger buffer @@ -109,7 +109,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // - For each particle in the large buffer, find whether it is in // the smaller buffer, by looking up the mask. Store the answer in `inexflag`. amrex::ParallelFor( np - n_fine, - fillBufferFlagRemainingParticles(pti, bmasks, inexflag, Geom(lev), pid, n_fine) ); + fillBufferFlagRemainingParticles(pti, bmasks, inexflag, Geom(lev), pid, int(n_fine)) ); auto *const sep2 = stablePartition( sep, pid.end(), inexflag ); if (bmasks == gather_masks) { diff --git a/Source/Particles/Sorting/SortingUtils.H b/Source/Particles/Sorting/SortingUtils.H index ac2c63e88f8..ba7761bf48a 100644 --- a/Source/Particles/Sorting/SortingUtils.H +++ b/Source/Particles/Sorting/SortingUtils.H @@ -12,6 +12,7 @@ #include #include +#include /** \brief Fill the elements of the input vector with consecutive integer, @@ -19,7 +20,7 @@ * * \param[inout] v Vector of integers, to be filled by this routine */ -void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ); +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ); /** \brief Find the indices that would reorder the elements of `predicate` * so that the elements with non-zero value precede the other elements @@ -41,7 +42,7 @@ ForwardIterator stablePartition(ForwardIterator const index_begin, int const* AMREX_RESTRICT predicate_ptr = predicate.dataPtr(); int N = static_cast(std::distance(index_begin, index_end)); auto num_true = amrex::StablePartition(&(*index_begin), N, - [predicate_ptr] AMREX_GPU_DEVICE (long i) { return predicate_ptr[i]; }); + [predicate_ptr] AMREX_GPU_DEVICE (int i) { return predicate_ptr[i]; }); ForwardIterator sep = index_begin; std::advance(sep, num_true); @@ -49,7 +50,7 @@ ForwardIterator stablePartition(ForwardIterator const index_begin, // On CPU: Use std library ForwardIterator const sep = std::stable_partition( index_begin, index_end, - [&predicate](long i) { return predicate[i]; } + [&predicate](int i) { return predicate[i]; } ); #endif return sep; @@ -88,7 +89,7 @@ class fillBufferFlag // Extract simple structure that can be used directly on the GPU m_domain{geom.Domain()}, m_inexflag_ptr{inexflag.dataPtr()}, - m_particles{pti.GetArrayOfStructs().data()}, + m_ptd{pti.GetParticleTile().getConstParticleTileData()}, m_buffer_mask{(*bmasks)[pti].array()} { for (int idim=0; idim m_buffer_mask; amrex::GpuArray m_prob_lo; amrex::GpuArray m_inv_cell_size; @@ -141,12 +140,12 @@ class fillBufferFlagRemainingParticles amrex::iMultiFab const* bmasks, amrex::Gpu::DeviceVector& inexflag, amrex::Geometry const& geom, - amrex::Gpu::DeviceVector const& particle_indices, - long const start_index ) : + amrex::Gpu::DeviceVector const& particle_indices, + int start_index ) : m_domain{geom.Domain()}, // Extract simple structure that can be used directly on the GPU m_inexflag_ptr{inexflag.dataPtr()}, - m_particles{pti.GetArrayOfStructs().data()}, + m_ptd{pti.GetParticleTile().getConstParticleTileData()}, m_buffer_mask{(*bmasks)[pti].array()}, m_start_index{start_index}, m_indices_ptr{particle_indices.dataPtr()} @@ -159,11 +158,11 @@ class fillBufferFlagRemainingParticles AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator()( const long i ) const { + void operator()( const int i ) const { // Select a particle - auto const& p = m_particles[m_indices_ptr[i+m_start_index]]; + auto const j = m_indices_ptr[i+m_start_index]; // Find the index of the cell where this particle is located - amrex::IntVect const iv = amrex::getParticleCell( p, + amrex::IntVect const iv = amrex::getParticleCell( m_ptd, j, m_prob_lo, m_inv_cell_size, m_domain ); // Find the value of the buffer flag in this cell and // store it at the corresponding particle position in the array `inexflag` @@ -175,10 +174,10 @@ class fillBufferFlagRemainingParticles amrex::GpuArray m_inv_cell_size; amrex::Box m_domain; int* m_inexflag_ptr; - WarpXParticleContainer::ParticleType const* m_particles; + WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType const m_ptd; amrex::Array4 m_buffer_mask; - long const m_start_index; - long const* m_indices_ptr; + int const m_start_index; + int const* m_indices_ptr; }; /** \brief Functor that copies the elements of `src` into `dst`, @@ -195,7 +194,7 @@ class copyAndReorder copyAndReorder( amrex::Gpu::DeviceVector const& src, amrex::Gpu::DeviceVector& dst, - amrex::Gpu::DeviceVector const& indices ): + amrex::Gpu::DeviceVector const& indices ): // Extract simple structure that can be used directly on the GPU m_src_ptr{src.dataPtr()}, m_dst_ptr{dst.dataPtr()}, @@ -203,14 +202,14 @@ class copyAndReorder {} AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void operator()( const long ip ) const { + void operator()( const int ip ) const { m_dst_ptr[ip] = m_src_ptr[ m_indices_ptr[ip] ]; } private: T const* m_src_ptr; T* m_dst_ptr; - long const* m_indices_ptr; + int const* m_indices_ptr; }; #endif // WARPX_PARTICLES_SORTING_SORTINGUTILS_H_ diff --git a/Source/Particles/Sorting/SortingUtils.cpp b/Source/Particles/Sorting/SortingUtils.cpp index 699119e8e18..cd4b6a13c76 100644 --- a/Source/Particles/Sorting/SortingUtils.cpp +++ b/Source/Particles/Sorting/SortingUtils.cpp @@ -8,7 +8,7 @@ #include "SortingUtils.H" -void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) { #ifdef AMREX_USE_GPU // On GPU: Use amrex diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 33aa71d1c7d..a244afc2389 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -49,10 +49,10 @@ class WarpXParIter - : public amrex::ParIter<0,0,PIdx::nattribs> + : public amrex::ParIterSoA { public: - using amrex::ParIter<0,0,PIdx::nattribs>::ParIter; + using amrex::ParIterSoA::ParIterSoA; WarpXParIter (ContainerType& pc, int level); @@ -89,7 +89,7 @@ public: * particle container classes (that store a collection of particles) derive. Derived * classes can be used for plasma particles, photon particles, or non-physical * particles (e.g., for the laser antenna). - * It derives from amrex::ParticleContainer<0,0,PIdx::nattribs>, where the + * It derives from amrex::ParticleContainerPureSoA, where the * template arguments stand for the number of int and amrex::Real SoA and AoS * data in amrex::Particle. * - AoS amrex::Real: x, y, z (default), 0 additional (first template @@ -164,11 +164,9 @@ public: * class. */ virtual void DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>& pinned_tile, - int n_external_attr_real, - int n_external_attr_int) = 0; + typename ContainerLike::ParticleTileType& pinned_tile, + int n_external_attr_real, + int n_external_attr_int) = 0; /// /// This pushes the particle positions by one half time step. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index e94aae5d573..48344b20fef 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -72,13 +72,13 @@ using namespace amrex; WarpXParIter::WarpXParIter (ContainerType& pc, int level) - : amrex::ParIter<0,0,PIdx::nattribs>(pc, level, + : amrex::ParIterSoA(pc, level, MFItInfo().SetDynamic(WarpX::do_dynamic_scheduling)) { } WarpXParIter::WarpXParIter (ContainerType& pc, int level, MFItInfo& info) - : amrex::ParIter<0,0,PIdx::nattribs>(pc, level, + : amrex::ParIterSoA(pc, level, info.SetDynamic(WarpX::do_dynamic_scheduling)) { } @@ -195,52 +195,54 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, // Redistribute() will move them to proper places. auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); - using PinnedTile = amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>; + using PinnedTile = typename ContainerLike::ParticleTileType; PinnedTile pinned_tile; pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps()); const std::size_t np = iend-ibegin; #ifdef WARPX_DIM_RZ + amrex::Vector r(np); amrex::Vector theta(np); #endif for (auto i = ibegin; i < iend; ++i) { - ParticleType p; - if (id==-1) - { - p.id() = ParticleType::NextID(); + auto & idcpu_data = pinned_tile.GetStructOfArrays().GetIdCPUData(); + idcpu_data.push_back(0); + if (id==-1) { + amrex::ParticleIDWrapper{idcpu_data.back()} = ParticleType::NextID(); } else { - p.id() = id; + amrex::ParticleIDWrapper{idcpu_data.back()} = id; } - p.cpu() = amrex::ParallelDescriptor::MyProc(); + amrex::ParticleCPUWrapper(idcpu_data.back()) = ParallelDescriptor::MyProc(); + +#ifdef WARPX_DIM_RZ + r[i-ibegin] = std::sqrt(x[i]*x[i] + y[i]*y[i]); + theta[i-ibegin] = std::atan2(y[i], x[i]); +#endif + } + + if (np > 0) + { #if defined(WARPX_DIM_3D) - p.pos(0) = x[i]; - p.pos(1) = y[i]; - p.pos(2) = z[i]; + pinned_tile.push_back_real(PIdx::x, x.data() + ibegin, x.data() + iend); + pinned_tile.push_back_real(PIdx::y, y.data() + ibegin, y.data() + iend); + pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(y); #ifdef WARPX_DIM_RZ - theta[i-ibegin] = std::atan2(y[i], x[i]); - p.pos(0) = std::sqrt(x[i]*x[i] + y[i]*y[i]); + pinned_tile.push_back_real(PIdx::x, r.data(), r.data() + np); #else - p.pos(0) = x[i]; + pinned_tile.push_back_real(PIdx::x, x.data() + ibegin, x.data() + iend); #endif - p.pos(1) = z[i]; + pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); #else //AMREX_SPACEDIM == 1 amrex::ignore_unused(x,y); - p.pos(0) = z[i]; + pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); #endif - pinned_tile.push_back(p); - } - - if (np > 0) - { - pinned_tile.push_back_real(PIdx::w , attr_real[0].data() + ibegin, attr_real[0].data() + iend); + pinned_tile.push_back_real(PIdx::w, attr_real[0].data() + ibegin, attr_real[0].data() + iend); pinned_tile.push_back_real(PIdx::ux, ux.data() + ibegin, ux.data() + iend); pinned_tile.push_back_real(PIdx::uy, uy.data() + ibegin, uy.data() + iend); pinned_tile.push_back_real(PIdx::uz, uz.data() + ibegin, uz.data() + iend); @@ -466,15 +468,14 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, //sort particles by bin WARPX_PROFILE_VAR_START(blp_sort); - amrex::DenseBins bins; + amrex::DenseBins bins; { auto& ptile = ParticlesAt(lev, pti); - auto& aos = ptile.GetArrayOfStructs(); - auto *pstruct_ptr = aos().dataPtr(); + auto ptd = ptile.getParticleTileData(); const int ntiles = numTilesInBox(box, true, bin_size); - bins.build(ptile.numParticles(), pstruct_ptr, ntiles, + bins.build(ptile.numParticles(), ptd, ntiles, [=] AMREX_GPU_HOST_DEVICE (const ParticleType& p) -> unsigned int { Box tbox; @@ -874,7 +875,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // HACK - sort particles by bin here. WARPX_PROFILE_VAR_START(blp_sort); - amrex::DenseBins bins; + amrex::DenseBins bins; { const Geometry& geom = Geom(lev); const auto dxi = geom.InvCellSizeArray(); @@ -882,16 +883,15 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const auto domain = geom.Domain(); auto& ptile = ParticlesAt(lev, pti); - auto& aos = ptile.GetArrayOfStructs(); - auto *pstruct_ptr = aos().dataPtr(); + auto ptd = ptile.getParticleTileData(); Box box = pti.validbox(); box.grow(ng_rho); const amrex::IntVect bin_size = WarpX::shared_tilesize; const int ntiles = numTilesInBox(box, true, bin_size); - bins.build(ptile.numParticles(), pstruct_ptr, ntiles, - [=] AMREX_GPU_HOST_DEVICE (const ParticleType& p) -> unsigned int + bins.build(ptile.numParticles(), ptd, ntiles, + [=] AMREX_GPU_HOST_DEVICE (ParticleType const & p) -> unsigned int { Box tbx; auto iv = getParticleCell(p, plo, dxi, domain); @@ -911,8 +911,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const auto domain = geom.Domain(); auto& ptile = ParticlesAt(lev, pti); - auto& aos = ptile.GetArrayOfStructs(); - auto *pstruct_ptr = aos().dataPtr(); + auto ptd = ptile.getParticleTileData(); Box box = pti.validbox(); box.grow(ng_rho); @@ -926,9 +925,10 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const auto bin_start = offsets_ptr[ibin]; const auto bin_stop = offsets_ptr[ibin+1]; if (bin_start < bin_stop) { - auto p = pstruct_ptr[permutation[bin_start]]; + // static_cast until https://github.com/AMReX-Codes/amrex/pull/3684 + auto const i = static_cast(permutation[bin_start]); Box tbx; - auto iv = getParticleCell(p, plo, dxi, domain); + auto iv = getParticleCell(ptd, i, plo, dxi, domain); AMREX_ASSERT(box.contains(iv)); [[maybe_unused]] auto tid = getTileIndex(iv, box, true, bin_size, tbx); AMREX_ASSERT(tid == ibin); @@ -1417,10 +1417,10 @@ WarpXParticleContainer::particlePostLocate(ParticleType& p, // Tag particle if goes to higher level. // It will be split later in the loop if (pld.m_lev == lev+1 - and p.id() != NoSplitParticleID + and p.id() != amrex::LongParticleIds::NoSplitParticleID and p.id() >= 0) { - p.id() = DoSplitParticleID; + p.id() = amrex::LongParticleIds::DoSplitParticleID; } if (pld.m_lev == lev-1){ @@ -1459,9 +1459,9 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ const Real zmax = Geom(lev).ProbHi(WARPX_ZINDEX); ParticleTileType& ptile = ParticlesAt(lev, pti); - ParticleType * const pp = ptile.GetArrayOfStructs()().data(); auto& soa = ptile.GetStructOfArrays(); + uint64_t * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); amrex::ParticleReal * const AMREX_RESTRICT ux = soa.GetRealData(PIdx::ux).data(); amrex::ParticleReal * const AMREX_RESTRICT uy = soa.GetRealData(PIdx::uy).data(); amrex::ParticleReal * const AMREX_RESTRICT uz = soa.GetRealData(PIdx::uz).data(); @@ -1470,10 +1470,9 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ amrex::ParallelForRNG( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i, amrex::RandomEngine const& engine) { - ParticleType& p = pp[i]; - // skip particles that are already flagged for removal - if (p.id() < 0) { return; } + auto const id = amrex::ParticleIDWrapper{idcpu[i]}; + if (id < 0) { return; } ParticleReal x, y, z; GetPosition.AsStored(i, x, y, z); @@ -1495,7 +1494,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ boundary_conditions, engine); if (particle_lost) { - p.id() = -p.id(); + amrex::ParticleIDWrapper{idcpu[i]} = -id; } else { SetPosition.AsStored(i, x, y, z); } diff --git a/Source/Python/Particles/ParticleBoundaryBuffer.cpp b/Source/Python/Particles/ParticleBoundaryBuffer.cpp index 2a35faece9b..b04ac75e600 100644 --- a/Source/Python/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Python/Particles/ParticleBoundaryBuffer.cpp @@ -10,13 +10,13 @@ namespace warpx { class BoundaryBufferParIter - : public amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> + : public amrex::ParIterSoA { public: - using amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator>::ParIter; + using amrex::ParIterSoA::ParIterSoA; BoundaryBufferParIter(ContainerType& pc, int level) : - amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator>(pc, level) {} + amrex::ParIterSoA(pc, level) {} }; } @@ -24,9 +24,9 @@ void init_BoundaryBufferParIter (py::module& m) { py::class_< warpx::BoundaryBufferParIter, - amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> + amrex::ParIterSoA >(m, "BoundaryBufferParIter") - .def(py::init::ContainerType&, int>(), + .def(py::init::ContainerType&, int>(), py::arg("particle_container"), py::arg("level") ) ; diff --git a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp index 600d56a62c9..d4f6a422dbe 100644 --- a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp +++ b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp @@ -13,6 +13,6 @@ void init_PinnedMemoryParticleContainer (py::module& m) { py::class_< PinnedMemoryParticleContainer, - amrex::ParticleContainer<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> + amrex::ParticleContainerPureSoA > pmpc (m, "PinnedMemoryParticleContainer"); } diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index 1473a750941..07793a373f3 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -12,11 +12,11 @@ void init_WarpXParIter (py::module& m) { py::class_< - WarpXParIter, amrex::ParIter<0,0,PIdx::nattribs> + WarpXParIter, amrex::ParIterSoA >(m, "WarpXParIter") - .def(py::init::ContainerType&, int>(), + .def(py::init::ContainerType&, int>(), py::arg("particle_container"), py::arg("level")) - .def(py::init::ContainerType&, int, amrex::MFItInfo&>(), + .def(py::init::ContainerType&, int, amrex::MFItInfo&>(), py::arg("particle_container"), py::arg("level"), py::arg("info")) ; @@ -26,11 +26,11 @@ void init_WarpXParticleContainer (py::module& m) { py::class_< WarpXParticleContainer, - amrex::ParticleContainer<0, 0, PIdx::nattribs, 0> + amrex::ParticleContainerPureSoA > wpc (m, "WarpXParticleContainer"); wpc .def("add_real_comp", - [](WarpXParticleContainer& pc, const std::string& name, bool const comm) { pc.AddRealComp(name, comm); }, + [](WarpXParticleContainer& pc, const std::string& name, bool comm) { pc.AddRealComp(name, comm); }, py::arg("name"), py::arg("comm") ) .def("add_n_particles", @@ -93,6 +93,14 @@ void init_WarpXParticleContainer (py::module& m) }, py::arg("comp_name") ) + .def("get_icomp_index", + [](WarpXParticleContainer& pc, std::string comp_name) + { + auto particle_comps = pc.getParticleiComps(); + return particle_comps.at(comp_name); + }, + py::arg("comp_name") + ) .def("num_local_tiles_at_level", &WarpXParticleContainer::numLocalTilesAtLevel, py::arg("level") diff --git a/Source/Utils/ParticleUtils.H b/Source/Utils/ParticleUtils.H index b04176d4d83..7e3c89228ea 100644 --- a/Source/Utils/ParticleUtils.H +++ b/Source/Utils/ParticleUtils.H @@ -28,9 +28,10 @@ namespace ParticleUtils { * @param[in] mfi the MultiFAB iterator. * @param[in] ptile the particle tile. */ - amrex::DenseBins - findParticlesInEachCell(int lev, amrex::MFIter const& mfi, - WarpXParticleContainer::ParticleTileType const& ptile); + amrex::DenseBins + findParticlesInEachCell (int lev, + amrex::MFIter const & mfi, + WarpXParticleContainer::ParticleTileType & ptile); /** * \brief Return (relativistic) particle energy given velocity and mass. diff --git a/Source/Utils/ParticleUtils.cpp b/Source/Utils/ParticleUtils.cpp index 60e04f12b86..b8207b61fa0 100644 --- a/Source/Utils/ParticleUtils.cpp +++ b/Source/Utils/ParticleUtils.cpp @@ -22,24 +22,28 @@ #include #include -namespace ParticleUtils { +namespace ParticleUtils +{ using namespace amrex; + // Define shortcuts for frequently-used type names - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = DenseBins; - using index_type = ParticleBins::index_type; + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = DenseBins; + using index_type = typename ParticleBins::index_type; /* Find the particles and count the particles that are in each cell. Note that this does *not* rearrange particle arrays */ ParticleBins - findParticlesInEachCell( int const lev, MFIter const& mfi, - ParticleTileType const& ptile) { + findParticlesInEachCell (int lev, + MFIter const & mfi, + ParticleTileType & ptile) { // Extract particle structures for this tile int const np = ptile.numParticles(); - ParticleType const* particle_ptr = ptile.GetArrayOfStructs()().data(); + auto ptd = ptile.getParticleTileData(); // Extract box properties Geometry const& geom = WarpX::GetInstance().Geom(lev); @@ -51,9 +55,9 @@ namespace ParticleUtils { // Find particles that are in each cell; // results are stored in the object `bins`. ParticleBins bins; - bins.build(np, particle_ptr, cbx, + bins.build(np, ptd, cbx, // Pass lambda function that returns the cell index - [=] AMREX_GPU_DEVICE (const ParticleType& p) noexcept + [=] AMREX_GPU_DEVICE (ParticleType const & p) noexcept -> amrex::IntVect { return IntVect{AMREX_D_DECL( static_cast((p.pos(0)-plo[0])*dxi[0] - lo.x), @@ -64,4 +68,4 @@ namespace ParticleUtils { return bins; } -} +} // namespace ParticleUtils diff --git a/Source/ablastr/particles/IndexHandling.H b/Source/ablastr/particles/IndexHandling.H deleted file mode 100644 index 0ad5ca60446..00000000000 --- a/Source/ablastr/particles/IndexHandling.H +++ /dev/null @@ -1,41 +0,0 @@ -/* Copyright 2019-2022 Axel Huebl - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#ifndef ABLASTR_INDEX_HANDLING_H -#define ABLASTR_INDEX_HANDLING_H - -#include - - -namespace ablastr::particles { - - /** A helper function to derive a globally unique particle ID - * - * @param[in] id AMReX particle ID (on local cpu/rank), AoS .id - * @param[in] cpu AMReX particle CPU (rank) at creation of the particle, AoS .cpu - * @return global particle ID that is unique and permanent in the whole simulation - */ - constexpr uint64_t - localIDtoGlobal (int const id, int const cpu) - { - static_assert(sizeof(int) * 2u <= sizeof(uint64_t), - "int size might cause collisions in global IDs"); - // implementation: - // - we cast both 32-bit (or smaller) ints to a 64bit unsigned int - // - this will leave half of the "upper range" bits in the 64bit unsigned int zeroed out - // because the corresponding (extended) value range was not part of the value range in - // the int representation - // - we bit-shift the cpu into the upper half of zero bits in the 64 bit unsigned int - // (imagine this step as "adding a per-cpu/rank offset to the local integers") - // - then we add this offset - // note: the add is expressed as bitwise OR (|) since this saves us from writing - // brackets for operator precedence between + and << - return uint64_t(id) | uint64_t(cpu) << 32u; - } - -} // namespace ablastr::particles - -#endif // ABLASTR_INDEX_HANDLING_H diff --git a/Source/ablastr/particles/ParticleMoments.H b/Source/ablastr/particles/ParticleMoments.H index e45fb574cce..b648ccb28aa 100644 --- a/Source/ablastr/particles/ParticleMoments.H +++ b/Source/ablastr/particles/ParticleMoments.H @@ -35,7 +35,7 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal> MinAndMaxPositions (T_PC const & pc) { - using PType = typename T_PC::SuperParticleType; + using ConstParticleTileDataType = typename T_PC::ParticleTileType::ConstParticleTileDataType; // Get min and max for the local rank amrex::ReduceOps< @@ -46,11 +46,11 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal, amrex::ParticleReal> >( pc, - [=] AMREX_GPU_DEVICE(PType const & p) noexcept + [=] AMREX_GPU_DEVICE(const ConstParticleTileDataType& ptd, const int i) noexcept { - amrex::ParticleReal const x = p.pos(0); - amrex::ParticleReal const y = p.pos(1); - amrex::ParticleReal const z = p.pos(2); + const amrex::ParticleReal x = ptd.rdata(0)[i]; + const amrex::ParticleReal y = ptd.rdata(1)[i]; + const amrex::ParticleReal z = ptd.rdata(2)[i]; return amrex::makeTuple(x, y, z, x, y, z); }, @@ -90,7 +90,8 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal> MeanAndStdPositions (T_PC const & pc) { - using PType = typename T_PC::SuperParticleType; + + using ConstParticleTileDataType = typename T_PC::ParticleTileType::ConstParticleTileDataType; amrex::ReduceOps< amrex::ReduceOpSum, amrex::ReduceOpSum, amrex::ReduceOpSum, @@ -103,12 +104,14 @@ namespace particles { amrex::ParticleReal> >( pc, - [=] AMREX_GPU_DEVICE(const PType& p) noexcept + [=] AMREX_GPU_DEVICE(const ConstParticleTileDataType& ptd, const int i) noexcept { - amrex::ParticleReal const x = p.pos(0); - amrex::ParticleReal const y = p.pos(1); - amrex::ParticleReal const z = p.pos(2); - amrex::ParticleReal const w = p.rdata(T_RealSoAWeight); + + const amrex::ParticleReal x = ptd.rdata(0)[i]; + const amrex::ParticleReal y = ptd.rdata(1)[i]; + const amrex::ParticleReal z = ptd.rdata(2)[i]; + + const amrex::ParticleReal w = ptd.rdata(T_RealSoAWeight)[i]; return amrex::makeTuple(x, x*x, y, y*y, z, z*z, w); }, From 67c9aa2b9ff9f8a611b4a98ae1332e4794c4ac3a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 29 Jan 2024 11:36:15 -0800 Subject: [PATCH 119/176] Remove particles that are initialized in the EB (#4585) --- Source/Particles/PhysicalParticleContainer.cpp | 16 ++++++++++++++++ Source/Particles/WarpXParticleContainer.cpp | 11 ++++++++++- Source/WarpX.H | 4 +++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index fd55e369040..17c238e1121 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -39,6 +39,10 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#ifdef AMREX_USE_EB +# include "EmbeddedBoundary/ParticleBoundaryProcess.H" +# include "EmbeddedBoundary/ParticleScraper.H" +#endif #include "WarpX.H" #include @@ -1428,6 +1432,12 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int } } + // Remove particles that are inside the embedded boundaries +#ifdef AMREX_USE_EB + auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); +#endif + // The function that calls this is responsible for redistributing particles. } @@ -1920,6 +1930,12 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, } } + // Remove particles that are inside the embedded boundaries +#ifdef AMREX_USE_EB + auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticles(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); +#endif + // Redistribute the new particles that were added to the temporary container. // (This eliminates invalid particles, and makes sure that particles // are in the right tile.) diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 48344b20fef..0d927caacc8 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -60,7 +60,10 @@ #include #include #include - +#ifdef AMREX_USE_EB +# include "EmbeddedBoundary/ParticleBoundaryProcess.H" +# include "EmbeddedBoundary/ParticleScraper.H" +#endif #ifdef AMREX_USE_OMP # include @@ -293,6 +296,12 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, ); } + // Remove particles that are inside the embedded boundaries +#ifdef AMREX_USE_EB + auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); +#endif + Redistribute(); } diff --git a/Source/WarpX.H b/Source/WarpX.H index c9aa24d3973..e8c7ae79f7e 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -132,7 +132,9 @@ public: MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; } HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} - +#ifdef AMREX_USE_EB + amrex::Vector >& GetDistanceToEB () {return m_distance_to_eb;} +#endif ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } static void shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, From c9df4485fc29e1ec5111906975d359efcace269a Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 29 Jan 2024 11:43:50 -0800 Subject: [PATCH 120/176] Update DSMC picmi class initialization function (#4646) --- Python/pywarpx/picmi.py | 2 +- Regression/WarpX-tests.ini | 36 ++++++++++++++++++------------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 857f857a554..89bd5af2eab 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1575,7 +1575,7 @@ def __init__(self, name, species, scattering_processes, ndt=None, **kw): self.handle_init(kw) - def initialize_inputs(self): + def collision_initialize_inputs(self): collision = pywarpx.Collisions.newcollision(self.name) collision.type = 'dsmc' collision.species = [species.name for species in self.species] diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 75939c1fd12..7af5eb4b002 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3079,24 +3079,24 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py -# [Python_dsmc_1d] -# buildDir = . -# inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py -# runtime_params = -# customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc -# dim = 1 -# addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -# cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -# target = pip_install -# restartTest = 0 -# useMPI = 1 -# numprocs = 2 -# useOMP = 1 -# numthreads = 1 -# compileTest = 0 -# doVis = 0 -# compareParticles = 0 -# analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py +[Python_dsmc_1d] +buildDir = . +inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +runtime_params = +customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc +dim = 1 +addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py [Python_ElectrostaticSphereEB] buildDir = . From 0284b5731ebba43dbe571b7c53c4692c0271f4c3 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 29 Jan 2024 16:03:09 -0800 Subject: [PATCH 121/176] AMReX: Weekly Update (#4647) --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- run_test.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 09f7f9e80fa..eca39369a08 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 73b215557c0e842c3e829b683939bbb7a7e12373 && cd - + cd ../amrex && git checkout --detach ab99ea69089f4fffbfd727ed965d5ceb3d905baa && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index f0557e05fc4..51ae03b60fa 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 73b215557c0e842c3e829b683939bbb7a7e12373 +branch = ab99ea69089f4fffbfd727ed965d5ceb3d905baa [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 7af5eb4b002..ba5ae86c28e 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 73b215557c0e842c3e829b683939bbb7a7e12373 +branch = ab99ea69089f4fffbfd727ed965d5ceb3d905baa [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 31c8919e38f..72173b2acb0 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "73b215557c0e842c3e829b683939bbb7a7e12373" +set(WarpX_amrex_branch "ab99ea69089f4fffbfd727ed965d5ceb3d905baa" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/run_test.sh b/run_test.sh index 77980bb00b0..20c6638f4d0 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 73b215557c0e842c3e829b683939bbb7a7e12373 && cd - +cd amrex && git checkout --detach ab99ea69089f4fffbfd727ed965d5ceb3d905baa && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 3e551e0a0b559114b91a341a5f5b7e0a85faeedd Mon Sep 17 00:00:00 2001 From: GivenChen <52523185+GivenChen@users.noreply.github.com> Date: Mon, 29 Jan 2024 22:18:33 -0500 Subject: [PATCH 122/176] Fix neutral particle got pushed (#4643) * Update PushSelector.H add the pointer of the array of ion_lev to the pusher function, and use this pointer to determine whether to change the charge instead of the value of ion_lev * Update PhysicalParticleContainer.cpp Change the variable in the targeted function * Alternative approach --------- Co-authored-by: Axel Huebl --- Source/Particles/PhysicalParticleContainer.cpp | 8 ++++---- Source/Particles/Pusher/PushSelector.H | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 17c238e1121..a8ed42ce419 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2880,7 +2880,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, + ion_lev ? ion_lev[ip] : 1, m, q, pusher_algo, do_crr, #ifdef WARPX_QED t_chi_max, @@ -2900,7 +2900,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, + ion_lev ? ion_lev[ip] : 1, m, q, pusher_algo, do_crr, t_chi_max, dt); @@ -3140,7 +3140,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, { doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, + ion_lev ? ion_lev[ip] : 1, m, q, pusher_algo, do_crr, #ifdef WARPX_QED t_chi_max, @@ -3152,7 +3152,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, if constexpr (qed_control == has_qed) { doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, + ion_lev ? ion_lev[ip] : 1, m, q, pusher_algo, do_crr, t_chi_max, dt); diff --git a/Source/Particles/Pusher/PushSelector.H b/Source/Particles/Pusher/PushSelector.H index 2a6ac8a7d6c..4a82e582bfb 100644 --- a/Source/Particles/Pusher/PushSelector.H +++ b/Source/Particles/Pusher/PushSelector.H @@ -57,7 +57,7 @@ void doParticleMomentumPush(amrex::ParticleReal& ux, const amrex::Real dt) { amrex::ParticleReal qp = a_q; - if (ion_lev) { qp *= ion_lev; } + qp *= ion_lev; if (do_crr) { #ifdef WARPX_QED From 33d7b496d11ca41ce9025c6df6feb56a2ecb8b30 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Tue, 30 Jan 2024 06:50:50 -0800 Subject: [PATCH 123/176] reactivate CI test (Python_background_mcc_1d_tridiag) (#4650) --- Regression/WarpX-tests.ini | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index ba5ae86c28e..3489e8ad100 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3021,24 +3021,24 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py -# [Python_background_mcc_1d_tridiag] -# buildDir = . -# inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py -# runtime_params = -# customRunCmd = python3 PICMI_inputs_1d.py --test -# dim = 1 -# addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE -# cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF -# target = pip_install -# restartTest = 0 -# useMPI = 1 -# numprocs = 2 -# useOMP = 1 -# numthreads = 1 -# compileTest = 0 -# doVis = 0 -# compareParticles = 0 -# analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py +[Python_background_mcc_1d_tridiag] +buildDir = . +inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +runtime_params = +customRunCmd = python3 PICMI_inputs_1d.py --test +dim = 1 +addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py [Python_collisionXZ] buildDir = . From 2708b4fb8fc16c847a49d5f7f367a9d0287d6042 Mon Sep 17 00:00:00 2001 From: Sarah Vickers <136610602+sevicky@users.noreply.github.com> Date: Tue, 30 Jan 2024 06:51:53 -0800 Subject: [PATCH 124/176] Grammar Change in Theory Intro (#4617) Split a run-on sentence into two sentences in the first paragraph of the introduction. --- Docs/source/theory/intro.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Docs/source/theory/intro.rst b/Docs/source/theory/intro.rst index 10fbba679c2..2101d9d81cb 100644 --- a/Docs/source/theory/intro.rst +++ b/Docs/source/theory/intro.rst @@ -8,8 +8,7 @@ Introduction Plasma laser-driven (top) and charged-particles-driven (bottom) acceleration (rendering from 3-D Particle-In-Cell simulations). A laser beam (red and blue disks in top picture) or a charged particle beam (red dots in bottom picture) propagating (from left to right) through an under-dense plasma (not represented) displaces electrons, creating a plasma wakefield that supports very high electric fields (pale blue and yellow). These electric fields, which can be orders of magnitude larger than with conventional techniques, can be used to accelerate a short charged particle beam (white) to high-energy over a very short distance. -Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`i-Tsungpop06,i-Geddesjp08,i-Geddesscidac09,i-Geddespac09,i-Huangscidac09`, with -accurate modeling of wake formation, electron self-trapping and acceleration requiring fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`i-LeemansPRL2014,i-Blumenfeld2007,i-BulanovSV2014,i-Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play. +Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`i-Tsungpop06,i-Geddesjp08,i-Geddesscidac09,i-Geddespac09,i-Huangscidac09`. Accurate modeling of wake formation, electron self-trapping and acceleration require fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`i-LeemansPRL2014,i-Blumenfeld2007,i-BulanovSV2014,i-Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play. For such simulations, the most popular algorithm is the Particle-In-Cell (or PIC) technique, From 56e4a2a6e180c4f79ef1822493e11cc7561b59ed Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 30 Jan 2024 14:00:27 -0800 Subject: [PATCH 125/176] Revert "Update Particle Container to Pure SoA (#3850)" (#4652) This reverts commit 94ae11900131846e9f8b3704194673f7f02d8959. --- Docs/source/usage/workflows/python_extend.rst | 3 +- .../particle_data_python/PICMI_inputs_2d.py | 6 +- .../PICMI_inputs_prev_pos_2d.py | 6 +- .../PICMI_inputs_runtime_component_analyze.py | 6 +- Python/pywarpx/particle_containers.py | 255 ++++++++++-------- .../BackTransformParticleFunctor.H | 16 +- .../FlushFormats/FlushFormatAscent.cpp | 3 + .../FlushFormats/FlushFormatCheckpoint.cpp | 9 +- .../FlushFormats/FlushFormatPlotfile.cpp | 13 +- Source/Diagnostics/ParticleIO.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 6 +- .../FieldProbeParticleContainer.H | 20 +- .../FieldProbeParticleContainer.cpp | 36 ++- .../ReducedDiags/LoadBalanceCosts.cpp | 3 +- Source/Diagnostics/WarpXOpenPMD.H | 4 +- Source/Diagnostics/WarpXOpenPMD.cpp | 139 ++++++++-- .../ParticleBoundaryProcess.H | 5 +- Source/EmbeddedBoundary/ParticleScraper.H | 3 +- .../BinaryCollision/BinaryCollision.H | 16 +- .../Coulomb/PairWiseCoulombCollisionFunc.H | 7 +- .../Collision/BinaryCollision/DSMC/DSMC.H | 3 +- .../DSMC/SplitAndScatterFunc.H | 30 +-- .../NuclearFusion/NuclearFusionFunc.H | 14 +- .../ProtonBoronFusionInitializeMomentum.H | 10 +- .../TwoProductFusionInitializeMomentum.H | 4 +- .../BinaryCollision/ParticleCreationFunc.H | 60 ++--- .../BinaryCollision/ShuffleFisherYates.H | 2 +- .../Particles/Deposition/ChargeDeposition.H | 2 +- .../Particles/Deposition/CurrentDeposition.H | 2 +- .../ElementaryProcess/QEDPairGeneration.H | 2 +- .../ElementaryProcess/QEDPhotonEmission.H | 16 +- Source/Particles/LaserParticleContainer.H | 7 +- .../NamedComponentParticleContainer.H | 47 +--- Source/Particles/ParticleBoundaryBuffer.cpp | 4 +- Source/Particles/ParticleCreation/SmartCopy.H | 5 +- .../Particles/ParticleCreation/SmartCreate.H | 22 +- .../Particles/ParticleCreation/SmartUtils.H | 9 +- Source/Particles/PhysicalParticleContainer.H | 8 +- .../Particles/PhysicalParticleContainer.cpp | 129 +++++---- Source/Particles/Pusher/GetAndSetPosition.H | 152 ++++------- .../Particles/Resampling/LevelingThinning.cpp | 5 +- Source/Particles/Sorting/Partition.cpp | 4 +- Source/Particles/Sorting/SortingUtils.H | 41 +-- Source/Particles/Sorting/SortingUtils.cpp | 2 +- Source/Particles/WarpXParticleContainer.H | 14 +- Source/Particles/WarpXParticleContainer.cpp | 89 +++--- .../Particles/ParticleBoundaryBuffer.cpp | 10 +- .../PinnedMemoryParticleContainer.cpp | 2 +- .../Particles/WarpXParticleContainer.cpp | 18 +- Source/Utils/ParticleUtils.H | 7 +- Source/Utils/ParticleUtils.cpp | 26 +- Source/ablastr/particles/IndexHandling.H | 41 +++ Source/ablastr/particles/ParticleMoments.H | 25 +- 53 files changed, 709 insertions(+), 661 deletions(-) create mode 100644 Source/ablastr/particles/IndexHandling.H diff --git a/Docs/source/usage/workflows/python_extend.rst b/Docs/source/usage/workflows/python_extend.rst index 7a3ef0e7af9..6c9286c02ce 100644 --- a/Docs/source/usage/workflows/python_extend.rst +++ b/Docs/source/usage/workflows/python_extend.rst @@ -252,8 +252,7 @@ Particles can be added to the simulation at specific positions and with specific .. autoclass:: pywarpx.particle_containers.ParticleContainerWrapper :members: -The ``get_particle_real_arrays()``, ``get_particle_int_arrays()`` and -``get_particle_idcpu_arrays()`` functions are called +The ``get_particle_structs()`` and ``get_particle_arrays()`` functions are called by several utility functions of the form ``get_particle_{comp_name}`` where ``comp_name`` is one of ``x``, ``y``, ``z``, ``r``, ``theta``, ``id``, ``cpu``, ``weight``, ``ux``, ``uy`` or ``uz``. diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py index 572871b8ed5..a4b7d9e134e 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py @@ -153,10 +153,10 @@ def add_particles(): ########################## assert (elec_wrapper.nps == 270 / (2 - args.unique)) -assert (elec_wrapper.particle_container.get_comp_index('w') == 2) -assert (elec_wrapper.particle_container.get_comp_index('newPid') == 6) +assert (elec_wrapper.particle_container.get_comp_index('w') == 0) +assert (elec_wrapper.particle_container.get_comp_index('newPid') == 4) -new_pid_vals = elec_wrapper.get_particle_real_arrays('newPid', 0) +new_pid_vals = elec_wrapper.get_particle_arrays('newPid', 0) for vals in new_pid_vals: assert np.allclose(vals, 5) diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py index 5de9879f0f8..1becd4464e7 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py @@ -120,12 +120,12 @@ elec_count = elec_wrapper.nps # check that the runtime attributes have the right indices -assert (elec_wrapper.particle_container.get_comp_index('prev_x') == 6) -assert (elec_wrapper.particle_container.get_comp_index('prev_z') == 7) +assert (elec_wrapper.particle_container.get_comp_index('prev_x') == 4) +assert (elec_wrapper.particle_container.get_comp_index('prev_z') == 5) # sanity check that the prev_z values are reasonable and # that the correct number of values are returned -prev_z_vals = elec_wrapper.get_particle_real_arrays('prev_z', 0) +prev_z_vals = elec_wrapper.get_particle_arrays('prev_z', 0) running_count = 0 for z_vals in prev_z_vals: diff --git a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py index 32c9f4e5808..706dedb6959 100755 --- a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py +++ b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py @@ -158,10 +158,10 @@ def add_particles(): ########################## assert electron_wrapper.nps == 90 -assert electron_wrapper.particle_container.get_comp_index("w") == 2 -assert electron_wrapper.particle_container.get_comp_index("newPid") == 6 +assert electron_wrapper.particle_container.get_comp_index("w") == 0 +assert electron_wrapper.particle_container.get_comp_index("newPid") == 4 -new_pid_vals = electron_wrapper.get_particle_real_arrays("newPid", 0) +new_pid_vals = electron_wrapper.get_particle_arrays("newPid", 0) for vals in new_pid_vals: assert np.allclose(vals, 5) diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index 72a675ec43c..273a981f4bd 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -117,10 +117,8 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, kwargs[key] = np.full(maxlen, val) # --- The number of built in attributes - # --- The positions - built_in_attrs = libwarpx.dim # --- The three velocities - built_in_attrs += 3 + built_in_attrs = 3 if libwarpx.geometry_dim == 'rz': # --- With RZ, there is also theta built_in_attrs += 1 @@ -190,25 +188,28 @@ def add_real_comp(self, pid_name, comm=True): self.particle_container.add_real_comp(pid_name, comm) - def get_particle_real_arrays(self, comp_name, level, copy_to_host=False): + def get_particle_structs(self, level, copy_to_host=False): ''' - This returns a list of numpy or cupy arrays containing the particle real array data - on each tile for this process. + This returns a list of numpy or cupy arrays containing the particle struct data + on each tile for this process. The particle data is represented as a structured + array and contains the particle 'x', 'y', 'z', and 'idcpu'. - Unless copy_to_host is specified, the data for the arrays are not - copied, but share the underlying memory buffer with WarpX. The + Unless copy_to_host is specified, the data for the structs are + not copied, but share the underlying memory buffer with WarpX. The arrays are fully writeable. + Note that cupy does not support structs: + https://github.com/cupy/cupy/issues/2031 + and will return arrays of binary blobs for the AoS (DP: ``"|V24"``). If copied + to host via copy_to_host, we correct for the right numpy AoS type. + Parameters ---------- - comp_name : str - The component of the array data that will be returned - - level : int + level : int The refinement level to reference (default=0) - copy_to_host : bool + copy_to_host : bool For GPU-enabled runs, one can either return the GPU arrays (the default) or force a device-to-host copy. @@ -216,29 +217,30 @@ def get_particle_real_arrays(self, comp_name, level, copy_to_host=False): ------- List of arrays - The requested particle array data + The requested particle struct data ''' - comp_idx = self.particle_container.get_comp_index(comp_name) - - data_array = [] + particle_data = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): - soa = pti.soa() - idx = soa.GetRealData(comp_idx) if copy_to_host: - data_array.append(idx.to_numpy(copy=True)) + particle_data.append(pti.aos().to_numpy(copy=True)) else: + if libwarpx.amr.Config.have_gpu: + libwarpx.amr.Print( + "get_particle_structs: cupy does not yet support structs. " + "https://github.com/cupy/cupy/issues/2031" + "Did you mean copy_to_host=True?" + ) xp, cupy_status = load_cupy() if cupy_status is not None: libwarpx.amr.Print(cupy_status) - - data_array.append(xp.array(idx, copy=False)) - - return data_array + aos_arr = xp.array(pti.aos(), copy=False) # void blobs for cupy + particle_data.append(aos_arr) + return particle_data - def get_particle_int_arrays(self, comp_name, level, copy_to_host=False): + def get_particle_arrays(self, comp_name, level, copy_to_host=False): ''' - This returns a list of numpy or cupy arrays containing the particle int array data + This returns a list of numpy or cupy arrays containing the particle array data on each tile for this process. Unless copy_to_host is specified, the data for the arrays are not @@ -264,52 +266,12 @@ def get_particle_int_arrays(self, comp_name, level, copy_to_host=False): List of arrays The requested particle array data ''' - comp_idx = self.particle_container.get_icomp_index(comp_name) - - data_array = [] - for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): - soa = pti.soa() - idx = soa.GetIntData(comp_idx) - if copy_to_host: - data_array.append(idx.to_numpy(copy=True)) - else: - xp, cupy_status = load_cupy() - if cupy_status is not None: - libwarpx.amr.Print(cupy_status) - - data_array.append(xp.array(idx, copy=False)) - - return data_array - - - def get_particle_idcpu_arrays(self, level, copy_to_host=False): - ''' - This returns a list of numpy or cupy arrays containing the particle idcpu data - on each tile for this process. - - Unless copy_to_host is specified, the data for the arrays are not - copied, but share the underlying memory buffer with WarpX. The - arrays are fully writeable. - - Parameters - ---------- - level : int - The refinement level to reference (default=0) - - copy_to_host : bool - For GPU-enabled runs, one can either return the GPU - arrays (the default) or force a device-to-host copy. - - Returns - ------- + comp_idx = self.particle_container.get_comp_index(comp_name) - List of arrays - The requested particle array data - ''' data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): soa = pti.soa() - idx = soa.GetIdCPUData() + idx = soa.GetRealData(comp_idx) if copy_to_host: data_array.append(idx.to_numpy(copy=True)) else: @@ -322,31 +284,6 @@ def get_particle_idcpu_arrays(self, level, copy_to_host=False): return data_array - def get_particle_idcpu(self, level=0, copy_to_host=False): - ''' - Return a list of numpy or cupy arrays containing the particle 'idcpu' - numbers on each tile. - - Parameters - ---------- - - level : int - The refinement level to reference (default=0) - - copy_to_host : bool - For GPU-enabled runs, one can either return the GPU - arrays (the default) or force a device-to-host copy. - - Returns - ------- - - List of arrays - The requested particle idcpu - ''' - return self.get_particle_idcpu_arrays(level, copy_to_host=copy_to_host) - idcpu = property(get_particle_idcpu) - - def get_particle_id(self, level=0, copy_to_host=False): ''' Return a list of numpy or cupy arrays containing the particle 'id' @@ -368,8 +305,8 @@ def get_particle_id(self, level=0, copy_to_host=False): List of arrays The requested particle ids ''' - idcpu = self.get_particle_idcpu(level, copy_to_host) - return [libwarpx.amr.unpack_ids(tile) for tile in idcpu] + structs = self.get_particle_structs(level, copy_to_host) + return [libwarpx.amr.unpack_ids(struct['cpuid']) for struct in structs] def get_particle_cpu(self, level=0, copy_to_host=False): @@ -393,8 +330,8 @@ def get_particle_cpu(self, level=0, copy_to_host=False): List of arrays The requested particle cpus ''' - idcpu = self.get_particle_idcpu(level, copy_to_host) - return [libwarpx.amr.unpack_cpus(tile) for tile in idcpu] + structs = self.get_particle_structs(level, copy_to_host) + return [libwarpx.amr.unpack_cpus(struct['cpuid']) for struct in structs] def get_particle_x(self, level=0, copy_to_host=False): @@ -418,7 +355,20 @@ def get_particle_x(self, level=0, copy_to_host=False): List of arrays The requested particle x position ''' - return self.get_particle_real_arrays('x', level, copy_to_host=copy_to_host) + if copy_to_host: + # Use the numpy version of cosine + xp = np + else: + xp, cupy_status = load_cupy() + + structs = self.get_particle_structs(level, copy_to_host) + if libwarpx.geometry_dim == '3d' or libwarpx.geometry_dim == '2d': + return [struct['x'] for struct in structs] + elif libwarpx.geometry_dim == 'rz': + theta = self.get_particle_theta(level, copy_to_host) + return [struct['x']*xp.cos(theta) for struct, theta in zip(structs, theta)] + elif libwarpx.geometry_dim == '1d': + raise Exception('get_particle_x: There is no x coordinate with 1D Cartesian') xp = property(get_particle_x) @@ -443,7 +393,20 @@ def get_particle_y(self, level=0, copy_to_host=False): List of arrays The requested particle y position ''' - return self.get_particle_real_arrays('y', level, copy_to_host=copy_to_host) + if copy_to_host: + # Use the numpy version of sine + xp = np + else: + xp, cupy_status = load_cupy() + + structs = self.get_particle_structs(level, copy_to_host) + if libwarpx.geometry_dim == '3d': + return [struct['y'] for struct in structs] + elif libwarpx.geometry_dim == 'rz': + theta = self.get_particle_theta(level, copy_to_host) + return [struct['x']*xp.sin(theta) for struct, theta in zip(structs, theta)] + elif libwarpx.geometry_dim == '1d' or libwarpx.geometry_dim == '2d': + raise Exception('get_particle_y: There is no y coordinate with 1D or 2D Cartesian') yp = property(get_particle_y) @@ -470,12 +433,11 @@ def get_particle_r(self, level=0, copy_to_host=False): ''' xp, cupy_status = load_cupy() + structs = self.get_particle_structs(level, copy_to_host) if libwarpx.geometry_dim == 'rz': - return self.get_particle_x(level, copy_to_host) + return [struct['x'] for struct in structs] elif libwarpx.geometry_dim == '3d': - x = self.get_particle_x(level, copy_to_host) - y = self.get_particle_y(level, copy_to_host) - return xp.sqrt(x**2 + y**2) + return [xp.sqrt(struct['x']**2 + struct['y']**2) for struct in structs] elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': raise Exception('get_particle_r: There is no r coordinate with 1D or 2D Cartesian') rp = property(get_particle_r) @@ -505,11 +467,10 @@ def get_particle_theta(self, level=0, copy_to_host=False): xp, cupy_status = load_cupy() if libwarpx.geometry_dim == 'rz': - return self.get_particle_real_arrays('theta', level, copy_to_host) + return self.get_particle_arrays('theta', level, copy_to_host) elif libwarpx.geometry_dim == '3d': - x = self.get_particle_x(level, copy_to_host) - y = self.get_particle_y(level, copy_to_host) - return xp.arctan2(y, x) + structs = self.get_particle_structs(level, copy_to_host) + return [xp.arctan2(struct['y'], struct['x']) for struct in structs] elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': raise Exception('get_particle_theta: There is no theta coordinate with 1D or 2D Cartesian') thetap = property(get_particle_theta) @@ -536,7 +497,13 @@ def get_particle_z(self, level=0, copy_to_host=False): List of arrays The requested particle z position ''' - return self.get_particle_real_arrays('z', level, copy_to_host=copy_to_host) + structs = self.get_particle_structs(level, copy_to_host) + if libwarpx.geometry_dim == '3d': + return [struct['z'] for struct in structs] + elif libwarpx.geometry_dim == 'rz' or libwarpx.geometry_dim == '2d': + return [struct['y'] for struct in structs] + elif libwarpx.geometry_dim == '1d': + return [struct['x'] for struct in structs] zp = property(get_particle_z) @@ -561,7 +528,7 @@ def get_particle_weight(self, level=0, copy_to_host=False): List of arrays The requested particle weight ''' - return self.get_particle_real_arrays('w', level, copy_to_host=copy_to_host) + return self.get_particle_arrays('w', level, copy_to_host=copy_to_host) wp = property(get_particle_weight) @@ -586,7 +553,7 @@ def get_particle_ux(self, level=0, copy_to_host=False): List of arrays The requested particle x momentum ''' - return self.get_particle_real_arrays('ux', level, copy_to_host=copy_to_host) + return self.get_particle_arrays('ux', level, copy_to_host=copy_to_host) uxp = property(get_particle_ux) @@ -611,7 +578,7 @@ def get_particle_uy(self, level=0, copy_to_host=False): List of arrays The requested particle y momentum ''' - return self.get_particle_real_arrays('uy', level, copy_to_host=copy_to_host) + return self.get_particle_arrays('uy', level, copy_to_host=copy_to_host) uyp = property(get_particle_uy) @@ -637,7 +604,7 @@ def get_particle_uz(self, level=0, copy_to_host=False): The requested particle z momentum ''' - return self.get_particle_real_arrays('uz', level, copy_to_host=copy_to_host) + return self.get_particle_arrays('uz', level, copy_to_host=copy_to_host) uzp = property(get_particle_uz) @@ -753,6 +720,70 @@ def get_particle_boundary_buffer_size(self, species_name, boundary, local=False) ) + def get_particle_boundary_buffer_structs( + self, species_name, boundary, level, copy_to_host=False + ): + ''' + This returns a list of numpy or cupy arrays containing the particle struct data + for a species that has been scraped by a specific simulation boundary. The + particle data is represented as a structured array and contains the + particle 'x', 'y', 'z', and 'idcpu'. + + Unless copy_to_host is specified, the data for the structs are + not copied, but share the underlying memory buffer with WarpX. The + arrays are fully writeable. + + Note that cupy does not support structs: + https://github.com/cupy/cupy/issues/2031 + and will return arrays of binary blobs for the AoS (DP: ``"|V24"``). If copied + to host via copy_to_host, we correct for the right numpy AoS type. + + Parameters + ---------- + + species_name : str + The species name that the data will be returned for + + boundary : str + The boundary from which to get the scraped particle data in the + form x/y/z_hi/lo or eb. + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle struct data + ''' + particle_container = self.particle_buffer.get_particle_container( + species_name, self._get_boundary_number(boundary) + ) + + particle_data = [] + for pti in libwarpx.libwarpx_so.BoundaryBufferParIter(particle_container, level): + if copy_to_host: + particle_data.append(pti.aos().to_numpy(copy=True)) + else: + if libwarpx.amr.Config.have_gpu: + libwarpx.amr.Print( + "get_particle_structs: cupy does not yet support structs. " + "https://github.com/cupy/cupy/issues/2031" + "Did you mean copy_to_host=True?" + ) + xp, cupy_status = load_cupy() + if cupy_status is not None: + libwarpx.amr.Print(cupy_status) + aos_arr = xp.array(pti.aos(), copy=False) # void blobs for cupy + particle_data.append(aos_arr) + return particle_data + + def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level): ''' This returns a list of numpy or cupy arrays containing the particle array data diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H index fa00f6288f9..e88b522f148 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H @@ -143,19 +143,19 @@ struct LorentzTransformParticles const amrex::ParticleReal uzp = uz_old_p * weight_old + uz_new_p * weight_new; #if defined (WARPX_DIM_3D) - dst.m_rdata[PIdx::x][i_dst] = xp; - dst.m_rdata[PIdx::y][i_dst] = yp; - dst.m_rdata[PIdx::z][i_dst] = zp; + dst.m_aos[i_dst].pos(0) = xp; + dst.m_aos[i_dst].pos(1) = yp; + dst.m_aos[i_dst].pos(2) = zp; #elif defined (WARPX_DIM_RZ) - dst.m_rdata[PIdx::x][i_dst] = std::sqrt(xp*xp + yp*yp); - dst.m_rdata[PIdx::z][i_dst] = zp; + dst.m_aos[i_dst].pos(0) = std::sqrt(xp*xp + yp*yp); + dst.m_aos[i_dst].pos(1) = zp; dst.m_rdata[PIdx::theta][i_dst] = std::atan2(yp, xp); #elif defined (WARPX_DIM_XZ) - dst.m_rdata[PIdx::x][i_dst] = xp; - dst.m_rdata[PIdx::z][i_dst] = zp; + dst.m_aos[i_dst].pos(0) = xp; + dst.m_aos[i_dst].pos(1) = zp; amrex::ignore_unused(yp); #elif defined (WARPX_DIM_1D_Z) - dst.m_rdata[PIdx::z][i_dst] = zp; + dst.m_aos[i_dst].pos(0) = zp; amrex::ignore_unused(xp, yp); #else amrex::ignore_unused(xp, yp, zp); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp index c224bc4f871..980047e3b46 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp @@ -94,6 +94,9 @@ FlushFormatAscent::WriteParticles(const amrex::Vector& particle_di // get names of real comps std::map real_comps_map = pc->getParticleComps(); + // WarpXParticleContainer compile-time extra AoS attributes (Real): 0 + // WarpXParticleContainer compile-time extra AoS attributes (int): 0 + // WarpXParticleContainer compile-time extra SoA attributes (Real): PIdx::nattribs // not an efficient search, but N is small... for(int j = 0; j < PIdx::nattribs; ++j) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 00c28eead51..5f59cd723da 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -178,8 +178,8 @@ FlushFormatCheckpoint::CheckpointParticles ( Vector real_names; Vector int_names; - // note: positions skipped here, since we reconstruct a plotfile SoA from them real_names.push_back("weight"); + real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); @@ -189,12 +189,9 @@ FlushFormatCheckpoint::CheckpointParticles ( #endif // get the names of the real comps - // note: skips the mandatory AMREX_SPACEDIM positions for pure SoA - real_names.resize(pc->NumRealComps() - AMREX_SPACEDIM); + real_names.resize(pc->NumRealComps()); auto runtime_rnames = pc->getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) { - real_names[x.second + PIdx::nattribs - AMREX_SPACEDIM] = x.first; - } + for (auto const& x : runtime_rnames) { real_names[x.second+PIdx::nattribs] = x.first; } // and the int comps int_names.resize(pc->NumIntComps()); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 5ec067a6eac..df73ed34c94 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -355,8 +355,8 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, Vector int_flags; Vector real_flags; - // note: positions skipped here, since we reconstruct a plotfile SoA from them real_names.push_back("weight"); + real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); @@ -366,21 +366,14 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, #endif // get the names of the real comps - - // note: skips the mandatory AMREX_SPACEDIM positions for pure SoA - real_names.resize(tmp.NumRealComps() - AMREX_SPACEDIM); + real_names.resize(tmp.NumRealComps()); auto runtime_rnames = tmp.getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) { - real_names[x.second + PIdx::nattribs - AMREX_SPACEDIM] = x.first; - } + for (auto const& x : runtime_rnames) { real_names[x.second+PIdx::nattribs] = x.first; } // plot any "extra" fields by default real_flags = part_diag.m_plot_flags; real_flags.resize(tmp.NumRealComps(), 1); - // note: skip the mandatory AMREX_SPACEDIM positions for pure SoA - real_flags.erase(real_flags.begin(), real_flags.begin() + AMREX_SPACEDIM); - // and the names int_names.resize(tmp.NumIntComps()); auto runtime_inames = tmp.getParticleRuntimeiComps(); diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index a8bb9303fe1..7ca5e6541d7 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -160,7 +160,7 @@ MultiParticleContainer::Restart (const std::string& dir) ); } - for (int j = PIdx::nattribs-AMREX_SPACEDIM; j < nr; ++j) { + for (int j = PIdx::nattribs; j < nr; ++j) { const auto& comp_name = real_comp_names[j]; auto current_comp_names = pc->getParticleComps(); auto search = current_comp_names.find(comp_name); diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 24ad0e64ea8..9f45392bb0a 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -431,6 +431,8 @@ void FieldProbe::ComputeDiags (int step) { const auto getPosition = GetParticlePosition(pti); auto setPosition = SetParticlePosition(pti); + const auto& aos = pti.GetArrayOfStructs(); + const auto* AMREX_RESTRICT m_structs = aos().dataPtr(); auto const np = pti.numParticles(); if (update_particles_moving_window) @@ -480,8 +482,6 @@ void FieldProbe::ComputeDiags (int step) ParticleReal* const AMREX_RESTRICT part_Bz = attribs[FieldProbePIdx::Bz].dataPtr(); ParticleReal* const AMREX_RESTRICT part_S = attribs[FieldProbePIdx::S].dataPtr(); - auto * const AMREX_RESTRICT idcpu = pti.GetStructOfArrays().GetIdCPUData().data(); - const auto &xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const std::array &dx = WarpX::CellSize(lev); @@ -556,7 +556,7 @@ void FieldProbe::ComputeDiags (int step) amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); long idx = ip*noutputs; - dvp[idx++] = amrex::ParticleIDWrapper{idcpu[ip]}; // all particles created on IO cpu + dvp[idx++] = m_structs[ip].id(); dvp[idx++] = xp; dvp[idx++] = yp; dvp[idx++] = zp; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H index 7d59ade5dc6..c85bf8fd541 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H @@ -24,14 +24,7 @@ struct FieldProbePIdx { enum { -#if !defined (WARPX_DIM_1D_Z) - x, -#endif -#if defined (WARPX_DIM_3D) - y, -#endif - z, - Ex, Ey, Ez, + Ex = 0, Ey, Ez, Bx, By, Bz, S, //!< the Poynting vector #ifdef WARPX_DIM_RZ @@ -47,14 +40,9 @@ struct FieldProbePIdx * nattribs tells the particle container to allot 7 SOA values. */ class FieldProbeParticleContainer - : public amrex::ParticleContainerPureSoA + : public amrex::ParticleContainer<0, 0, FieldProbePIdx::nattribs> { public: - static constexpr int NStructReal = 0; - static constexpr int NStructInt = 0; - static constexpr int NReal = FieldProbePIdx::nattribs; - static constexpr int NInt = 0; - FieldProbeParticleContainer (amrex::AmrCore* amr_core); ~FieldProbeParticleContainer() override = default; @@ -64,9 +52,9 @@ public: FieldProbeParticleContainer& operator= ( FieldProbeParticleContainer&& ) = default; //! amrex iterator for our number of attributes - using iterator = amrex::ParIterSoA; + using iterator = amrex::ParIter<0, 0, FieldProbePIdx::nattribs, 0>; //! amrex iterator for our number of attributes (read-only) - using const_iterator = amrex::ParConstIterSoA; + using const_iterator = amrex::ParConstIter<0, 0, FieldProbePIdx::nattribs, 0>; //! similar to WarpXParticleContainer::AddNParticles but does not include u(x,y,z) void AddNParticles (int lev, amrex::Vector const & x, amrex::Vector const & y, amrex::Vector const & z); diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp index 18137fe9b2c..1fd741ddc47 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp @@ -59,7 +59,7 @@ using namespace amrex; FieldProbeParticleContainer::FieldProbeParticleContainer (AmrCore* amr_core) - : ParticleContainerPureSoA(amr_core->GetParGDB()) + : ParticleContainer<0, 0, FieldProbePIdx::nattribs>(amr_core->GetParGDB()) { SetParticleSize(); } @@ -89,17 +89,33 @@ FieldProbeParticleContainer::AddNParticles (int lev, * is then coppied to the permament tile which is stored on the particle * (particle_tile). */ - using PinnedTile = typename ContainerLike::ParticleTileType; + using PinnedTile = ParticleTile, + NArrayReal, NArrayInt, + amrex::PinnedArenaAllocator>; PinnedTile pinned_tile; pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps()); for (int i = 0; i < np; i++) { - auto & idcpu_data = pinned_tile.GetStructOfArrays().GetIdCPUData(); - idcpu_data.push_back(0); - amrex::ParticleIDWrapper{idcpu_data.back()} = ParticleType::NextID(); - amrex::ParticleCPUWrapper(idcpu_data.back()) = ParallelDescriptor::MyProc(); + ParticleType p; + p.id() = ParticleType::NextID(); + p.cpu() = ParallelDescriptor::MyProc(); +#if defined(WARPX_DIM_3D) + p.pos(0) = x[i]; + p.pos(1) = y[i]; + p.pos(2) = z[i]; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ignore_unused(y); + p.pos(0) = x[i]; + p.pos(1) = z[i]; +#elif defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(x, y); + p.pos(0) = z[i]; +#endif + + // write position, cpu id, and particle id to particle + pinned_tile.push_back(p); } // write Real attributes (SoA) to particle initialized zero @@ -109,13 +125,7 @@ FieldProbeParticleContainer::AddNParticles (int lev, #ifdef WARPX_DIM_RZ pinned_tile.push_back_real(FieldProbePIdx::theta, np, 0.0); #endif -#if !defined (WARPX_DIM_1D_Z) - pinned_tile.push_back_real(FieldProbePIdx::x, x); -#endif -#if defined (WARPX_DIM_3D) - pinned_tile.push_back_real(FieldProbePIdx::y, y); -#endif - pinned_tile.push_back_real(FieldProbePIdx::z, z); + pinned_tile.push_back_real(FieldProbePIdx::Ex, np, 0.0); pinned_tile.push_back_real(FieldProbePIdx::Ey, np, 0.0); pinned_tile.push_back_real(FieldProbePIdx::Ez, np, 0.0); diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index b4e07b51982..893b00a5f00 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -56,7 +56,8 @@ namespace auto const & plev = pc.GetParticles(lev); auto const & ptile = plev.at(box_index); - auto const np = ptile.numParticles(); + auto const & aos = ptile.GetArrayOfStructs(); + auto const np = aos.numParticles(); num_macro_particles += np; } diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 110f1ada649..e3b7b893d0a 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -41,7 +41,7 @@ class WarpXParticleCounter { public: using ParticleContainer = typename WarpXParticleContainer::ContainerLike; - using ParticleIter = typename amrex::ParIterSoA; + using ParticleIter = typename amrex::ParIter<0, 0, PIdx::nattribs, 0, amrex::PinnedArenaAllocator>; WarpXParticleCounter (ParticleContainer* pc); [[nodiscard]] unsigned long GetTotalNumParticles () const {return m_Total;} @@ -77,7 +77,7 @@ class WarpXOpenPMDPlot { public: using ParticleContainer = typename WarpXParticleContainer::ContainerLike; - using ParticleIter = typename amrex::ParConstIterSoA; + using ParticleIter = typename amrex::ParConstIter<0, 0, PIdx::nattribs, 0, amrex::PinnedArenaAllocator>; /** Initialize openPMD I/O routines * diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index d9b70a2e18d..71d96a47927 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -18,9 +18,11 @@ #include "WarpX.H" #include "OpenPMDHelpFunction.H" +#include #include #include +#include #include #include #include @@ -546,13 +548,6 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { // see openPMD ED-PIC extension for namings // note: an underscore separates the record name from its component // for non-scalar records -#if !defined (WARPX_DIM_1D_Z) - real_names.push_back("position_x"); -#endif -#if defined (WARPX_DIM_3D) - real_names.push_back("position_y"); -#endif - real_names.push_back("position_z"); real_names.push_back("weighting"); real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); @@ -737,7 +732,77 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, contributed_particles = true; - // save particle properties + // get position and particle ID from aos + // note: this implementation iterates the AoS 4x... + // if we flush late as we do now, we can also copy out the data in one go + const auto &aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile + { + // Save positions +#if defined(WARPX_DIM_RZ) + { + const std::shared_ptr z( + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + for (auto i = 0; i < numParticleOnTile; i++) { + z.get()[i] = aos[i].pos(1); // {0: "r", 1: "z"} + } + std::string const positionComponent = "z"; + currSpecies["position"]["z"].storeChunk(z, {offset}, {numParticleOnTile64}); + } + + // reconstruct x and y from polar coordinates r, theta + auto const& soa = pti.GetStructOfArrays(); + amrex::ParticleReal const* theta = soa.GetRealData(PIdx::theta).dataPtr(); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(theta != nullptr, "openPMD: invalid theta pointer."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(int(soa.GetRealData(PIdx::theta).size()) == numParticleOnTile, + "openPMD: theta and tile size do not match"); + { + const std::shared_ptr< amrex::ParticleReal > x( + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p){ delete[] p; } + ); + const std::shared_ptr< amrex::ParticleReal > y( + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p){ delete[] p; } + ); + for (auto i=0; i curr( + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + for (auto i = 0; i < numParticleOnTile; i++) { + curr.get()[i] = aos[i].pos(currDim); + } + std::string const positionComponent = positionComponents[currDim]; + currSpecies["position"][positionComponent].storeChunk(curr, {offset}, + {numParticleOnTile64}); + } +#endif + + // save particle ID after converting it to a globally unique ID + const std::shared_ptr ids( + new uint64_t[numParticleOnTile], + [](uint64_t const *p) { delete[] p; } + ); + for (auto i = 0; i < numParticleOnTile; i++) { + ids.get()[i] = ablastr::particles::localIDtoGlobal(static_cast(aos[i].id()), static_cast(aos[i].cpu())); + } + const auto *const scalar = openPMD::RecordComponent::SCALAR; + currSpecies["id"][scalar].storeChunk(ids, {offset}, {numParticleOnTile64}); + + } + // save "extra" particle properties in AoS and SoA SaveRealProperty(pti, currSpecies, offset, @@ -838,9 +903,10 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, std::set< std::string > addedRecords; // add meta-data per record only once for (auto idx=0; idxNumRealComps(); idx++) { - if (write_real_comp[idx]) { + auto ii = ParticleContainer::NStructReal + idx; // jump over extra AoS names + if (write_real_comp[ii]) { // handle scalar and non-scalar records by name - const auto [record_name, component_name] = detail::name2openPMD(real_comp_names[idx]); + const auto [record_name, component_name] = detail::name2openPMD(real_comp_names[ii]); auto currRecord = currSpecies[record_name]; // meta data for ED-PIC extension @@ -861,9 +927,10 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, } } for (auto idx=0; idx(numParticleOnTile); + auto const numParticleOnTile64 = static_cast( numParticleOnTile ); + auto const& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile auto const& soa = pti.GetStructOfArrays(); + // first we concatenate the AoS into contiguous arrays + { + // note: WarpX does not yet use extra AoS Real attributes + for( auto idx=0; idx d( + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p){ delete[] p; } + ); + + for( auto kk=0; kk AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator() (PData& ptd, int i, + void operator() (const PData& ptd, int i, const amrex::RealVect& /*pos*/, const amrex::RealVect& /*normal*/, amrex::RandomEngine const& /*engine*/) const noexcept { - ptd.id(i) = -ptd.id(i); + auto& p = ptd.m_aos[i]; + p.id() = -p.id(); } }; } diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index 312b3762a7e..d6196c35f44 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -170,12 +170,13 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t auto& tile = pti.GetParticleTile(); auto ptd = tile.getParticleTileData(); const auto np = tile.numParticles(); + amrex::Particle<0,0> * const particles = tile.GetArrayOfStructs()().data(); auto phi = (*distance_to_eb[lev])[pti].array(); // signed distance function amrex::ParallelForRNG( np, [=] AMREX_GPU_DEVICE (const int ip, amrex::RandomEngine const& engine) noexcept { // skip particles that are already flagged for removal - if (ptd.id(ip) < 0) return; + if (particles[ip].id() < 0) return; amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index c69f07acdb2..5c90dab25e6 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -72,8 +72,7 @@ class BinaryCollision final // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = ParticleTileType::ParticleTileDataType; - using ParticleBins = amrex::DenseBins; + using ParticleBins = amrex::DenseBins; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using index_type = ParticleBins::index_type; @@ -262,6 +261,9 @@ public: const amrex::ParticleReal q1 = species_1.getCharge(); const amrex::ParticleReal m1 = species_1.getMass(); auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); + // Needed to access the particle id + ParticleType * AMREX_RESTRICT + particle_ptr_1 = ptile_1.GetArrayOfStructs()().data(); amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); #if defined WARPX_DIM_1D_Z @@ -369,7 +371,7 @@ public: soa_1, soa_1, product_species_vector, tile_products_data, - m1, m1, + particle_ptr_1, particle_ptr_1, m1, m1, products_mass, p_mask, products_np, copy_species1, copy_species2, p_pair_indices_1, p_pair_indices_2, @@ -401,6 +403,9 @@ public: const amrex::ParticleReal q1 = species_1.getCharge(); const amrex::ParticleReal m1 = species_1.getMass(); auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); + // Needed to access the particle id + ParticleType * AMREX_RESTRICT + particle_ptr_1 = ptile_1.GetArrayOfStructs()().data(); // - Species 2 const auto soa_2 = ptile_2.getParticleTileData(); index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); @@ -408,6 +413,9 @@ public: const amrex::ParticleReal q2 = species_2.getCharge(); const amrex::ParticleReal m2 = species_2.getMass(); auto get_position_2 = GetParticlePosition(ptile_2, getpos_offset); + // Needed to access the particle id + ParticleType * AMREX_RESTRICT + particle_ptr_2 = ptile_2.GetArrayOfStructs()().data(); amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); #if defined WARPX_DIM_1D_Z @@ -527,7 +535,7 @@ public: soa_1, soa_2, product_species_vector, tile_products_data, - m1, m2, + particle_ptr_1, particle_ptr_2, m1, m2, products_mass, p_mask, products_np, copy_species1, copy_species2, p_pair_indices_1, p_pair_indices_2, diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index cfdc36d3c50..feb7acf81d3 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -23,13 +23,10 @@ * \brief This functor performs pairwise Coulomb collision on a single cell by calling the function * ElasticCollisionPerez. It also reads and contains the Coulomb logarithm. */ -class PairWiseCoulombCollisionFunc -{ +class PairWiseCoulombCollisionFunc{ // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = ParticleTileType::ParticleTileDataType; - using ParticleBins = amrex::DenseBins; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H index ab01eba2c81..c1be307b811 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H @@ -38,8 +38,7 @@ class DSMC final // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = ParticleTileType::ParticleTileDataType; - using ParticleBins = amrex::DenseBins; + using ParticleBins = amrex::DenseBins; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using index_type = ParticleBins::index_type; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index 5a7a4f3237a..c1fb7ee7e38 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -10,9 +10,6 @@ #define SPLIT_AND_SCATTER_FUNC_H_ #include "Particles/Collision/ScatteringProcess.H" -#include "Particles/NamedComponentParticleContainer.H" - -#include /** * \brief Function that performs the particle scattering and injection due @@ -58,6 +55,8 @@ int splitScatteringParticles ( const auto ptile1_data = ptile1.getParticleTileData(); const auto ptile2_data = ptile2.getParticleTileData(); + const Long minus_one_long = -1; + ParallelForRNG(n_total_pairs, [=] AMREX_GPU_DEVICE (int i, RandomEngine const& engine) noexcept { @@ -71,37 +70,20 @@ int splitScatteringParticles ( // starting with the parent particles auto& w1 = ptile1_data.m_rdata[PIdx::w][p_pair_indices_1[i]]; auto& w2 = ptile2_data.m_rdata[PIdx::w][p_pair_indices_2[i]]; - uint64_t* AMREX_RESTRICT idcpu1 = ptile1_data.m_idcpu; - uint64_t* AMREX_RESTRICT idcpu2 = ptile2_data.m_idcpu; - - // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX - // to replace the following lambda. - auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) - { - constexpr amrex::Long minus_one_long = -1; - uint64_t tmp = 0; - amrex::ParticleIDWrapper wrapper(tmp); - wrapper = minus_one_long; -#if defined(AMREX_USE_OMP) -#pragma omp atomic write - idcpu = wrapper.m_idata; -#else - auto *old_ptr = reinterpret_cast(&idcpu); - amrex::Gpu::Atomic::Exch(old_ptr, (unsigned long long) wrapper.m_idata); -#endif - }; // Remove p_pair_reaction_weight[i] from the colliding particles' weights. // If the colliding particle weight decreases to zero, remove particle by // setting its id to -1. Gpu::Atomic::AddNoRet(&w1, -p_pair_reaction_weight[i]); if (w1 <= 0._prt) { - atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); + auto& p = ptile1_data.m_aos[p_pair_indices_1[i]]; + p.atomicSetID(minus_one_long); } Gpu::Atomic::AddNoRet(&w2, -p_pair_reaction_weight[i]); if (w2 <= 0._prt) { - atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); + auto& p = ptile2_data.m_aos[p_pair_indices_2[i]]; + p.atomicSetID(minus_one_long); } // Set the child particle properties appropriately diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index b2a2112ca68..397536b67bf 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -33,13 +33,10 @@ * creation functor. * This functor also reads and contains the fusion multiplier. */ -class NuclearFusionFunc -{ +class NuclearFusionFunc{ // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = ParticleTileType::ParticleTileDataType; - using ParticleBins = amrex::DenseBins; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; @@ -157,13 +154,12 @@ public: // other species and we need to decrease their weight accordingly. // c1 corresponds to the minimum number of times a particle of species 1 will be paired // with a particle of species 2. Same for c2. - // index_type(1): https://github.com/AMReX-Codes/amrex/pull/3684 - const index_type c1 = amrex::max(NI2/NI1, index_type(1)); - const index_type c2 = amrex::max(NI1/NI2, index_type(1)); + const index_type c1 = amrex::max(NI2/NI1,1u); + const index_type c2 = amrex::max(NI1/NI2,1u); // multiplier ratio to take into account unsampled pairs const auto multiplier_ratio = static_cast( - m_isSameSpecies ? 2*max_N - 1 : max_N); + (m_isSameSpecies)?(2u*max_N - 1):(max_N)); #if (defined WARPX_DIM_RZ) amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 0b51d6b4b61..7b29267ec32 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -22,12 +22,10 @@ namespace { // Define shortcuts for frequently-used type names - using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - using ParticleType = typename WarpXParticleContainer::ParticleType; - using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; - using ParticleBins = amrex::DenseBins; - using index_type = typename ParticleBins::index_type; + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using ParticleType = WarpXParticleContainer::ParticleType; + using ParticleBins = amrex::DenseBins; + using index_type = ParticleBins::index_type; /** * \brief This function initializes the momentum of the alpha particles produced from diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H index 52e9db8aa94..be3f5b2d957 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H @@ -24,9 +24,7 @@ namespace { // Define shortcuts for frequently-used type names using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = ParticleTileType::ParticleTileDataType; - using ParticleBins = amrex::DenseBins; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; /** diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index 3928c74223d..dc830b477df 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -30,15 +30,13 @@ * \brief This functor creates particles produced from a binary collision and sets their initial * properties (position, momentum, weight). */ -class ParticleCreationFunc -{ +class ParticleCreationFunc{ // Define shortcuts for frequently-used type names - using ParticleType = typename WarpXParticleContainer::ParticleType; - using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; - using ParticleBins = amrex::DenseBins; - using index_type = typename ParticleBins::index_type; - using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using ParticleType = WarpXParticleContainer::ParticleType; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleBins = amrex::DenseBins; + using index_type = ParticleBins::index_type; + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; public: /** @@ -71,6 +69,12 @@ public: * @param[in, out] soa_1 struct of array data of the first colliding particle species * @param[in, out] soa_2 struct of array data of the second colliding particle species * @param[out] tile_products array containing tile data of the product particles. + * @param[out] particle_ptr_1 pointer to data of the first colliding particle species. Is + * needed to set the id of a particle to -1 in order to delete it when its weight + * reaches 0. + * @param[out] particle_ptr_2 pointer to data of the second colliding particle species. Is + * needed to set the id of a particle to -1 in order to delete it when its weight + * reaches 0. * @param[in] m1 mass of the first colliding particle species * @param[in] m2 mass of the second colliding particle species * @param[in] products_mass array storing the mass of product particles @@ -98,6 +102,7 @@ public: const SoaData_type& soa_1, const SoaData_type& soa_2, const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, + ParticleType* particle_ptr_1, ParticleType* particle_ptr_2, const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, const amrex::Vector& products_mass, const index_type* AMREX_RESTRICT p_mask, @@ -132,8 +137,6 @@ public: amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; - uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; - uint64_t* AMREX_RESTRICT idcpu2 = soa_2.m_idcpu; // Create necessary GPU vectors, that will be used in the kernel below amrex::Vector soa_products; @@ -202,32 +205,16 @@ public: amrex::Gpu::Atomic::AddNoRet(&w2[p_pair_indices_2[i]], -p_pair_reaction_weight[i]); - // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX - // to replace the following lambda. - auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) - { - constexpr amrex::Long minus_one_long = -1; - uint64_t tmp = 0; - amrex::ParticleIDWrapper wrapper(tmp); - wrapper = minus_one_long; -#if defined(AMREX_USE_OMP) -#pragma omp atomic write - idcpu = wrapper.m_idata; -#else - auto *old_ptr = reinterpret_cast(&idcpu); - amrex::Gpu::Atomic::Exch(old_ptr, (unsigned long long) wrapper.m_idata); -#endif - }; - // If the colliding particle weight decreases to zero, remove particle by // setting its id to -1 + constexpr amrex::Long minus_one_long = -1; if (w1[p_pair_indices_1[i]] <= amrex::ParticleReal(0.)) { - atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); + particle_ptr_1[p_pair_indices_1[i]].atomicSetID(minus_one_long); } if (w2[p_pair_indices_2[i]] <= amrex::ParticleReal(0.)) { - atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); + particle_ptr_2[p_pair_indices_2[i]].atomicSetID(minus_one_long); } // Initialize the product particles' momentum, using a function depending on the @@ -307,14 +294,12 @@ private: * \brief This class does nothing and is used as second template parameter for binary collisions * that do not create particles. */ -class NoParticleCreationFunc -{ - using ParticleType = typename WarpXParticleContainer::ParticleType; - using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; - using ParticleBins = amrex::DenseBins; - using index_type = typename ParticleBins::index_type; - using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; +class NoParticleCreationFunc{ + using ParticleType = WarpXParticleContainer::ParticleType; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleBins = amrex::DenseBins; + using index_type = ParticleBins::index_type; + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; public: NoParticleCreationFunc () = default; @@ -328,6 +313,7 @@ public: const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, amrex::Vector& /*pc_products*/, ParticleTileType** /*tile_products*/, + ParticleType* /*particle_ptr_1*/, ParticleType* /*particle_ptr_2*/, const amrex::ParticleReal& /*m1*/, const amrex::ParticleReal& /*m2*/, const amrex::Vector& /*products_mass*/, const index_type* /*p_mask*/, const amrex::Vector& /*products_np*/, diff --git a/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H b/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H index 3b8f72f4b84..42259512b0d 100644 --- a/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H +++ b/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H @@ -12,7 +12,7 @@ /* \brief Shuffle array according to Fisher-Yates algorithm. * Only shuffle the part between is <= i < ie, n = ie-is. * T_index shall be - * amrex::DenseBins::index_type + * amrex::DenseBins::index_type */ template diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index d0822789015..d0db678dfda 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -252,7 +252,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio const int n_rz_azimuthal_modes, amrex::Real* cost, const long load_balance_costs_update_algo, - const amrex::DenseBins& a_bins, + const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, const amrex::IntVect& a_tbox_max_size, diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 7343a8c5626..efe6efcc788 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -592,7 +592,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, int n_rz_azimuthal_modes, amrex::Real* cost, long load_balance_costs_update_algo, - const amrex::DenseBins& a_bins, + const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, const amrex::IntVect& a_tbox_max_size) diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index ca00b56323a..5abc9282d4f 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -167,7 +167,7 @@ public: p_ux, p_uy, p_uz, engine); - amrex::ParticleIDWrapper{src.m_idcpu[i_src]} = -1; // destroy photon after pair generation + src.m_aos[i_src].id() = -1; //destroy photon after pair generation } private: diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index f3099bf997f..8ba5c63ad57 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -22,7 +22,6 @@ #include #include #include -#include #include #include @@ -238,11 +237,12 @@ void cleanLowEnergyPhotons( const int old_size, const int num_added, const amrex::ParticleReal energy_threshold) { - auto& soa = ptile.GetStructOfArrays(); - auto p_idcpu = soa.GetIdCPUData().data() + old_size; + auto pp = ptile.GetArrayOfStructs()().data() + old_size; + + const auto& soa = ptile.GetStructOfArrays(); const auto p_ux = soa.GetRealData(PIdx::ux).data() + old_size; - const auto p_uy = soa.GetRealData(PIdx::uy).data() + old_size; - const auto p_uz = soa.GetRealData(PIdx::uz).data() + old_size; + const auto p_uy = soa.GetRealData(PIdx::uy).data() + old_size; + const auto p_uz = soa.GetRealData(PIdx::uz).data() + old_size; //The square of the energy threshold const auto energy_threshold2 = std::max( @@ -251,6 +251,8 @@ void cleanLowEnergyPhotons( amrex::ParallelFor(num_added, [=] AMREX_GPU_DEVICE (int ip) noexcept { + auto& p = pp[ip]; + const auto ux = p_ux[ip]; const auto uy = p_uy[ip]; const auto uz = p_uz[ip]; @@ -260,8 +262,8 @@ void cleanLowEnergyPhotons( constexpr amrex::ParticleReal me_c = PhysConst::m_e*PhysConst::c; const auto phot_energy2 = (ux*ux + uy*uy + uz*uz)*me_c*me_c; - if (phot_energy2 < energy_threshold2) { - amrex::ParticleIDWrapper{p_idcpu[ip]} = -1; + if (phot_energy2 < energy_threshold2){ + p.id() = - 1; } }); } diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index fac94ff20a3..e6fa308431c 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -56,9 +56,10 @@ public: * \brief Method to initialize runtime attributes. Does nothing for LaserParticleContainer. */ void DefaultInitializeRuntimeAttributes ( - typename ContainerLike::ParticleTileType& /*pinned_tile*/, - int /*n_external_attr_real*/, - int /*n_external_attr_int*/) final {} + amrex::ParticleTile, + NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& /*pinned_tile*/, + const int /*n_external_attr_real*/, + const int /*n_external_attr_int*/) final {} void ReadHeader (std::istream& is) final; diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index e04e7dba6df..3be0886425d 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -18,19 +18,12 @@ #include -/** Real Particle Attributes stored in amrex::ParticleContainer's struct of array +/** Particle Attributes stored in amrex::ParticleContainer's struct of array */ struct PIdx { enum { -#if !defined (WARPX_DIM_1D_Z) - x, -#endif -#if defined (WARPX_DIM_3D) - y, -#endif - z, - w, ///< weight + w = 0, ///< weight ux, uy, uz, #ifdef WARPX_DIM_RZ theta, ///< RZ needs all three position components @@ -39,15 +32,6 @@ struct PIdx }; }; -/** Integer Particle Attributes stored in amrex::ParticleContainer's struct of array - */ -struct PIdxInt -{ - enum { - nattribs ///< number of attributes - }; -}; - /** Particle Container class that allows to add/access particle components * with a name (string) instead of doing so with an integer index. * (The "components" are all the particle quantities - except those @@ -61,11 +45,11 @@ struct PIdxInt */ template class T_Allocator=amrex::DefaultAllocator> class NamedComponentParticleContainer : -public amrex::ParticleContainerPureSoA +public amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator> { public: /** Construct an empty NamedComponentParticleContainer **/ - NamedComponentParticleContainer () : amrex::ParticleContainerPureSoA() {} + NamedComponentParticleContainer () : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>() {} /** Construct a NamedComponentParticleContainer from an AmrParGDB object * @@ -77,15 +61,8 @@ public: * AMR hierarchy. Usually, this is generated by an AmrCore or AmrLevel object. */ NamedComponentParticleContainer (amrex::AmrParGDB* amr_pgdb) - : amrex::ParticleContainerPureSoA(amr_pgdb) { + : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>(amr_pgdb) { // build up the map of string names to particle component numbers -#if !defined (WARPX_DIM_1D_Z) - particle_comps["x"] = PIdx::x; -#endif -#if defined (WARPX_DIM_3D) - particle_comps["y"] = PIdx::y; -#endif - particle_comps["z"] = PIdx::z; particle_comps["w"] = PIdx::w; particle_comps["ux"] = PIdx::ux; particle_comps["uy"] = PIdx::uy; @@ -108,12 +85,12 @@ public: * @param p_ricomps name-to-index map for run-time integer components */ NamedComponentParticleContainer( - amrex::ParticleContainerPureSoA && pc, + amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator> && pc, std::map p_comps, std::map p_icomps, std::map p_rcomps, std::map p_ricomps) - : amrex::ParticleContainerPureSoA(std::move(pc)), + : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>(std::move(pc)), particle_comps(std::move(p_comps)), particle_icomps(std::move(p_icomps)), particle_runtime_comps(std::move(p_rcomps)), @@ -141,7 +118,7 @@ public: NamedComponentParticleContainer make_alike () const { auto tmp = NamedComponentParticleContainer( - amrex::ParticleContainerPureSoA::template make_alike(), + amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::template make_alike(), particle_comps, particle_icomps, particle_runtime_comps, @@ -150,10 +127,10 @@ public: return tmp; } - using amrex::ParticleContainerPureSoA::NumRealComps; - using amrex::ParticleContainerPureSoA::NumIntComps; - using amrex::ParticleContainerPureSoA::AddRealComp; - using amrex::ParticleContainerPureSoA::AddIntComp; + using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::NumRealComps; + using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::NumIntComps; + using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::AddRealComp; + using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::AddIntComp; /** Allocate a new run-time real component * diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 88304bd8a9c..54c4396379d 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -50,7 +50,7 @@ struct CopyAndTimestamp { void operator() (const DstData& dst, const SrcData& src, int src_i, int dst_i) const noexcept { - dst.m_idcpu[dst_i] = src.m_idcpu[src_i]; + dst.m_aos[dst_i] = src.m_aos[src_i]; for (int j = 0; j < SrcData::NAR; ++j) { dst.m_rdata[j][dst_i] = src.m_rdata[j][src_i]; } @@ -222,7 +222,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles"); - using PIter = amrex::ParConstIterSoA; + using PIter = amrex::ParConstIter<0,0,PIdx::nattribs>; const auto& warpx_instance = WarpX::GetInstance(); const amrex::Geometry& geom = warpx_instance.Geom(0); auto plo = geom.ProbLoArray(); diff --git a/Source/Particles/ParticleCreation/SmartCopy.H b/Source/Particles/ParticleCreation/SmartCopy.H index 6a6ceb3d290..2c04baa18bb 100644 --- a/Source/Particles/ParticleCreation/SmartCopy.H +++ b/Source/Particles/ParticleCreation/SmartCopy.H @@ -26,7 +26,7 @@ * type. Second, if a given component name is found in both the src * and the dst, then the src value is copied. * - * Particle positions and id numbers are always copied. + * Particle structs - positions and id numbers - are always copied. * * You don't create this directly - use the SmartCopyFactory object below. */ @@ -48,6 +48,9 @@ struct SmartCopy void operator() (DstData& dst, const SrcData& src, int i_src, int i_dst, amrex::RandomEngine const& engine) const noexcept { + // the particle struct is always copied over + dst.m_aos[i_dst] = src.m_aos[i_src]; + // initialize the real components for (int j = 0; j < DstData::NAR; ++j) { dst.m_rdata[j][i_dst] = initializeRealValue(m_policy_real[j], engine); diff --git a/Source/Particles/ParticleCreation/SmartCreate.H b/Source/Particles/ParticleCreation/SmartCreate.H index 7fb854ee083..67d7767a5d3 100644 --- a/Source/Particles/ParticleCreation/SmartCreate.H +++ b/Source/Particles/ParticleCreation/SmartCreate.H @@ -14,8 +14,6 @@ #include #include #include -#include -#include /** * \brief This is a functor for performing a "smart create" that works @@ -49,23 +47,23 @@ struct SmartCreate const int id = 0) const noexcept { #if defined(WARPX_DIM_3D) - prt.m_rdata[PIdx::x][i_prt] = x; - prt.m_rdata[PIdx::y][i_prt] = y; - prt.m_rdata[PIdx::z][i_prt] = z; + prt.m_aos[i_prt].pos(0) = x; + prt.m_aos[i_prt].pos(1) = y; + prt.m_aos[i_prt].pos(2) = z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - prt.m_rdata[PIdx::x][i_prt] = x; - prt.m_rdata[PIdx::z][i_prt] = z; + prt.m_aos[i_prt].pos(0) = x; + prt.m_aos[i_prt].pos(1) = z; amrex::ignore_unused(y); #else - prt.m_rdata[PIdx::z][i_prt] = z; + prt.m_aos[i_prt].pos(0) = z; amrex::ignore_unused(x,y); #endif - amrex::ParticleIDWrapper{prt.m_idcpu[i_prt]} = id; - amrex::ParticleCPUWrapper{prt.m_idcpu[i_prt]} = cpu; + prt.m_aos[i_prt].cpu() = cpu; + prt.m_aos[i_prt].id() = id; - // initialize the real components after position - for (int j = AMREX_SPACEDIM; j < PartData::NAR; ++j) { + // initialize the real components + for (int j = 0; j < PartData::NAR; ++j) { prt.m_rdata[j][i_prt] = initializeRealValue(m_policy_real[j], engine); } for (int j = 0; j < prt.m_num_runtime_real; ++j) { diff --git a/Source/Particles/ParticleCreation/SmartUtils.H b/Source/Particles/ParticleCreation/SmartUtils.H index 6b3d396900d..732a12bb729 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.H +++ b/Source/Particles/ParticleCreation/SmartUtils.H @@ -17,7 +17,6 @@ #include #include #include -#include #include #include @@ -61,12 +60,12 @@ void setNewParticleIDs (PTile& ptile, int old_size, int num_added) } const int cpuid = amrex::ParallelDescriptor::MyProc(); - auto ptd = ptile.getParticleTileData(); + auto pp = ptile.GetArrayOfStructs()().data() + old_size; amrex::ParallelFor(num_added, [=] AMREX_GPU_DEVICE (int ip) noexcept { - auto const new_id = ip + old_size; - amrex::ParticleIDWrapper{ptd.m_idcpu[new_id]} = pid+ip; - amrex::ParticleCPUWrapper{ptd.m_idcpu[new_id]} = cpuid; + auto& p = pp[ip]; + p.id() = pid+ip; + p.cpu() = cpuid; }); } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index edf91a84526..a12ae75f629 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -268,9 +268,11 @@ public: * @param[in] engine the random engine, used in initialization of QED optical depths */ void DefaultInitializeRuntimeAttributes ( - typename ContainerLike::ParticleTileType& pinned_tile, - int n_external_attr_real, - int n_external_attr_int) final; + amrex::ParticleTile, + NArrayReal, NArrayInt, + amrex::PinnedArenaAllocator>& pinned_tile, + int n_external_attr_real, + int n_external_attr_int) final; /** * \brief Apply NCI Godfrey filter to all components of E and B before gather diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index a8ed42ce419..e39cd79b55d 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -198,8 +198,8 @@ namespace * and avoid any possible undefined behavior before the next call to redistribute) and sets * the particle id to -1 so that it can be effectively deleted. * - * \param idcpu particle id soa data - * \param pa particle real soa data + * \param p particle aos data + * \param pa particle soa data * \param ip index for soa data * \param do_field_ionization whether species has ionization * \param pi ionization level data @@ -210,21 +210,20 @@ namespace */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void ZeroInitializeAndSetNegativeID ( - uint64_t * AMREX_RESTRICT idcpu, - const GpuArray& pa, long& ip, + ParticleType& p, const GpuArray& pa, long& ip, const bool& do_field_ionization, int* pi #ifdef WARPX_QED - ,const bool& has_quantum_sync, amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR - ,const bool& has_breit_wheeler, amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_BW + ,const bool& has_quantum_sync, amrex::ParticleReal* p_optical_depth_QSR + ,const bool& has_breit_wheeler, amrex::ParticleReal* p_optical_depth_BW #endif ) noexcept { - pa[PIdx::z][ip] = 0._rt; + p.pos(0) = 0._rt; #if (AMREX_SPACEDIM >= 2) - pa[PIdx::x][ip] = 0._rt; + p.pos(1) = 0._rt; #endif #if defined(WARPX_DIM_3D) - pa[PIdx::y][ip] = 0._rt; + p.pos(2) = 0._rt; #endif pa[PIdx::w ][ip] = 0._rt; pa[PIdx::ux][ip] = 0._rt; @@ -239,7 +238,7 @@ namespace if (has_breit_wheeler) {p_optical_depth_BW[ip] = 0._rt;} #endif - amrex::ParticleIDWrapper{idcpu[ip]} = -1; + p.id() = -1; } } @@ -781,9 +780,11 @@ PhysicalParticleContainer::AddPlasmaFromFile(PlasmaInjector & plasma_injector, void PhysicalParticleContainer::DefaultInitializeRuntimeAttributes ( - typename ContainerLike::ParticleTileType& pinned_tile, - int n_external_attr_real, - int n_external_attr_int) + amrex::ParticleTile, + NArrayReal, NArrayInt, + amrex::PinnedArenaAllocator>& pinned_tile, + const int n_external_attr_real, + const int n_external_attr_int) { ParticleCreation::DefaultInitializeRuntimeAttributes(pinned_tile, n_external_attr_real, n_external_attr_int, @@ -1083,7 +1084,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - int pid; + Long pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1092,7 +1093,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid) + static_cast(max_new_particles) < LongParticleIds::LastParticleID, + static_cast(pid + max_new_particles) < LastParticleID, "ERROR: overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); @@ -1103,16 +1104,16 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int DefineAndReturnParticleTile(lev, grid_id, tile_id); } - auto old_size = particle_tile.size(); + auto old_size = particle_tile.GetArrayOfStructs().size(); auto new_size = old_size + max_new_particles; particle_tile.resize(new_size); + ParticleType* pp = particle_tile.GetArrayOfStructs()().data() + old_size; auto& soa = particle_tile.GetStructOfArrays(); GpuArray pa; for (int ia = 0; ia < PIdx::nattribs; ++ia) { pa[ia] = soa.GetRealData(ia).data() + old_size; } - uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); @@ -1225,8 +1226,9 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int for (int i_part = 0; i_part < pcounts[index]; ++i_part) { long ip = poffset[index] + i_part; - amrex::ParticleIDWrapper{pa_idcpu[ip]} = pid+ip; - amrex::ParticleCPUWrapper{pa_idcpu[ip]} = cpuid; + ParticleType& p = pp[ip]; + p.id() = pid+ip; + p.cpu() = cpuid; const XDim3 r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? // In the refined injection region: use refinement ratio `lrrfac` inj_pos->getPositionUnitBox(i_part, lrrfac, engine) : @@ -1236,7 +1238,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int #if defined(WARPX_DIM_3D) if (!tile_realbox.contains(XDim3{pos.x,pos.y,pos.z})) { - ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1247,7 +1249,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(k); if (!tile_realbox.contains(XDim3{pos.x,pos.z,0.0_rt})) { - ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1258,7 +1260,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int #else amrex::ignore_unused(j,k); if (!tile_realbox.contains(XDim3{pos.z,0.0_rt,0.0_rt})) { - ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1297,7 +1299,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const Real z0 = applyBallisticCorrection(pos, inj_mom, gamma_boost, beta_boost, t); if (!inj_pos->insideBounds(xb, yb, z0)) { - ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1311,7 +1313,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // Remove particle if density below threshold if ( dens < density_min ){ - ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1329,7 +1331,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // If the particle is not within the lab-frame zmin, zmax, etc. // go to the next generated particle. if (!inj_pos->insideBounds(xb, yb, z0_lab)) { - ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1341,7 +1343,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int dens = inj_rho->getDensity(pos.x, pos.y, z0_lab); // Remove particle if density below threshold if ( dens < density_min ){ - ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1408,17 +1410,17 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int pa[PIdx::uz][ip] = u.z; #if defined(WARPX_DIM_3D) - pa[PIdx::x][ip] = pos.x; - pa[PIdx::y][ip] = pos.y; - pa[PIdx::z][ip] = pos.z; + p.pos(0) = pos.x; + p.pos(1) = pos.y; + p.pos(2) = pos.z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) #ifdef WARPX_DIM_RZ pa[PIdx::theta][ip] = theta; #endif - pa[PIdx::x][ip] = xb; - pa[PIdx::z][ip] = pos.z; + p.pos(0) = xb; + p.pos(1) = pos.z; #else - pa[PIdx::z][ip] = pos.z; + p.pos(0) = pos.z; #endif } }); @@ -1643,7 +1645,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - int pid; + Long pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1652,23 +1654,23 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid) + static_cast(max_new_particles) < LongParticleIds::LastParticleID, + static_cast(pid + max_new_particles) < LastParticleID, "overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); auto& particle_tile = tmp_pc.DefineAndReturnParticleTile(0, grid_id, tile_id); - auto old_size = particle_tile.size(); + auto old_size = particle_tile.GetArrayOfStructs().size(); auto new_size = old_size + max_new_particles; particle_tile.resize(new_size); + ParticleType* pp = particle_tile.GetArrayOfStructs()().data() + old_size; auto& soa = particle_tile.GetStructOfArrays(); GpuArray pa; for (int ia = 0; ia < PIdx::nattribs; ++ia) { pa[ia] = soa.GetRealData(ia).data() + old_size; } - uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); @@ -1766,8 +1768,9 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, for (int i_part = 0; i_part < pcounts[index]; ++i_part) { const long ip = poffset[index] + i_part; - amrex::ParticleIDWrapper{pa_idcpu[ip]} = pid+ip; - amrex::ParticleCPUWrapper{pa_idcpu[ip]} = cpuid; + ParticleType& p = pp[ip]; + p.id() = pid+ip; + p.cpu() = cpuid; // This assumes the flux_pos is of type InjectorPositionRandomPlane const XDim3 r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? @@ -1792,19 +1795,19 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // the particles will be within the domain. #if defined(WARPX_DIM_3D) if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.x,ppos.y,ppos.z})) { - amrex::ParticleIDWrapper{pa_idcpu[ip]} = -1; + p.id() = -1; continue; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(k); if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.x,ppos.z,0.0_prt})) { - amrex::ParticleIDWrapper{pa_idcpu[ip]} = -1; + p.id() = -1; continue; } #else amrex::ignore_unused(j,k); if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.z,0.0_prt,0.0_prt})) { - amrex::ParticleIDWrapper{pa_idcpu[ip]} = -1; + p.id() = -1; continue; } #endif @@ -1812,7 +1815,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // If the particle's initial position is not within or on the species's // xmin, xmax, ymin, ymax, zmin, zmax, go to the next generated particle. if (!flux_pos->insideBoundsInclusive(ppos.x, ppos.y, ppos.z)) { - amrex::ParticleIDWrapper{pa_idcpu[ip]} = -1; + p.id() = -1; continue; } @@ -1846,7 +1849,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, Real flux = inj_flux->getFlux(ppos.x, ppos.y, ppos.z, t); // Remove particle if flux is negative or 0 if ( flux <=0 ){ - amrex::ParticleIDWrapper{pa_idcpu[ip]} = -1; + p.id() = -1; continue; } @@ -1905,18 +1908,18 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, UpdatePosition(ppos.x, ppos.y, ppos.z, pu.x, pu.y, pu.z, t_fract); #if defined(WARPX_DIM_3D) - pa[PIdx::x][ip] = ppos.x; - pa[PIdx::y][ip] = ppos.y; - pa[PIdx::z][ip] = ppos.z; + p.pos(0) = ppos.x; + p.pos(1) = ppos.y; + p.pos(2) = ppos.z; #elif defined(WARPX_DIM_RZ) pa[PIdx::theta][ip] = std::atan2(ppos.y, ppos.x); - pa[PIdx::x][ip] = std::sqrt(ppos.x*ppos.x + ppos.y*ppos.y); - pa[PIdx::z][ip] = ppos.z; + p.pos(0) = std::sqrt(ppos.x*ppos.x + ppos.y*ppos.y); + p.pos(1) = ppos.z; #elif defined(WARPX_DIM_XZ) - pa[PIdx::x][ip] = ppos.x; - pa[PIdx::z][ip] = ppos.z; + p.pos(0) = ppos.x; + p.pos(1) = ppos.z; #else - pa[PIdx::z][ip] = ppos.z; + p.pos(0) = ppos.z; #endif } }); @@ -2339,22 +2342,20 @@ PhysicalParticleContainer::SplitParticles (int lev) split_offset[1] /= ppc_nd[1]; split_offset[2] /= ppc_nd[2]; } + // particle Array Of Structs data + auto& particles = pti.GetArrayOfStructs(); // particle Struct Of Arrays data auto& attribs = pti.GetAttribs(); auto& wp = attribs[PIdx::w ]; auto& uxp = attribs[PIdx::ux]; auto& uyp = attribs[PIdx::uy]; auto& uzp = attribs[PIdx::uz]; - - ParticleTileType& ptile = ParticlesAt(lev, pti); - auto& soa = ptile.GetStructOfArrays(); - uint64_t * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); - const long np = pti.numParticles(); for(int i=0; i> attr_int; pctmp_split.AddNParticles(lev, np_split_to_add, - xp, - yp, - zp, - uxp, - uyp, - uzp, - 1, - attr, + xp, yp, zp, uxp, uyp, uzp, + 1, attr, 0, attr_int, - 1, LongParticleIds::NoSplitParticleID); + 1, NoSplitParticleID); // Copy particles from tmp to current particle container constexpr bool local_flag = true; addParticles(pctmp_split,local_flag); diff --git a/Source/Particles/Pusher/GetAndSetPosition.H b/Source/Particles/Pusher/GetAndSetPosition.H index 44641557756..e4477a2a60d 100644 --- a/Source/Particles/Pusher/GetAndSetPosition.H +++ b/Source/Particles/Pusher/GetAndSetPosition.H @@ -30,26 +30,24 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, amrex::ParticleReal& y, amrex::ParticleReal& z) noexcept { - using namespace amrex::literals; - -#if defined(WARPX_DIM_RZ) - amrex::ParticleReal const theta = p.rdata(T_PIdx::theta); - amrex::ParticleReal const r = p.pos(T_PIdx::x); +#ifdef WARPX_DIM_RZ + const amrex::ParticleReal theta = p.rdata(T_PIdx::theta); + const amrex::ParticleReal r = p.pos(0); x = r*std::cos(theta); y = r*std::sin(theta); - z = p.pos(PIdx::z); -#elif defined(WARPX_DIM_3D) - x = p.pos(PIdx::x); - y = p.pos(PIdx::y); - z = p.pos(PIdx::z); -#elif defined(WARPX_DIM_XZ) - x = p.pos(PIdx::x); - y = 0_prt; - z = p.pos(PIdx::z); + z = p.pos(1); +#elif WARPX_DIM_3D + x = p.pos(0); + y = p.pos(1); + z = p.pos(2); +#elif WARPX_DIM_XZ + x = p.pos(0); + y = amrex::ParticleReal(0.0); + z = p.pos(1); #else - x = 0_prt; - y = 0_prt; - z = p.pos(PIdx::z); + x = amrex::ParticleReal(0.0); + y = amrex::ParticleReal(0.0); + z = p.pos(0); #endif } @@ -61,19 +59,10 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, template struct GetParticlePosition { + using PType = WarpXParticleContainer::ParticleType; using RType = amrex::ParticleReal; -#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - const RType* AMREX_RESTRICT m_x = nullptr; - const RType* AMREX_RESTRICT m_z = nullptr; -#elif defined(WARPX_DIM_3D) - const RType* AMREX_RESTRICT m_x = nullptr; - const RType* AMREX_RESTRICT m_y = nullptr; - const RType* AMREX_RESTRICT m_z = nullptr; -#elif defined(WARPX_DIM_1D_Z) - const RType* AMREX_RESTRICT m_z = nullptr; -#endif - + const PType* AMREX_RESTRICT m_structs = nullptr; #if defined(WARPX_DIM_RZ) const RType* m_theta = nullptr; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) @@ -95,19 +84,10 @@ struct GetParticlePosition template GetParticlePosition (const ptiType& a_pti, long a_offset = 0) noexcept { - const auto& soa = a_pti.GetStructOfArrays(); - -#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; - m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; -#elif defined(WARPX_DIM_3D) - m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; - m_y = soa.GetRealData(PIdx::y).dataPtr() + a_offset; - m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; -#elif defined(WARPX_DIM_1D_Z) - m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; -#endif + const auto& aos = a_pti.GetArrayOfStructs(); + m_structs = aos().dataPtr() + a_offset; #if defined(WARPX_DIM_RZ) + const auto& soa = a_pti.GetStructOfArrays(); m_theta = soa.GetRealData(T_PIdx::theta).dataPtr() + a_offset; #endif } @@ -118,23 +98,24 @@ struct GetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void operator() (const long i, RType& x, RType& y, RType& z) const noexcept { + const PType& p = m_structs[i]; #ifdef WARPX_DIM_RZ - RType const r = m_x[i]; + const RType r = p.pos(0); x = r*std::cos(m_theta[i]); y = r*std::sin(m_theta[i]); - z = m_z[i]; + z = p.pos(1); #elif WARPX_DIM_3D - x = m_x[i]; - y = m_y[i]; - z = m_z[i]; + x = p.pos(0); + y = p.pos(1); + z = p.pos(2); #elif WARPX_DIM_XZ - x = m_x[i]; + x = p.pos(0); y = m_y_default; - z = m_z[i]; + z = p.pos(1); #else x = m_x_default; y = m_y_default; - z = m_z[i]; + z = p.pos(0); #endif } @@ -146,22 +127,23 @@ struct GetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void AsStored (const long i, RType& x, RType& y, RType& z) const noexcept { + const PType& p = m_structs[i]; #ifdef WARPX_DIM_RZ - x = m_x[i]; + x = p.pos(0); y = m_theta[i]; - z = m_z[i]; + z = p.pos(1); #elif WARPX_DIM_3D - x = m_x[i]; - y = m_y[i]; - z = m_z[i]; + x = p.pos(0); + y = p.pos(1); + z = p.pos(2); #elif WARPX_DIM_XZ - x = m_x[i]; + x = p.pos(0); y = m_y_default; - z = m_z[i]; + z = p.pos(1); #else x = m_x_default; y = m_y_default; - z = m_z[i]; + z = p.pos(0); #endif } }; @@ -176,18 +158,10 @@ struct GetParticlePosition template struct SetParticlePosition { + using PType = WarpXParticleContainer::ParticleType; using RType = amrex::ParticleReal; -#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - RType* AMREX_RESTRICT m_x; - RType* AMREX_RESTRICT m_z; -#elif defined(WARPX_DIM_3D) - RType* AMREX_RESTRICT m_x; - RType* AMREX_RESTRICT m_y; - RType* AMREX_RESTRICT m_z; -#elif defined(WARPX_DIM_1D_Z) - RType* AMREX_RESTRICT m_z; -#endif + PType* AMREX_RESTRICT m_structs; #if defined(WARPX_DIM_RZ) RType* AMREX_RESTRICT m_theta; #endif @@ -195,18 +169,10 @@ struct SetParticlePosition template SetParticlePosition (const ptiType& a_pti, long a_offset = 0) noexcept { - auto& soa = a_pti.GetStructOfArrays(); -#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; - m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; -#elif defined(WARPX_DIM_3D) - m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; - m_y = soa.GetRealData(PIdx::y).dataPtr() + a_offset; - m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; -#elif defined(WARPX_DIM_1D_Z) - m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; -#endif + auto& aos = a_pti.GetArrayOfStructs(); + m_structs = aos().dataPtr() + a_offset; #if defined(WARPX_DIM_RZ) + auto& soa = a_pti.GetStructOfArrays(); m_theta = soa.GetRealData(T_PIdx::theta).dataPtr() + a_offset; #endif } @@ -224,17 +190,17 @@ struct SetParticlePosition #endif #ifdef WARPX_DIM_RZ m_theta[i] = std::atan2(y, x); - m_x[i] = std::sqrt(x*x + y*y); - m_z[i] = z; + m_structs[i].pos(0) = std::sqrt(x*x + y*y); + m_structs[i].pos(1) = z; #elif WARPX_DIM_3D - m_x[i] = x; - m_y[i] = y; - m_z[i] = z; + m_structs[i].pos(0) = x; + m_structs[i].pos(1) = y; + m_structs[i].pos(2) = z; #elif WARPX_DIM_XZ - m_x[i] = x; - m_z[i] = z; + m_structs[i].pos(0) = x; + m_structs[i].pos(1) = z; #else - m_z[i] = z; + m_structs[i].pos(0) = z; #endif } @@ -252,18 +218,18 @@ struct SetParticlePosition amrex::ignore_unused(x,y); #endif #ifdef WARPX_DIM_RZ - m_x[i] = x; + m_structs[i].pos(0) = x; m_theta[i] = y; - m_z[i] = z; + m_structs[i].pos(1) = z; #elif WARPX_DIM_3D - m_x[i] = x; - m_y[i] = y; - m_z[i] = z; + m_structs[i].pos(0) = x; + m_structs[i].pos(1) = y; + m_structs[i].pos(2) = z; #elif WARPX_DIM_XZ - m_x[i] = x; - m_z[i] = z; + m_structs[i].pos(0) = x; + m_structs[i].pos(1) = z; #else - m_z[i] = z; + m_structs[i].pos(0) = z; #endif } }; diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index 40f75c76eab..680e33ebe6a 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -60,7 +60,8 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, auto& ptile = pc->ParticlesAt(lev, pti); auto& soa = ptile.GetStructOfArrays(); amrex::ParticleReal * const AMREX_RESTRICT w = soa.GetRealData(PIdx::w).data(); - auto * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); + WarpXParticleContainer::ParticleType * const AMREX_RESTRICT + particle_ptr = ptile.GetArrayOfStructs()().data(); // Using this function means that we must loop over the cells in the ParallelFor. In the case // of the leveling thinning algorithm, it would have possibly been more natural and more @@ -113,7 +114,7 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, // Remove particle with probability 1 - particle_weight/level_weight if (random_number > w[indices[i]]/level_weight) { - amrex::ParticleIDWrapper{idcpu[indices[i]]} = -1; + particle_ptr[indices[i]].id() = -1; } // Set particle weight to level weight otherwise else diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp index 58e3450f47d..58511cfd5e7 100644 --- a/Source/Particles/Sorting/Partition.cpp +++ b/Source/Particles/Sorting/Partition.cpp @@ -61,7 +61,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // Initialize temporary arrays Gpu::DeviceVector inexflag; inexflag.resize(np); - Gpu::DeviceVector pid; + Gpu::DeviceVector pid; pid.resize(np); // First, partition particles into the larger buffer @@ -109,7 +109,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // - For each particle in the large buffer, find whether it is in // the smaller buffer, by looking up the mask. Store the answer in `inexflag`. amrex::ParallelFor( np - n_fine, - fillBufferFlagRemainingParticles(pti, bmasks, inexflag, Geom(lev), pid, int(n_fine)) ); + fillBufferFlagRemainingParticles(pti, bmasks, inexflag, Geom(lev), pid, n_fine) ); auto *const sep2 = stablePartition( sep, pid.end(), inexflag ); if (bmasks == gather_masks) { diff --git a/Source/Particles/Sorting/SortingUtils.H b/Source/Particles/Sorting/SortingUtils.H index ba7761bf48a..ac2c63e88f8 100644 --- a/Source/Particles/Sorting/SortingUtils.H +++ b/Source/Particles/Sorting/SortingUtils.H @@ -12,7 +12,6 @@ #include #include -#include /** \brief Fill the elements of the input vector with consecutive integer, @@ -20,7 +19,7 @@ * * \param[inout] v Vector of integers, to be filled by this routine */ -void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ); +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ); /** \brief Find the indices that would reorder the elements of `predicate` * so that the elements with non-zero value precede the other elements @@ -42,7 +41,7 @@ ForwardIterator stablePartition(ForwardIterator const index_begin, int const* AMREX_RESTRICT predicate_ptr = predicate.dataPtr(); int N = static_cast(std::distance(index_begin, index_end)); auto num_true = amrex::StablePartition(&(*index_begin), N, - [predicate_ptr] AMREX_GPU_DEVICE (int i) { return predicate_ptr[i]; }); + [predicate_ptr] AMREX_GPU_DEVICE (long i) { return predicate_ptr[i]; }); ForwardIterator sep = index_begin; std::advance(sep, num_true); @@ -50,7 +49,7 @@ ForwardIterator stablePartition(ForwardIterator const index_begin, // On CPU: Use std library ForwardIterator const sep = std::stable_partition( index_begin, index_end, - [&predicate](int i) { return predicate[i]; } + [&predicate](long i) { return predicate[i]; } ); #endif return sep; @@ -89,7 +88,7 @@ class fillBufferFlag // Extract simple structure that can be used directly on the GPU m_domain{geom.Domain()}, m_inexflag_ptr{inexflag.dataPtr()}, - m_ptd{pti.GetParticleTile().getConstParticleTileData()}, + m_particles{pti.GetArrayOfStructs().data()}, m_buffer_mask{(*bmasks)[pti].array()} { for (int idim=0; idim m_buffer_mask; amrex::GpuArray m_prob_lo; amrex::GpuArray m_inv_cell_size; @@ -140,12 +141,12 @@ class fillBufferFlagRemainingParticles amrex::iMultiFab const* bmasks, amrex::Gpu::DeviceVector& inexflag, amrex::Geometry const& geom, - amrex::Gpu::DeviceVector const& particle_indices, - int start_index ) : + amrex::Gpu::DeviceVector const& particle_indices, + long const start_index ) : m_domain{geom.Domain()}, // Extract simple structure that can be used directly on the GPU m_inexflag_ptr{inexflag.dataPtr()}, - m_ptd{pti.GetParticleTile().getConstParticleTileData()}, + m_particles{pti.GetArrayOfStructs().data()}, m_buffer_mask{(*bmasks)[pti].array()}, m_start_index{start_index}, m_indices_ptr{particle_indices.dataPtr()} @@ -158,11 +159,11 @@ class fillBufferFlagRemainingParticles AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator()( const int i ) const { + void operator()( const long i ) const { // Select a particle - auto const j = m_indices_ptr[i+m_start_index]; + auto const& p = m_particles[m_indices_ptr[i+m_start_index]]; // Find the index of the cell where this particle is located - amrex::IntVect const iv = amrex::getParticleCell( m_ptd, j, + amrex::IntVect const iv = amrex::getParticleCell( p, m_prob_lo, m_inv_cell_size, m_domain ); // Find the value of the buffer flag in this cell and // store it at the corresponding particle position in the array `inexflag` @@ -174,10 +175,10 @@ class fillBufferFlagRemainingParticles amrex::GpuArray m_inv_cell_size; amrex::Box m_domain; int* m_inexflag_ptr; - WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType const m_ptd; + WarpXParticleContainer::ParticleType const* m_particles; amrex::Array4 m_buffer_mask; - int const m_start_index; - int const* m_indices_ptr; + long const m_start_index; + long const* m_indices_ptr; }; /** \brief Functor that copies the elements of `src` into `dst`, @@ -194,7 +195,7 @@ class copyAndReorder copyAndReorder( amrex::Gpu::DeviceVector const& src, amrex::Gpu::DeviceVector& dst, - amrex::Gpu::DeviceVector const& indices ): + amrex::Gpu::DeviceVector const& indices ): // Extract simple structure that can be used directly on the GPU m_src_ptr{src.dataPtr()}, m_dst_ptr{dst.dataPtr()}, @@ -202,14 +203,14 @@ class copyAndReorder {} AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void operator()( const int ip ) const { + void operator()( const long ip ) const { m_dst_ptr[ip] = m_src_ptr[ m_indices_ptr[ip] ]; } private: T const* m_src_ptr; T* m_dst_ptr; - int const* m_indices_ptr; + long const* m_indices_ptr; }; #endif // WARPX_PARTICLES_SORTING_SORTINGUTILS_H_ diff --git a/Source/Particles/Sorting/SortingUtils.cpp b/Source/Particles/Sorting/SortingUtils.cpp index cd4b6a13c76..699119e8e18 100644 --- a/Source/Particles/Sorting/SortingUtils.cpp +++ b/Source/Particles/Sorting/SortingUtils.cpp @@ -8,7 +8,7 @@ #include "SortingUtils.H" -void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) { #ifdef AMREX_USE_GPU // On GPU: Use amrex diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index a244afc2389..33aa71d1c7d 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -49,10 +49,10 @@ class WarpXParIter - : public amrex::ParIterSoA + : public amrex::ParIter<0,0,PIdx::nattribs> { public: - using amrex::ParIterSoA::ParIterSoA; + using amrex::ParIter<0,0,PIdx::nattribs>::ParIter; WarpXParIter (ContainerType& pc, int level); @@ -89,7 +89,7 @@ public: * particle container classes (that store a collection of particles) derive. Derived * classes can be used for plasma particles, photon particles, or non-physical * particles (e.g., for the laser antenna). - * It derives from amrex::ParticleContainerPureSoA, where the + * It derives from amrex::ParticleContainer<0,0,PIdx::nattribs>, where the * template arguments stand for the number of int and amrex::Real SoA and AoS * data in amrex::Particle. * - AoS amrex::Real: x, y, z (default), 0 additional (first template @@ -164,9 +164,11 @@ public: * class. */ virtual void DefaultInitializeRuntimeAttributes ( - typename ContainerLike::ParticleTileType& pinned_tile, - int n_external_attr_real, - int n_external_attr_int) = 0; + amrex::ParticleTile, + NArrayReal, NArrayInt, + amrex::PinnedArenaAllocator>& pinned_tile, + int n_external_attr_real, + int n_external_attr_int) = 0; /// /// This pushes the particle positions by one half time step. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 0d927caacc8..85bb1c3f4b8 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -75,13 +75,13 @@ using namespace amrex; WarpXParIter::WarpXParIter (ContainerType& pc, int level) - : amrex::ParIterSoA(pc, level, + : amrex::ParIter<0,0,PIdx::nattribs>(pc, level, MFItInfo().SetDynamic(WarpX::do_dynamic_scheduling)) { } WarpXParIter::WarpXParIter (ContainerType& pc, int level, MFItInfo& info) - : amrex::ParIterSoA(pc, level, + : amrex::ParIter<0,0,PIdx::nattribs>(pc, level, info.SetDynamic(WarpX::do_dynamic_scheduling)) { } @@ -198,54 +198,52 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, // Redistribute() will move them to proper places. auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); - using PinnedTile = typename ContainerLike::ParticleTileType; + using PinnedTile = amrex::ParticleTile, + NArrayReal, NArrayInt, + amrex::PinnedArenaAllocator>; PinnedTile pinned_tile; pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps()); const std::size_t np = iend-ibegin; #ifdef WARPX_DIM_RZ - amrex::Vector r(np); amrex::Vector theta(np); #endif for (auto i = ibegin; i < iend; ++i) { - auto & idcpu_data = pinned_tile.GetStructOfArrays().GetIdCPUData(); - idcpu_data.push_back(0); - if (id==-1) { - amrex::ParticleIDWrapper{idcpu_data.back()} = ParticleType::NextID(); + ParticleType p; + if (id==-1) + { + p.id() = ParticleType::NextID(); } else { - amrex::ParticleIDWrapper{idcpu_data.back()} = id; + p.id() = id; } - amrex::ParticleCPUWrapper(idcpu_data.back()) = ParallelDescriptor::MyProc(); - -#ifdef WARPX_DIM_RZ - r[i-ibegin] = std::sqrt(x[i]*x[i] + y[i]*y[i]); - theta[i-ibegin] = std::atan2(y[i], x[i]); -#endif - } - - if (np > 0) - { + p.cpu() = amrex::ParallelDescriptor::MyProc(); #if defined(WARPX_DIM_3D) - pinned_tile.push_back_real(PIdx::x, x.data() + ibegin, x.data() + iend); - pinned_tile.push_back_real(PIdx::y, y.data() + ibegin, y.data() + iend); - pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); + p.pos(0) = x[i]; + p.pos(1) = y[i]; + p.pos(2) = z[i]; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(y); #ifdef WARPX_DIM_RZ - pinned_tile.push_back_real(PIdx::x, r.data(), r.data() + np); + theta[i-ibegin] = std::atan2(y[i], x[i]); + p.pos(0) = std::sqrt(x[i]*x[i] + y[i]*y[i]); #else - pinned_tile.push_back_real(PIdx::x, x.data() + ibegin, x.data() + iend); + p.pos(0) = x[i]; #endif - pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); + p.pos(1) = z[i]; #else //AMREX_SPACEDIM == 1 amrex::ignore_unused(x,y); - pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); + p.pos(0) = z[i]; #endif - pinned_tile.push_back_real(PIdx::w, attr_real[0].data() + ibegin, attr_real[0].data() + iend); + pinned_tile.push_back(p); + } + + if (np > 0) + { + pinned_tile.push_back_real(PIdx::w , attr_real[0].data() + ibegin, attr_real[0].data() + iend); pinned_tile.push_back_real(PIdx::ux, ux.data() + ibegin, ux.data() + iend); pinned_tile.push_back_real(PIdx::uy, uy.data() + ibegin, uy.data() + iend); pinned_tile.push_back_real(PIdx::uz, uz.data() + ibegin, uz.data() + iend); @@ -477,14 +475,15 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, //sort particles by bin WARPX_PROFILE_VAR_START(blp_sort); - amrex::DenseBins bins; + amrex::DenseBins bins; { auto& ptile = ParticlesAt(lev, pti); - auto ptd = ptile.getParticleTileData(); + auto& aos = ptile.GetArrayOfStructs(); + auto *pstruct_ptr = aos().dataPtr(); const int ntiles = numTilesInBox(box, true, bin_size); - bins.build(ptile.numParticles(), ptd, ntiles, + bins.build(ptile.numParticles(), pstruct_ptr, ntiles, [=] AMREX_GPU_HOST_DEVICE (const ParticleType& p) -> unsigned int { Box tbox; @@ -884,7 +883,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // HACK - sort particles by bin here. WARPX_PROFILE_VAR_START(blp_sort); - amrex::DenseBins bins; + amrex::DenseBins bins; { const Geometry& geom = Geom(lev); const auto dxi = geom.InvCellSizeArray(); @@ -892,15 +891,16 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const auto domain = geom.Domain(); auto& ptile = ParticlesAt(lev, pti); - auto ptd = ptile.getParticleTileData(); + auto& aos = ptile.GetArrayOfStructs(); + auto *pstruct_ptr = aos().dataPtr(); Box box = pti.validbox(); box.grow(ng_rho); const amrex::IntVect bin_size = WarpX::shared_tilesize; const int ntiles = numTilesInBox(box, true, bin_size); - bins.build(ptile.numParticles(), ptd, ntiles, - [=] AMREX_GPU_HOST_DEVICE (ParticleType const & p) -> unsigned int + bins.build(ptile.numParticles(), pstruct_ptr, ntiles, + [=] AMREX_GPU_HOST_DEVICE (const ParticleType& p) -> unsigned int { Box tbx; auto iv = getParticleCell(p, plo, dxi, domain); @@ -920,7 +920,8 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const auto domain = geom.Domain(); auto& ptile = ParticlesAt(lev, pti); - auto ptd = ptile.getParticleTileData(); + auto& aos = ptile.GetArrayOfStructs(); + auto *pstruct_ptr = aos().dataPtr(); Box box = pti.validbox(); box.grow(ng_rho); @@ -934,10 +935,9 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const auto bin_start = offsets_ptr[ibin]; const auto bin_stop = offsets_ptr[ibin+1]; if (bin_start < bin_stop) { - // static_cast until https://github.com/AMReX-Codes/amrex/pull/3684 - auto const i = static_cast(permutation[bin_start]); + auto p = pstruct_ptr[permutation[bin_start]]; Box tbx; - auto iv = getParticleCell(ptd, i, plo, dxi, domain); + auto iv = getParticleCell(p, plo, dxi, domain); AMREX_ASSERT(box.contains(iv)); [[maybe_unused]] auto tid = getTileIndex(iv, box, true, bin_size, tbx); AMREX_ASSERT(tid == ibin); @@ -1426,10 +1426,10 @@ WarpXParticleContainer::particlePostLocate(ParticleType& p, // Tag particle if goes to higher level. // It will be split later in the loop if (pld.m_lev == lev+1 - and p.id() != amrex::LongParticleIds::NoSplitParticleID + and p.id() != NoSplitParticleID and p.id() >= 0) { - p.id() = amrex::LongParticleIds::DoSplitParticleID; + p.id() = DoSplitParticleID; } if (pld.m_lev == lev-1){ @@ -1468,9 +1468,9 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ const Real zmax = Geom(lev).ProbHi(WARPX_ZINDEX); ParticleTileType& ptile = ParticlesAt(lev, pti); + ParticleType * const pp = ptile.GetArrayOfStructs()().data(); auto& soa = ptile.GetStructOfArrays(); - uint64_t * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); amrex::ParticleReal * const AMREX_RESTRICT ux = soa.GetRealData(PIdx::ux).data(); amrex::ParticleReal * const AMREX_RESTRICT uy = soa.GetRealData(PIdx::uy).data(); amrex::ParticleReal * const AMREX_RESTRICT uz = soa.GetRealData(PIdx::uz).data(); @@ -1479,9 +1479,10 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ amrex::ParallelForRNG( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i, amrex::RandomEngine const& engine) { + ParticleType& p = pp[i]; + // skip particles that are already flagged for removal - auto const id = amrex::ParticleIDWrapper{idcpu[i]}; - if (id < 0) { return; } + if (p.id() < 0) { return; } ParticleReal x, y, z; GetPosition.AsStored(i, x, y, z); @@ -1503,7 +1504,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ boundary_conditions, engine); if (particle_lost) { - amrex::ParticleIDWrapper{idcpu[i]} = -id; + p.id() = -p.id(); } else { SetPosition.AsStored(i, x, y, z); } diff --git a/Source/Python/Particles/ParticleBoundaryBuffer.cpp b/Source/Python/Particles/ParticleBoundaryBuffer.cpp index b04ac75e600..2a35faece9b 100644 --- a/Source/Python/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Python/Particles/ParticleBoundaryBuffer.cpp @@ -10,13 +10,13 @@ namespace warpx { class BoundaryBufferParIter - : public amrex::ParIterSoA + : public amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> { public: - using amrex::ParIterSoA::ParIterSoA; + using amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator>::ParIter; BoundaryBufferParIter(ContainerType& pc, int level) : - amrex::ParIterSoA(pc, level) {} + amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator>(pc, level) {} }; } @@ -24,9 +24,9 @@ void init_BoundaryBufferParIter (py::module& m) { py::class_< warpx::BoundaryBufferParIter, - amrex::ParIterSoA + amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> >(m, "BoundaryBufferParIter") - .def(py::init::ContainerType&, int>(), + .def(py::init::ContainerType&, int>(), py::arg("particle_container"), py::arg("level") ) ; diff --git a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp index d4f6a422dbe..600d56a62c9 100644 --- a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp +++ b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp @@ -13,6 +13,6 @@ void init_PinnedMemoryParticleContainer (py::module& m) { py::class_< PinnedMemoryParticleContainer, - amrex::ParticleContainerPureSoA + amrex::ParticleContainer<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> > pmpc (m, "PinnedMemoryParticleContainer"); } diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index 07793a373f3..1473a750941 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -12,11 +12,11 @@ void init_WarpXParIter (py::module& m) { py::class_< - WarpXParIter, amrex::ParIterSoA + WarpXParIter, amrex::ParIter<0,0,PIdx::nattribs> >(m, "WarpXParIter") - .def(py::init::ContainerType&, int>(), + .def(py::init::ContainerType&, int>(), py::arg("particle_container"), py::arg("level")) - .def(py::init::ContainerType&, int, amrex::MFItInfo&>(), + .def(py::init::ContainerType&, int, amrex::MFItInfo&>(), py::arg("particle_container"), py::arg("level"), py::arg("info")) ; @@ -26,11 +26,11 @@ void init_WarpXParticleContainer (py::module& m) { py::class_< WarpXParticleContainer, - amrex::ParticleContainerPureSoA + amrex::ParticleContainer<0, 0, PIdx::nattribs, 0> > wpc (m, "WarpXParticleContainer"); wpc .def("add_real_comp", - [](WarpXParticleContainer& pc, const std::string& name, bool comm) { pc.AddRealComp(name, comm); }, + [](WarpXParticleContainer& pc, const std::string& name, bool const comm) { pc.AddRealComp(name, comm); }, py::arg("name"), py::arg("comm") ) .def("add_n_particles", @@ -93,14 +93,6 @@ void init_WarpXParticleContainer (py::module& m) }, py::arg("comp_name") ) - .def("get_icomp_index", - [](WarpXParticleContainer& pc, std::string comp_name) - { - auto particle_comps = pc.getParticleiComps(); - return particle_comps.at(comp_name); - }, - py::arg("comp_name") - ) .def("num_local_tiles_at_level", &WarpXParticleContainer::numLocalTilesAtLevel, py::arg("level") diff --git a/Source/Utils/ParticleUtils.H b/Source/Utils/ParticleUtils.H index 7e3c89228ea..b04176d4d83 100644 --- a/Source/Utils/ParticleUtils.H +++ b/Source/Utils/ParticleUtils.H @@ -28,10 +28,9 @@ namespace ParticleUtils { * @param[in] mfi the MultiFAB iterator. * @param[in] ptile the particle tile. */ - amrex::DenseBins - findParticlesInEachCell (int lev, - amrex::MFIter const & mfi, - WarpXParticleContainer::ParticleTileType & ptile); + amrex::DenseBins + findParticlesInEachCell(int lev, amrex::MFIter const& mfi, + WarpXParticleContainer::ParticleTileType const& ptile); /** * \brief Return (relativistic) particle energy given velocity and mass. diff --git a/Source/Utils/ParticleUtils.cpp b/Source/Utils/ParticleUtils.cpp index b8207b61fa0..60e04f12b86 100644 --- a/Source/Utils/ParticleUtils.cpp +++ b/Source/Utils/ParticleUtils.cpp @@ -22,28 +22,24 @@ #include #include -namespace ParticleUtils -{ +namespace ParticleUtils { using namespace amrex; - // Define shortcuts for frequently-used type names - using ParticleType = typename WarpXParticleContainer::ParticleType; - using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; - using ParticleBins = DenseBins; - using index_type = typename ParticleBins::index_type; + using ParticleType = WarpXParticleContainer::ParticleType; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleBins = DenseBins; + using index_type = ParticleBins::index_type; /* Find the particles and count the particles that are in each cell. Note that this does *not* rearrange particle arrays */ ParticleBins - findParticlesInEachCell (int lev, - MFIter const & mfi, - ParticleTileType & ptile) { + findParticlesInEachCell( int const lev, MFIter const& mfi, + ParticleTileType const& ptile) { // Extract particle structures for this tile int const np = ptile.numParticles(); - auto ptd = ptile.getParticleTileData(); + ParticleType const* particle_ptr = ptile.GetArrayOfStructs()().data(); // Extract box properties Geometry const& geom = WarpX::GetInstance().Geom(lev); @@ -55,9 +51,9 @@ namespace ParticleUtils // Find particles that are in each cell; // results are stored in the object `bins`. ParticleBins bins; - bins.build(np, ptd, cbx, + bins.build(np, particle_ptr, cbx, // Pass lambda function that returns the cell index - [=] AMREX_GPU_DEVICE (ParticleType const & p) noexcept -> amrex::IntVect + [=] AMREX_GPU_DEVICE (const ParticleType& p) noexcept { return IntVect{AMREX_D_DECL( static_cast((p.pos(0)-plo[0])*dxi[0] - lo.x), @@ -68,4 +64,4 @@ namespace ParticleUtils return bins; } -} // namespace ParticleUtils +} diff --git a/Source/ablastr/particles/IndexHandling.H b/Source/ablastr/particles/IndexHandling.H new file mode 100644 index 00000000000..0ad5ca60446 --- /dev/null +++ b/Source/ablastr/particles/IndexHandling.H @@ -0,0 +1,41 @@ +/* Copyright 2019-2022 Axel Huebl + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef ABLASTR_INDEX_HANDLING_H +#define ABLASTR_INDEX_HANDLING_H + +#include + + +namespace ablastr::particles { + + /** A helper function to derive a globally unique particle ID + * + * @param[in] id AMReX particle ID (on local cpu/rank), AoS .id + * @param[in] cpu AMReX particle CPU (rank) at creation of the particle, AoS .cpu + * @return global particle ID that is unique and permanent in the whole simulation + */ + constexpr uint64_t + localIDtoGlobal (int const id, int const cpu) + { + static_assert(sizeof(int) * 2u <= sizeof(uint64_t), + "int size might cause collisions in global IDs"); + // implementation: + // - we cast both 32-bit (or smaller) ints to a 64bit unsigned int + // - this will leave half of the "upper range" bits in the 64bit unsigned int zeroed out + // because the corresponding (extended) value range was not part of the value range in + // the int representation + // - we bit-shift the cpu into the upper half of zero bits in the 64 bit unsigned int + // (imagine this step as "adding a per-cpu/rank offset to the local integers") + // - then we add this offset + // note: the add is expressed as bitwise OR (|) since this saves us from writing + // brackets for operator precedence between + and << + return uint64_t(id) | uint64_t(cpu) << 32u; + } + +} // namespace ablastr::particles + +#endif // ABLASTR_INDEX_HANDLING_H diff --git a/Source/ablastr/particles/ParticleMoments.H b/Source/ablastr/particles/ParticleMoments.H index b648ccb28aa..e45fb574cce 100644 --- a/Source/ablastr/particles/ParticleMoments.H +++ b/Source/ablastr/particles/ParticleMoments.H @@ -35,7 +35,7 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal> MinAndMaxPositions (T_PC const & pc) { - using ConstParticleTileDataType = typename T_PC::ParticleTileType::ConstParticleTileDataType; + using PType = typename T_PC::SuperParticleType; // Get min and max for the local rank amrex::ReduceOps< @@ -46,11 +46,11 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal, amrex::ParticleReal> >( pc, - [=] AMREX_GPU_DEVICE(const ConstParticleTileDataType& ptd, const int i) noexcept + [=] AMREX_GPU_DEVICE(PType const & p) noexcept { - const amrex::ParticleReal x = ptd.rdata(0)[i]; - const amrex::ParticleReal y = ptd.rdata(1)[i]; - const amrex::ParticleReal z = ptd.rdata(2)[i]; + amrex::ParticleReal const x = p.pos(0); + amrex::ParticleReal const y = p.pos(1); + amrex::ParticleReal const z = p.pos(2); return amrex::makeTuple(x, y, z, x, y, z); }, @@ -90,8 +90,7 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal> MeanAndStdPositions (T_PC const & pc) { - - using ConstParticleTileDataType = typename T_PC::ParticleTileType::ConstParticleTileDataType; + using PType = typename T_PC::SuperParticleType; amrex::ReduceOps< amrex::ReduceOpSum, amrex::ReduceOpSum, amrex::ReduceOpSum, @@ -104,14 +103,12 @@ namespace particles { amrex::ParticleReal> >( pc, - [=] AMREX_GPU_DEVICE(const ConstParticleTileDataType& ptd, const int i) noexcept + [=] AMREX_GPU_DEVICE(const PType& p) noexcept { - - const amrex::ParticleReal x = ptd.rdata(0)[i]; - const amrex::ParticleReal y = ptd.rdata(1)[i]; - const amrex::ParticleReal z = ptd.rdata(2)[i]; - - const amrex::ParticleReal w = ptd.rdata(T_RealSoAWeight)[i]; + amrex::ParticleReal const x = p.pos(0); + amrex::ParticleReal const y = p.pos(1); + amrex::ParticleReal const z = p.pos(2); + amrex::ParticleReal const w = p.rdata(T_RealSoAWeight); return amrex::makeTuple(x, x*x, y, y*y, z, z*z, w); }, From fa422b501814a8beb3e3a7ac2e13bc176e010fac Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 31 Jan 2024 11:25:37 -0800 Subject: [PATCH 126/176] AMReX: Latest `development` (#4654) --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- run_test.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index eca39369a08..9960cdfbb29 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach ab99ea69089f4fffbfd727ed965d5ceb3d905baa && cd - + cd ../amrex && git checkout --detach 689144d157a0106faf3d0ae89f8d90b0250cf975 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 51ae03b60fa..70c62b190fb 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = ab99ea69089f4fffbfd727ed965d5ceb3d905baa +branch = 689144d157a0106faf3d0ae89f8d90b0250cf975 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 3489e8ad100..ab11d70dfcc 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = ab99ea69089f4fffbfd727ed965d5ceb3d905baa +branch = 689144d157a0106faf3d0ae89f8d90b0250cf975 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 72173b2acb0..9b74c8db7fd 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "ab99ea69089f4fffbfd727ed965d5ceb3d905baa" +set(WarpX_amrex_branch "689144d157a0106faf3d0ae89f8d90b0250cf975" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/run_test.sh b/run_test.sh index 20c6638f4d0..48857d264cb 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach ab99ea69089f4fffbfd727ed965d5ceb3d905baa && cd - +cd amrex && git checkout --detach 689144d157a0106faf3d0ae89f8d90b0250cf975 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From bcc67e19b44b6a7b11d2302c9487fd19ec722662 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 1 Feb 2024 19:07:28 +0100 Subject: [PATCH 127/176] Docs: update instructions for Adastra supercomputer (CINES, France) (#4655) * update instructions for Adastra supercomputer * remove empty line * fix bug * fix bug * fix bug --- Docs/source/install/hpc/adastra.rst | 30 ++++++---- .../adastra_warpx.profile.example | 28 ++++----- .../adastra-cines/install_dependencies.sh | 60 +++++++++---------- Tools/machines/adastra-cines/submit.sh | 17 ++++-- 4 files changed, 74 insertions(+), 61 deletions(-) diff --git a/Docs/source/install/hpc/adastra.rst b/Docs/source/install/hpc/adastra.rst index 44b07985670..0b984d5e2be 100644 --- a/Docs/source/install/hpc/adastra.rst +++ b/Docs/source/install/hpc/adastra.rst @@ -31,18 +31,26 @@ If you are new to this system, **please see the following resources**: Preparation ----------- +The following instructions will install WarpX in the ``$SHAREDHOMEDIR`` directory, +which is shared among all the members of a given project. Due to the inode +quota enforced for this machine, a shared installation of WarpX is advised. + Use the following commands to download the WarpX source code: .. code-block:: bash - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + # If you have multiple projects, activate the project that you want to use with: + # + # myproject -a YOUR_PROJECT_NAME + # + git clone https://github.com/ECP-WarpX/WarpX.git $SHAREDHOMEDIR/src/warpx -We use system software modules, add environment hints and further dependencies via the file ``$HOME/adastra_warpx.profile``. +We use system software modules, add environment hints and further dependencies via the file ``$SHAREDHOMEDIR/adastra_warpx.profile``. Create it now: .. code-block:: bash - cp $HOME/src/warpx/Tools/machines/adastra-cines/adastra_warpx.profile.example $HOME/adastra_warpx.profile + cp $SHAREDHOMEDIR/src/warpx/Tools/machines/adastra-cines/adastra_warpx.profile.example $SHAREDHOMEDIR/adastra_warpx.profile .. dropdown:: Script Details :color: light @@ -53,8 +61,8 @@ Create it now: :language: bash Edit the 2nd line of this script, which sets the ``export proj=""`` variable using a text editor -such as ``nano``, ``emacs``, or ``vim`` (all available by default on -Adastra login nodes). +such as ``nano``, ``emacs``, or ``vim`` (all available by default on Adastra login nodes) and +uncomment the 3rd line (which sets ``$proj`` as the active project). .. important:: @@ -62,14 +70,14 @@ Adastra login nodes). .. code-block:: bash - source $HOME/adastra_warpx.profile + source $SHAREDHOMEDIR/adastra_warpx.profile Finally, since Adastra does not yet provide software modules for some of our dependencies, install them once: .. code-block:: bash - bash $HOME/src/warpx/Tools/machines/adastra-cines/install_dependencies.sh - source $HOME/sw/adastra/gpu/venvs/warpx-adastra/bin/activate + bash $SHAREDHOMEDIR/src/warpx/Tools/machines/adastra-cines/install_dependencies.sh + source $SHAREDHOMEDIR/sw/adastra/gpu/venvs/warpx-adastra/bin/activate .. dropdown:: Script Details :color: light @@ -89,13 +97,13 @@ Use the following :ref:`cmake commands ` to compile the applicat .. code-block:: bash - cd $HOME/src/warpx + cd $SHAREDHOMEDIR/src/warpx rm -rf build_adastra cmake -S . -B build_adastra -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_adastra -j 16 -The WarpX application executables are now in ``$HOME/src/warpx/build_adastra/bin/``. +The WarpX application executables are now in ``$SHAREDHOMEDIR/src/warpx/build_adastra/bin/``. Additionally, the following commands will install WarpX as a Python module: .. code-block:: bash @@ -119,7 +127,7 @@ If you already installed WarpX in the past and want to update it, start by getti .. code-block:: bash - cd $HOME/src/warpx + cd $SHAREDHOMEDIR/src/warpx # read the output of this command - does it look ok? git status diff --git a/Tools/machines/adastra-cines/adastra_warpx.profile.example b/Tools/machines/adastra-cines/adastra_warpx.profile.example index 23441638893..0d55e869d6a 100644 --- a/Tools/machines/adastra-cines/adastra_warpx.profile.example +++ b/Tools/machines/adastra-cines/adastra_warpx.profile.example @@ -1,30 +1,33 @@ -# please set your project account +# please set your project account and uncomment the following two lines #export proj=your_project_id +#myproject -a $proj # required dependencies +module purge +module load cpe/23.12 module load craype-accel-amd-gfx90a craype-x86-trento module load PrgEnv-cray +module load CCE-GPU-3.0.0 module load amd-mixed/5.2.3 -module load CPE-23.02-cce-15.0.1-GPU-softs # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${HOME}/sw/adastra/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/adastra/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support -module load boost/1.81.0-mpi-python3 +module load boost/1.83.0-mpi-python3 # optional: for openPMD support module load cray-hdf5-parallel -export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export PATH=${HOME}/sw/adastra/gpu/adios2-2.8.3/bin:${PATH} # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load cray-python/3.11.5 # fix system defaults: do not escape $ with a \ on tab completion shopt -s direxpand @@ -49,7 +52,4 @@ export AMREX_AMD_ARCH=gfx90a # compiler environment hints export CC=$(which cc) export CXX=$(which CC) -export FC=$(which ftn) -export CFLAGS="-I${ROCM_PATH}/include" -export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed" -export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64" +export FC=$(which amdflang) diff --git a/Tools/machines/adastra-cines/install_dependencies.sh b/Tools/machines/adastra-cines/install_dependencies.sh index 8a4cef4a2ec..b48bf144c2a 100755 --- a/Tools/machines/adastra-cines/install_dependencies.sh +++ b/Tools/machines/adastra-cines/install_dependencies.sh @@ -20,7 +20,7 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo # Remove old dependencies ##################################################### # -SW_DIR="${HOME}/sw/adastra/gpu" +SW_DIR="${SHAREDHOMEDIR}/sw/adastra/gpu" rm -rf ${SW_DIR} mkdir -p ${SW_DIR} @@ -34,62 +34,62 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # # BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] +if [ -d $SHAREDHOMEDIR/src/blaspp ] then - cd $HOME/src/blaspp + cd $SHAREDHOMEDIR/src/blaspp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone https://github.com/icl-utk-edu/blaspp.git $SHAREDHOMEDIR/src/blaspp fi -rm -rf $HOME/src/blaspp-adastra-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-adastra-gpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-adastra-gpu-build +rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build +CXX=$(which CC) cmake -S $SHAREDHOMEDIR/src/blaspp -B $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build --target install --parallel 16 +rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build # LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] +if [ -d $SHAREDHOMEDIR/src/lapackpp ] then - cd $HOME/src/lapackpp + cd $SHAREDHOMEDIR/src/lapackpp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone https://github.com/icl-utk-edu/lapackpp.git $SHAREDHOMEDIR/src/lapackpp fi -rm -rf $HOME/src/lapackpp-adastra-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-adastra-gpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-adastra-gpu-build +rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $SHAREDHOMEDIR/src/lapackpp -B $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build --target install --parallel 16 +rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build # c-blosc (I/O compression, for OpenPMD) -if [ -d $HOME/src/c-blosc ] +if [ -d $SHAREDHOMEDIR/src/c-blosc ] then # git repository is already there : else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $SHAREDHOMEDIR/src/c-blosc fi -rm -rf $HOME/src/c-blosc-ad-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-ad-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/adastra/gpu/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-ad-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-ad-build +rm -rf $SHAREDHOMEDIR/src/c-blosc-ad-build +cmake -S $SHAREDHOMEDIR/src/c-blosc -B $SHAREDHOMEDIR/src/c-blosc-ad-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build $SHAREDHOMEDIR/src/c-blosc-ad-build --target install --parallel 16 +rm -rf $SHAREDHOMEDIR/src/c-blosc-ad-build # ADIOS2 v. 2.8.3 (for OpenPMD) -if [ -d $HOME/src/adios2 ] +if [ -d $SHAREDHOMEDIR/src/adios2 ] then # git repository is already there : else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $SHAREDHOMEDIR/src/adios2 fi -rm -rf $HOME/src/adios2-ad-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-ad-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/adastra/gpu/adios2-2.8.3 -cmake --build $HOME/src/adios2-ad-build --target install -j 16 -rm -rf $HOME/src/adios2-ad-build +rm -rf $SHAREDHOMEDIR/src/adios2-ad-build +cmake -S $SHAREDHOMEDIR/src/adios2 -B $SHAREDHOMEDIR/src/adios2-ad-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build $SHAREDHOMEDIR/src/adios2-ad-build --target install -j 16 +rm -rf $SHAREDHOMEDIR/src/adios2-ad-build # Python ###################################################################### @@ -114,9 +114,9 @@ python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install --upgrade -r $SHAREDHOMEDIR/src/warpx/requirements.txt # optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt +python3 -m pip install -r $SHAREDHOMEDIR/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) #python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/rocm5.4.2 -#python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt +#python3 -m pip install -r $SHAREDHOMEDIR/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/adastra-cines/submit.sh b/Tools/machines/adastra-cines/submit.sh index 0cb75e86e69..15a2b292b58 100644 --- a/Tools/machines/adastra-cines/submit.sh +++ b/Tools/machines/adastra-cines/submit.sh @@ -1,22 +1,26 @@ #!/bin/bash -#SBATCH --job-name=warpx #SBATCH --account= +#SBATCH --job-name=warpx #SBATCH --constraint=MI250 -#SBATCH --ntasks-per-node=8 --cpus-per-task=8 --gpus-per-node=8 -#SBATCH --threads-per-core=1 # --hint=nomultithread +#SBATCH --nodes=2 #SBATCH --exclusive #SBATCH --output=%x-%j.out #SBATCH --time=00:10:00 -#SBATCH --nodes=2 module purge -# Architecture +# A CrayPE environment version +module load cpe/23.12 +# An architecture module load craype-accel-amd-gfx90a craype-x86-trento # A compiler to target the architecture module load PrgEnv-cray # Some architecture related libraries and tools -module load amd-mixed +module load CCE-GPU-3.0.0 +module load amd-mixed/5.2.3 + +date +module list export MPICH_GPU_SUPPORT_ENABLED=1 @@ -36,4 +40,5 @@ export OMP_NUM_THREADS=1 export WARPX_NMPI_PER_NODE=8 export TOTAL_NMPI=$(( ${SLURM_JOB_NUM_NODES} * ${WARPX_NMPI_PER_NODE} )) srun -N${SLURM_JOB_NUM_NODES} -n${TOTAL_NMPI} --ntasks-per-node=${WARPX_NMPI_PER_NODE} \ + --cpus-per-task=8 --threads-per-core=1 --gpu-bind=closest \ ./warpx inputs > output.txt From d8df8f60f0dc584202165d62ac37d0947cb8eb3d Mon Sep 17 00:00:00 2001 From: David Grote Date: Thu, 1 Feb 2024 11:37:59 -0800 Subject: [PATCH 128/176] Fix doc for doChargeConservingDepositionShapeNImplicit (#4658) --- Source/Particles/Deposition/CurrentDeposition.H | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index efe6efcc788..5d1055278b2 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -1142,9 +1142,11 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, * particles positions are determined and in how the particle gamma is calculated. * * \tparam depos_order deposition order + * \param xp_n,yp_n,zp_n Pointer to arrays of particle position at time level n. * \param GetPosition A functor for returning the particle position. * \param wp Pointer to array of particle weights. - * \param uxp,uyp,uzp Pointer to arrays of particle momentum. + * \param uxp_n,uyp_n,uzp_n Pointer to arrays of particle momentum at time level n. + * \param uxp_nph,uyp_nph,uzp_nph Pointer to arrays of particle momentum at time level n + 1/2. * \param ion_lev Pointer to array of particle ionization level. This is required to have the charge of each macroparticle since q is a scalar. For non-ionizable species, From 282ae836de82ce38e31a79eda06564aaa92beb81 Mon Sep 17 00:00:00 2001 From: Harmen Stoppels Date: Thu, 1 Feb 2024 21:44:20 +0100 Subject: [PATCH 129/176] Add WarpX_CCACHE Option (#4637) WarpX autodetects `ccache` and uses it, and there's nothing you can do about it. This PR disables it by default, and lets developers enable it through `-DWarpX_CCACHE:BOOL=ON`. The reason for this is mostly related to compiler wrappers, of which there are many... If `g++` is a compiler wrapper, then `ccache g++` cannot see the effective flags passed to underlying real `g++`. That leads eventually to false positive cache hits, which is a pain to debug. For compiler wrappers you want `g++` the wrapper to invoke the call to `ccache ` to fix that, which is for example how Spack handle it. Further, if you use spack with ccache enabled, the defaults of WarpX cause `ccache` to be invoked twice (inner & outer), doubling the cache requirements. Finally, compiler wrapper that handle ccache themselves may set further ccache options / flags that WarpX does not set, such as disabling hashing of the build dir -- w/o that option cache may be useless. * Update Order and Docs --- CMakeLists.txt | 11 ++++++++++- Docs/source/install/cmake.rst | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a0b28c9f86..76a5ecdd3f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,15 @@ set_cxx17_superbuild() # this is an optional tool that stores compiled object files; allows fast # re-builds even with "make clean" in between. Mainly used to store AMReX # objects -set_ccache() +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + set(WarpX_CCACHE_DEFAULT ON) +else() + set(WarpX_CCACHE_DEFAULT OFF) # we are a subproject in a superbuild +endif() +option(WarpX_CCACHE "Enable ccache for faster rebuilds" ${WarpX_CCACHE_DEFAULT}) +if(WarpX_CCACHE) + set_ccache() +endif() # Output Directories ########################################################## @@ -149,6 +157,7 @@ endif() # this defined the variable BUILD_TESTING which is ON by default #include(CTest) + # Dependencies ################################################################ # diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index d1dc25cf095..0882efd7fe2 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -116,7 +116,7 @@ By default, the most important dependencies of WarpX are automatically downloade CMake Option Default & Values Description ============================= ============================================== =========================================================== ``BUILD_SHARED_LIBS`` ON/**OFF** `Build shared libraries for dependencies `__ -``CCACHE_PROGRAM`` First found ``ccache`` executable. Set to ``-DCCACHE_PROGRAM=NO`` to disable CCache. +``WarpX_CCACHE`` **ON**/OFF Search and use CCache to speed up rebuilds. ``AMReX_CUDA_PTX_VERBOSE`` ON/**OFF** Print CUDA code generation statistics from ``ptxas``. ``WarpX_amrex_src`` *None* Path to AMReX source directory (preferred if set) ``WarpX_amrex_repo`` ``https://github.com/AMReX-Codes/amrex.git`` Repository URI to pull and build AMReX from From 7a7c704ec0d34c46634452a49c07dcd34c1e3c13 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Fri, 2 Feb 2024 00:17:36 +0100 Subject: [PATCH 130/176] Update profile and job script for LUMI supercomputer (#4634) * update LUMI profile and LUMI job script * add advice to run on dev-g * update job script and profile --- .../lumi-csc/lumi_warpx.profile.example | 20 +++++++++------- Tools/machines/lumi-csc/submit.sh | 24 +++++++++++++++---- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/Tools/machines/lumi-csc/lumi_warpx.profile.example b/Tools/machines/lumi-csc/lumi_warpx.profile.example index 74b8aa8df17..2cb44035ce4 100644 --- a/Tools/machines/lumi-csc/lumi_warpx.profile.example +++ b/Tools/machines/lumi-csc/lumi_warpx.profile.example @@ -2,9 +2,9 @@ #export proj= # required dependencies -module load LUMI/23.03 partition/G +module load LUMI/23.09 partition/G module load rocm/5.2.3 # waiting for 5.5 for next bump -module load buildtools/23.03 +module load buildtools/23.09 # optional: just an additional text editor module load nano @@ -16,16 +16,16 @@ export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support -module load Boost/1.81.0-cpeCray-23.03 +module load Boost/1.82.0-cpeCray-23.09 # optional: for openPMD support -module load cray-hdf5/1.12.2.3 +module load cray-hdf5/1.12.2.7 export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3/bin:${PATH} # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load cray-python/3.10.10 # an alias to request an interactive batch node for one hour # for paralle execution, start on the batch node: srun @@ -41,9 +41,13 @@ export MPICH_GPU_SUPPORT_ENABLED=1 export AMREX_AMD_ARCH=gfx90a # compiler environment hints -export CC=$(which cc) -export CXX=$(which CC) -export FC=$(which ftn) +# Warning: using the compiler wrappers cc and CC +# instead of amdclang and amdclang++ +# currently results in a significant +# loss of performances +export CC=$(which amdclang) +export CXX=$(which amdclang++) +export FC=$(which amdflang) export CFLAGS="-I${ROCM_PATH}/include" export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed" export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64" diff --git a/Tools/machines/lumi-csc/submit.sh b/Tools/machines/lumi-csc/submit.sh index f6be702300a..d784471acd5 100644 --- a/Tools/machines/lumi-csc/submit.sh +++ b/Tools/machines/lumi-csc/submit.sh @@ -10,7 +10,7 @@ #SBATCH --gpus-per-node=8 #SBATCH --time=00:10:00 -export MPICH_GPU_SUPPORT_ENABLED=1 +date # note (12-12-22) # this environment setting is currently needed on LUMI to work-around a @@ -30,7 +30,11 @@ export FI_MR_CACHE_MONITOR=memhooks # alternative cache monitor # the home directory, which does not scale. export ROCFFT_RTC_CACHE_PATH=/dev/null -export OMP_NUM_THREADS=1 +# Seen since August 2023 +# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors +# https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#olcfdev-1597-ofi-poll-failed-undeliverable-errors +export MPICH_SMP_SINGLE_COPY_MODE=NONE +export FI_CXI_RX_MATCH_MODE=software # LUMI documentation suggests using the following wrapper script # to set the ROCR_VISIBLE_DEVICES to the value of SLURM_LOCALID @@ -47,9 +51,21 @@ chmod +x ./select_gpu sleep 1 # LUMI documentation suggests using the following CPU bind -# so that the node local rank and GPU ID match +# in order to have 6 threads per GPU (blosc compression in adios2 uses threads) # see https://docs.lumi-supercomputer.eu/runjobs/scheduled-jobs/lumig-job/ -CPU_BIND="map_cpu:48,56,16,24,1,8,32,40" +# +# WARNING: the following CPU_BIND options don't work on the dev-g partition. +# If you want to run your simulation on dev-g, please comment them +# out and replace them with CPU_BIND="map_cpu:49,57,17,25,1,9,33,41" +# +CPU_BIND="mask_cpu:7e000000000000,7e00000000000000" +CPU_BIND="${CPU_BIND},7e0000,7e000000" +CPU_BIND="${CPU_BIND},7e,7e00" +CPU_BIND="${CPU_BIND},7e00000000,7e0000000000" + +export OMP_NUM_THREADS=6 + +export MPICH_GPU_SUPPORT_ENABLED=1 srun --cpu-bind=${CPU_BIND} ./select_gpu ./warpx inputs | tee outputs.txt rm -rf ./select_gpu From 1093f618d9efb5d0971fa609ed24758364a17caa Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 1 Feb 2024 16:58:27 -0800 Subject: [PATCH 131/176] Fix BTD/Scrape Flush Count with Filters (#4657) * Fix BTD/Scrape Flush Count with Filters Move the counting of already flushed particles for writers that call the I/O backends multiple time per data set, e.g., BTD and boundary scraping, into the I/O backend. Currently, filtering is done as the first step in I/O backends and thus the previous count outside of the I/O backends was over-counting particles that might still get filtered out. Offset should be a `long`: Overflow risk is very high for pure `int`. Also, counter is `unsigned`, so `unsigned long` for now. * Simplify: Remove `m_totalParticles_flushed_already` Less state we can forget in checkpoint-restart and that we have to transfer across API boundaries. --- Source/Diagnostics/BTDiagnostics.H | 2 - Source/Diagnostics/BTDiagnostics.cpp | 31 ++++-------- .../BoundaryScrapingDiagnostics.cpp | 17 ++----- Source/Diagnostics/Diagnostics.H | 5 -- Source/Diagnostics/FlushFormats/FlushFormat.H | 3 +- .../FlushFormats/FlushFormatAscent.H | 3 +- .../FlushFormats/FlushFormatAscent.cpp | 2 +- .../FlushFormats/FlushFormatCheckpoint.H | 3 +- .../FlushFormats/FlushFormatCheckpoint.cpp | 2 +- .../FlushFormats/FlushFormatOpenPMD.H | 3 +- .../FlushFormats/FlushFormatOpenPMD.cpp | 4 +- .../FlushFormats/FlushFormatPlotfile.H | 3 +- .../FlushFormats/FlushFormatPlotfile.cpp | 6 +-- .../FlushFormats/FlushFormatSensei.H | 3 +- .../FlushFormats/FlushFormatSensei.cpp | 7 ++- Source/Diagnostics/FullDiagnostics.cpp | 5 +- Source/Diagnostics/OpenPMDHelpFunction.H | 18 +++++++ Source/Diagnostics/OpenPMDHelpFunction.cpp | 20 ++++++++ Source/Diagnostics/WarpXOpenPMD.H | 8 ++- Source/Diagnostics/WarpXOpenPMD.cpp | 50 ++++++++----------- 20 files changed, 94 insertions(+), 101 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index ab894da69c2..f6c44c777ea 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -399,8 +399,6 @@ private: lab-frame data. */ void InitializeParticleFunctors () override; - /** Update total number of particles flushed for all species for ith snapshot */ - void UpdateTotalParticlesFlushed(int i_buffer); /** Reset total number of particles in the particle buffer to 0 for ith snapshot */ void ResetTotalParticlesInBuffer(int i_buffer); /** Clear particle data stored in the particle buffer */ diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 0e517e8190c..f7965cd2688 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -129,7 +129,6 @@ void BTDiagnostics::DerivedInitData () } } m_particles_buffer.resize(m_num_buffers); - m_totalParticles_flushed_already.resize(m_num_buffers); m_totalParticles_in_buffer.resize(m_num_buffers); // check that simulation can fill all BTD snapshots @@ -1065,12 +1064,12 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) } } m_flush_format->WriteToFile( - m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), - labtime, m_output_species[i_buffer], nlev_output, file_name, m_file_min_digits, + m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + labtime, + m_output_species.at(i_buffer), nlev_output, file_name, m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards, - use_pinned_pc, isBTD, i_buffer, m_buffer_flush_counter[i_buffer], - m_max_buffer_multifabs[i_buffer], m_geom_snapshot[i_buffer][0], isLastBTDFlush, - m_totalParticles_flushed_already[i_buffer]); + use_pinned_pc, isBTD, i_buffer, m_buffer_flush_counter.at(i_buffer), + m_max_buffer_multifabs.at(i_buffer), m_geom_snapshot.at(i_buffer).at(0), isLastBTDFlush); // Rescaling the box for plotfile after WriteToFile. This is because, for plotfiles, when writing particles, amrex checks if the particles are within the bounds defined by the box. However, in BTD, particles can be (at max) 1 cell outside the bounds of the geometry. So we keep a one-cell bigger box for plotfile when writing out the particle data and rescale after. if (m_format == "plotfile") { @@ -1104,7 +1103,6 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) NullifyFirstFlush(i_buffer); // if particles are selected for output then update and reset counters if (!m_output_species_names.empty()) { - UpdateTotalParticlesFlushed(i_buffer); ResetTotalParticlesInBuffer(i_buffer); ClearParticleBuffer(i_buffer); } @@ -1271,10 +1269,10 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) InterleaveSpeciesHeader(recent_species_Header,snapshot_species_Header, m_output_species_names[i], m_buffer_flush_counter[i_snapshot]); if (BufferSpeciesHeader.m_total_particles == 0) { continue; } - if (m_totalParticles_flushed_already[i_snapshot][i]==0) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, - std::string("Renaming ").append(recent_ParticleHdrFilename).append(" to ").append(snapshot_ParticleHdrFilename).append(" has failed")); + if (!amrex::FileExists(snapshot_ParticleHdrFilename)) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, + std::string("Renaming ").append(recent_ParticleHdrFilename).append(" to ").append(snapshot_ParticleHdrFilename).append(" has failed")); } else { InterleaveParticleDataHeader(recent_ParticleHdrFilename, snapshot_ParticleHdrFilename); @@ -1435,10 +1433,8 @@ BTDiagnostics::InitializeParticleBuffer () const MultiParticleContainer& mpc = warpx.GetPartContainer(); for (int i = 0; i < m_num_buffers; ++i) { m_particles_buffer[i].resize(m_output_species_names.size()); - m_totalParticles_flushed_already[i].resize(m_output_species_names.size()); m_totalParticles_in_buffer[i].resize(m_output_species_names.size()); for (int isp = 0; isp < m_particles_buffer[i].size(); ++isp) { - m_totalParticles_flushed_already[i][isp] = 0; m_totalParticles_in_buffer[i][isp] = 0; m_particles_buffer[i][isp] = std::make_unique(WarpX::GetInstance().GetParGDB()); const int idx = mpc.getSpeciesID(m_output_species_names[isp]); @@ -1489,15 +1485,6 @@ BTDiagnostics::PrepareParticleDataForOutput() } } -void -BTDiagnostics::UpdateTotalParticlesFlushed(int i_buffer) -{ - for (int isp = 0; isp < m_totalParticles_flushed_already[i_buffer].size(); ++isp) { - m_totalParticles_flushed_already[i_buffer][isp] += static_cast( - m_particles_buffer[i_buffer][isp]->TotalNumberOfParticles()); - } -} - void BTDiagnostics::ResetTotalParticlesInBuffer(int i_buffer) { diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index c85dbd6b226..11ffce02f09 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -102,15 +102,6 @@ BoundaryScrapingDiagnostics::InitializeParticleBuffer () m_output_species[i_buffer].push_back(ParticleDiag(m_diag_name, species_name, pc, bnd_buffer)); } } - // Initialize total number of particles flushed - m_totalParticles_flushed_already.resize(m_num_buffers); - for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { - int const n_species = static_cast(m_output_species_names.size()); - m_totalParticles_flushed_already[i_buffer].resize(n_species); - for (int i_species=0; i_speciesWriteToFile( - m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), - warpx.gett_new(0), m_output_species[i_buffer], nlev_output, file_prefix, + m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + warpx.gett_new(0), + m_output_species.at(i_buffer), + nlev_output, file_prefix, m_file_min_digits, false, false, use_pinned_pc, isBTD, warpx.getistep(0), bufferID, numBTDBuffers, geom, - isLastBTD, m_totalParticles_flushed_already[i_buffer]); + isLastBTD); // Now that the data has been written out, clear out the buffer particle_buffer.clearParticles(i_buffer); diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 53ce319d747..c0d2a9f0d53 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -309,11 +309,6 @@ protected: /** Vector of pointers to functors to compute particle output per species*/ amrex::Vector< std::unique_ptr > m_all_particle_functors; - /** Vector of total number of particles previously flushed, per species, per snapshot. - * The first vector is for total number of snapshots and second vector loops - * over the total number of species selected for diagnostics. - */ - amrex::Vector< amrex::Vector > m_totalParticles_flushed_already; /** Vector of total number of particles in the buffer, per species, per snapshot. * The first vector is for total number of snapshots and second vector loops * over the total number of species selected for diagnostics. diff --git a/Source/Diagnostics/FlushFormats/FlushFormat.H b/Source/Diagnostics/FlushFormats/FlushFormat.H index 403e9df7857..65741e4ff20 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormat.H +++ b/Source/Diagnostics/FlushFormats/FlushFormat.H @@ -24,8 +24,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const = 0; + bool isLastBTDFlush = false) const = 0; FlushFormat () = default; virtual ~FlushFormat() = default; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H index 228e4bc5cf6..9d8d3fcd7d2 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H @@ -41,8 +41,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false ) const override; #ifdef AMREX_USE_ASCENT /** \brief Do in-situ visualization for particle data. diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp index 980047e3b46..abfba37cd15 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp @@ -21,7 +21,7 @@ FlushFormatAscent::WriteToFile ( const bool /*use_pinned_pc*/, bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, const amrex::Geometry& /*full_BTD_snapshot*/, - bool /*isLastBTDFlush*/, const amrex::Vector& /* totalParticlesFlushedAlready*/) const + bool /*isLastBTDFlush*/) const { #ifdef AMREX_USE_ASCENT WARPX_PROFILE("FlushFormatAscent::WriteToFile()"); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H index f6aad226d75..5c26ac97f61 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H @@ -28,8 +28,7 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const final; + bool isLastBTDFlush = false) const final; void CheckpointParticles (const std::string& dir, const amrex::Vector& particle_diags) const; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 5f59cd723da..d77437fb931 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -39,7 +39,7 @@ FlushFormatCheckpoint::WriteToFile ( bool /*isBTD*/, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, const amrex::Geometry& /*full_BTD_snapshot*/, - bool /*isLastBTDFlush*/, const amrex::Vector& /* totalParticlesFlushedAlready*/) const + bool /*isLastBTDFlush*/) const { WARPX_PROFILE("FlushFormatCheckpoint::WriteToFile()"); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H index 88380407f5e..141760ac2a3 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H @@ -40,8 +40,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false ) const override; ~FlushFormatOpenPMD () override = default; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp index 3b7006243e7..e0c8c4ef2d6 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp @@ -126,7 +126,7 @@ FlushFormatOpenPMD::WriteToFile ( const bool use_pinned_pc, bool isBTD, int snapshotID, int bufferID, int numBuffers, const amrex::Geometry& full_BTD_snapshot, - bool isLastBTDFlush, const amrex::Vector& totalParticlesFlushedAlready) const + bool isLastBTDFlush) const { WARPX_PROFILE("FlushFormatOpenPMD::WriteToFile()"); const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits); @@ -164,7 +164,7 @@ FlushFormatOpenPMD::WriteToFile ( // particles: all (reside only on locally finest level) m_OpenPMDPlotWriter->WriteOpenPMDParticles( - particle_diags, static_cast(time), use_pinned_pc, isBTD, isLastBTDFlush, totalParticlesFlushedAlready); + particle_diags, static_cast(time), use_pinned_pc, isBTD, isLastBTDFlush); // signal that no further updates will be written to this iteration m_OpenPMDPlotWriter->CloseStep(isBTD, isLastBTDFlush); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H index 486dcc3b5ee..c62056b8907 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H @@ -35,8 +35,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false) const override; /** Write general info of the run into the plotfile */ void WriteJobInfo(const std::string& dir) const; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index df73ed34c94..970d9a504d2 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -65,7 +65,7 @@ FlushFormatPlotfile::WriteToFile ( const bool /*use_pinned_pc*/, bool isBTD, int snapshotID, int bufferID, int numBuffers, const amrex::Geometry& /*full_BTD_snapshot*/, - bool isLastBTDFlush, const amrex::Vector& /* totalParticlesFlushedAlready*/) const + bool isLastBTDFlush) const { WARPX_PROFILE("FlushFormatPlotfile::WriteToFile()"); auto & warpx = WarpX::GetInstance(); @@ -340,9 +340,9 @@ FlushFormatPlotfile::WriteWarpXHeader( void FlushFormatPlotfile::WriteParticles(const std::string& dir, const amrex::Vector& particle_diags, - const amrex::Real time, bool isBTD) const + const amrex::Real time, + bool isBTD) const { - for (const auto& part_diag : particle_diags) { WarpXParticleContainer* pc = part_diag.getParticleContainer(); PinnedMemoryParticleContainer* pinned_pc = part_diag.getPinnedParticleContainer(); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H index 54eb7099ba4..d2ec9a5a4e0 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H @@ -61,8 +61,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false) const override; /** \brief Do in-situ visualization for particle data. * \param[in] particle_diags Each element of this vector handles output of 1 species. diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp index e162b8b3121..348e1da4a00 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp @@ -53,14 +53,13 @@ FlushFormatSensei::WriteToFile ( bool plot_raw_fields, bool plot_raw_fields_guards, const bool use_pinned_pc, bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, - const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/, - const amrex::Vector& totalParticlesFlushedAlready) const + const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/) const { amrex::ignore_unused( geom, nlev, prefix, file_min_digits, plot_raw_fields, plot_raw_fields_guards, - use_pinned_pc, - totalParticlesFlushedAlready); + use_pinned_pc + ); #ifndef AMREX_USE_SENSEI_INSITU amrex::ignore_unused(varnames, mf, iteration, time, particle_diags, diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 4f1e47a2a52..fd329a38220 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -133,8 +133,9 @@ FullDiagnostics::Flush ( int i_buffer, bool /* force_flush */ ) auto & warpx = WarpX::GetInstance(); m_flush_format->WriteToFile( - m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), - warpx.gett_new(0), m_output_species[i_buffer], nlev_output, m_file_prefix, + m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + warpx.gett_new(0), + m_output_species.at(i_buffer), nlev_output, m_file_prefix, m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards); FlushRaw(); diff --git a/Source/Diagnostics/OpenPMDHelpFunction.H b/Source/Diagnostics/OpenPMDHelpFunction.H index 9db4b9fb194..d2f2c4f9f9d 100644 --- a/Source/Diagnostics/OpenPMDHelpFunction.H +++ b/Source/Diagnostics/OpenPMDHelpFunction.H @@ -14,7 +14,25 @@ #include +/** Determine the preferred file ending if unspecified + * + * @return file ending without the "." + */ std::string WarpXOpenPMDFileType (); +#ifdef WARPX_USE_OPENPMD +/** Determine how many particles were already written in this species and step + * + * This checks for a particle species the current size of the id attribute, if it exists, + * and if it does it takes its extent as the number of particles already on disk. + * + * Note that this checks declared size, not necessarily written size. + * + * @return exisitng extent of the "id" attribute or zero. + */ +unsigned long +num_already_flushed (openPMD::ParticleSpecies & currSpecies); +#endif + #endif // WARPX_OPENPMDHELPFUNCTION_H_ diff --git a/Source/Diagnostics/OpenPMDHelpFunction.cpp b/Source/Diagnostics/OpenPMDHelpFunction.cpp index a898c97b6b4..6170249b52b 100644 --- a/Source/Diagnostics/OpenPMDHelpFunction.cpp +++ b/Source/Diagnostics/OpenPMDHelpFunction.cpp @@ -27,3 +27,23 @@ WarpXOpenPMDFileType () #endif // WARPX_USE_OPENPMD return openPMDFileType; } + +#ifdef WARPX_USE_OPENPMD +unsigned long +num_already_flushed (openPMD::ParticleSpecies & currSpecies) +{ + const auto *const scalar = openPMD::RecordComponent::SCALAR; + + unsigned long ParticleFlushOffset = 0; + + if (currSpecies.contains("id")) { + if (currSpecies["id"].contains(scalar)) { + if (!currSpecies["id"][scalar].empty()) { + ParticleFlushOffset = currSpecies["id"][scalar].getExtent().at(0); + } + } + } + + return ParticleFlushOffset; +} +#endif diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index e3b7b893d0a..4597dacd9ae 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -125,8 +125,7 @@ public: amrex::Real time, bool use_pinned_pc = false, bool isBTD = false, - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector()); + bool isLastBTDFlush = false); /** Write out all openPMD fields for all active MR levels * @@ -290,9 +289,9 @@ private: * @param[in] int_comp_names The int attribute names, from WarpX * @param[in] charge Charge of the particles (note: fix for ions) * @param[in] mass Mass of the particles + * @param[inout] ParticleFlushOffset previously flushed number of particles in BTD * @param[in] isBTD is this a backtransformed diagnostics (BTD) write? * @param[in] isLastBTDFlush is this the last time we will flush this BTD station? - * @param[in] ParticleFlushOffset previously flushed number of particles in BTD */ void DumpToFile (ParticleContainer* pc, const std::string& name, @@ -304,8 +303,7 @@ private: amrex::ParticleReal charge, amrex::ParticleReal mass, bool isBTD = false, - bool isLastBTDFlush = false, - int ParticleFlushOffset = 0); + bool isLastBTDFlush = false); /** Get the openPMD-api filename for openPMD::Series * diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 71d96a47927..64411ecf6e4 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -519,9 +519,11 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) void WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector& particle_diags, - const amrex::Real time, const bool use_pinned_pc, - const bool isBTD, const bool isLastBTDFlush, - const amrex::Vector& totalParticlesFlushedAlready) + const amrex::Real time, + const bool use_pinned_pc, + const bool isBTD, + const bool isLastBTDFlush +) { WARPX_PROFILE("WarpXOpenPMDPlot::WriteOpenPMDParticles()"); @@ -618,31 +620,15 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { // real_names contains a list of all real particle attributes. // real_flags is 1 or 0, whether quantity is dumped or not. - { - if (isBTD) { - DumpToFile(&tmp, - particle_diags[i].getSpeciesName(), - m_CurrentStep, - real_flags, - int_flags, - real_names, int_names, - pc->getCharge(), pc->getMass(), - isBTD, isLastBTDFlush, - totalParticlesFlushedAlready[i] - ); - } else { - DumpToFile(&tmp, - particle_diags[i].getSpeciesName(), - m_CurrentStep, - real_flags, - int_flags, - real_names, int_names, - pc->getCharge(), pc->getMass(), - isBTD, isLastBTDFlush, - 0 - ); - } - } + DumpToFile(&tmp, + particle_diags.at(i).getSpeciesName(), + m_CurrentStep, + real_flags, + int_flags, + real_names, int_names, + pc->getCharge(), pc->getMass(), + isBTD, isLastBTDFlush + ); } } @@ -657,8 +643,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, amrex::ParticleReal const charge, amrex::ParticleReal const mass, const bool isBTD, - const bool isLastBTDFlush, - int ParticleFlushOffset) { + const bool isLastBTDFlush +) +{ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_Series != nullptr, "openPMD: series must be initialized"); AMREX_ALWAYS_ASSERT(write_real_comp.size() == pc->NumRealComps()); @@ -672,6 +659,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, openPMD::Iteration currIteration = GetIteration(iteration, isBTD); openPMD::ParticleSpecies currSpecies = currIteration.particles[name]; + // only BTD writes multiple times into the same step, zero for other methods + unsigned long ParticleFlushOffset = isBTD ? num_already_flushed(currSpecies) : 0; + // prepare data structures the first time BTD has non-zero particles // we set some of them to zero extent, so we need to time that well bool const is_first_flush_with_particles = num_dump_particles > 0 && ParticleFlushOffset == 0; From 206b0815a060aa9ccbf0f71a46f1142137f5b8a8 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 1 Feb 2024 20:06:59 -0800 Subject: [PATCH 132/176] Release 24.02 (#4660) * AMReX: 24.02 * pyAMReX: 24.02 * WarpX: 24.02 --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 9960cdfbb29..79916c455d1 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 689144d157a0106faf3d0ae89f8d90b0250cf975 && cd - + cd ../amrex && git checkout --detach 24.02 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 76a5ecdd3f3..3a947b01dcd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.01) +project(WarpX VERSION 24.02) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 48a02c5d216..b34c437b829 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.01' +version = u'24.02' # The full version, including alpha/beta/rc tags. -release = u'24.01' +release = u'24.02' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index f82e6563a31..3c6c0f605c0 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.01', + version = '24.02', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 70c62b190fb..0659d530cf5 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 689144d157a0106faf3d0ae89f8d90b0250cf975 +branch = 24.02 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index ab11d70dfcc..ae22eba499d 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 689144d157a0106faf3d0ae89f8d90b0250cf975 +branch = 24.02 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 9b74c8db7fd..0f6a15a5ff4 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.01 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.02 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "689144d157a0106faf3d0ae89f8d90b0250cf975" +set(WarpX_amrex_branch "24.02" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 3c043f60226..b4cf9f3f9c1 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.01 CONFIG REQUIRED) + find_package(pyAMReX 24.02 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "cdf03496f6809527b97950e077508ca4b201fa9b" +set(WarpX_pyamrex_branch "24.02" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 48857d264cb..e1b45ab7c28 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 689144d157a0106faf3d0ae89f8d90b0250cf975 && cd - +cd amrex && git checkout --detach 24.02 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index d3efeaaacd5..197a39ce23f 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.01', + version = '24.02', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From 9d8ecf93df7c8713df08eac96c1f88f31b7fcd0d Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 1 Feb 2024 20:08:18 -0800 Subject: [PATCH 133/176] Add install instructions for ALCF's Polaris (#4636) * add polaris machine files * add doc page for Polaris --- Docs/source/install/hpc.rst | 1 + Docs/source/install/hpc/polaris.rst | 187 ++++++++++++++++++ .../polaris-alcf/install_gpu_dependencies.sh | 123 ++++++++++++ Tools/machines/polaris-alcf/polaris_gpu.pbs | 36 ++++ .../polaris_gpu_warpx.profile.example | 51 +++++ 5 files changed, 398 insertions(+) create mode 100644 Docs/source/install/hpc/polaris.rst create mode 100755 Tools/machines/polaris-alcf/install_gpu_dependencies.sh create mode 100644 Tools/machines/polaris-alcf/polaris_gpu.pbs create mode 100644 Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example diff --git a/Docs/source/install/hpc.rst b/Docs/source/install/hpc.rst index 9617f2a7fd6..a7b0f636b56 100644 --- a/Docs/source/install/hpc.rst +++ b/Docs/source/install/hpc.rst @@ -46,6 +46,7 @@ This section documents quick-start guides for a selection of supercomputers that hpc/lxplus hpc/ookami hpc/perlmutter + hpc/polaris hpc/quartz hpc/spock hpc/summit diff --git a/Docs/source/install/hpc/polaris.rst b/Docs/source/install/hpc/polaris.rst new file mode 100644 index 00000000000..d20ecccee32 --- /dev/null +++ b/Docs/source/install/hpc/polaris.rst @@ -0,0 +1,187 @@ +.. _building-polaris: + +Polaris (ALCF) +============== + +The `Polaris cluster `__ is located at ALCF. + + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `ALCF user guide `__ +* Batch system: `PBS `__ +* `Filesystems `__ + +.. _building-polaris-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code: + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + +On Polaris, you can run either on GPU nodes with fast A100 GPUs (recommended) or CPU nodes. + +.. tab-set:: + + .. tab-item:: A100 GPUs + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/polaris_gpu_warpx.profile``. + Create it now: + + .. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example $HOME/polaris_gpu_warpx.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``proj_name``, then run ``nano $HOME/polaris_gpu_warpx.profile`` and edit line 2 to read: + + .. code-block:: bash + + export proj="proj_name" + + Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + + .. important:: + + Now, and as the first step on future logins to Polaris, activate these environment settings: + + .. code-block:: bash + + source $HOME/polaris_gpu_warpx.profile + + Finally, since Polaris does not yet provide software modules for some of our dependencies, install them once: + + .. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/polaris-alcf/install_gpu_dependencies.sh + source ${CFS}/${proj%_g}/${USER}/sw/polaris/gpu/venvs/warpx/bin/activate + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/polaris-alcf/install_gpu_dependencies.sh + :language: bash + + + .. tab-item:: CPU Nodes + + *Under construction* + + +.. _building-polaris-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. tab-set:: + + .. tab-item:: A100 GPUs + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_pm_gpu + + cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_pm_gpu -j 16 + + The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``. + Additionally, the following commands will install WarpX as a Python module: + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_pm_gpu_py + + cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_pm_gpu_py -j 16 --target pip_install + + .. tab-item:: CPU Nodes + + *Under construction* + +Now, you can :ref:`submit Polaris compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Or, you can use the WarpX executables to submit Polaris jobs (:ref:`example inputs `). +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$PSCRATCH``. + + +.. _building-polaris-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd $HOME/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the polaris_gpu_warpx.profile or polaris_cpu_warpx files `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_pm_*`` and rebuild WarpX. + + +.. _running-cpp-polaris: + +Running +------- + +.. tab-set:: + + .. tab-item:: A100 (40GB) GPUs + + The batch script below can be used to run a WarpX simulation on multiple nodes (change ```` accordingly) on the supercomputer Polaris at ALCF. + + Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. + Note that we run one MPI rank per GPU. + + .. literalinclude:: ../../../../Tools/machines/polaris-alcf/polaris_gpu.pbs + :language: bash + :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/polaris-alcf/polaris_gpu.pbs``. + + To run a simulation, copy the lines above to a file ``polaris_gpu.pbs`` and run + + .. code-block:: bash + + qsub polaris_gpu.pbs + + to submit the job. + + + .. tab-item:: CPU Nodes + + *Under construction* diff --git a/Tools/machines/polaris-alcf/install_gpu_dependencies.sh b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh new file mode 100755 index 00000000000..e2cdca86fbc --- /dev/null +++ b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl (edited by Roelof Groenewald for Polaris) +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + +# Check: ###################################################################### +# +# Was polaris_gpu_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your polaris_gpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi + +# Remove old dependencies ##################################################### +# +SW_DIR="/home/${USER}/sw/polaris/gpu" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + +# General extra dependencies ################################################## +# + +# c-blosc (I/O compression) +if [ -d $HOME/src/c-blosc ] +then + cd $HOME/src/c-blosc + git fetch --prune + git checkout v1.21.1 + cd - +else + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc +fi +rm -rf $HOME/src/c-blosc-pm-gpu-build +cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build $HOME/src/c-blosc-pm-gpu-build --target install --parallel 16 +rm -rf $HOME/src/c-blosc-pm-gpu-build + +# ADIOS2 +if [ -d $HOME/src/adios2 ] +then + cd $HOME/src/adios2 + git fetch --prune + git checkout v2.8.3 + cd - +else + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 +fi +rm -rf $HOME/src/adios2-pm-gpu-build +cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build $HOME/src/adios2-pm-gpu-build --target install -j 16 +rm -rf $HOME/src/adios2-pm-gpu-build + +# BLAS++ (for PSATD+RZ) +if [ -d $HOME/src/blaspp ] +then + cd $HOME/src/blaspp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp +fi +rm -rf $HOME/src/blaspp-pm-gpu-build +CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 16 +rm -rf $HOME/src/blaspp-pm-gpu-build + +# LAPACK++ (for PSATD+RZ) +if [ -d $HOME/src/lapackpp ] +then + cd $HOME/src/lapackpp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp +fi +rm -rf $HOME/src/lapackpp-pm-gpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 16 +rm -rf $HOME/src/lapackpp-pm-gpu-build + +# Python ###################################################################### +# +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade virtualenv +python3 -m pip cache purge +rm -rf ${SW_DIR}/venvs/warpx +python3 -m venv --system-site-packages ${SW_DIR}/venvs/warpx +source ${SW_DIR}/venvs/warpx/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +# MPICC="cc -target-accel=nvidia80 -shared" python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install cupy-cuda11x # CUDA 11.7 compatible wheel +# optional: for libEnsemble +python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt +# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +python3 -m pip install --upgrade torch # CUDA 11.7 compatible wheel +python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/polaris-alcf/polaris_gpu.pbs b/Tools/machines/polaris-alcf/polaris_gpu.pbs new file mode 100644 index 00000000000..178db6ad6a2 --- /dev/null +++ b/Tools/machines/polaris-alcf/polaris_gpu.pbs @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#PBS -A +#PBS -l select=:system=polaris +#PBS -l place=scatter +#PBS -l walltime=0:10:00 +#PBS -l filesystems=home:eagle +#PBS -q debug +#PBS -N test_warpx + +# Set required environment variables +# support gpu-aware-mpi +# export MPICH_GPU_SUPPORT_ENABLED=1 + +# Change to working directory +echo Working directory is $PBS_O_WORKDIR +cd ${PBS_O_WORKDIR} + +echo Jobid: $PBS_JOBID +echo Running on host `hostname` +echo Running on nodes `cat $PBS_NODEFILE` + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx +INPUTS=input1d + +# MPI and OpenMP settings +NNODES=`wc -l < $PBS_NODEFILE` +NRANKS_PER_NODE=4 +NDEPTH=1 +NTHREADS=1 + +NTOTRANKS=$(( NNODES * NRANKS_PER_NODE )) +echo "NUM_OF_NODES= ${NNODES} TOTAL_NUM_RANKS= ${NTOTRANKS} RANKS_PER_NODE= ${NRANKS_PER_NODE} THREADS_PER_RANK= ${NTHREADS}" + +mpiexec -np ${NTOTRANKS} ${EXE} ${INPUTS} > output.txt diff --git a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example new file mode 100644 index 00000000000..d7a68bf16bb --- /dev/null +++ b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example @@ -0,0 +1,51 @@ +# Set the project name +export proj="" # change me! + +# swap to the Milan cray package +# module swap craype-x86-rome craype-x86-milan + +# required dependencies +module load cmake/3.23.2 +module load cudatoolkit-standalone + +# optional: for QED support with detailed tables +# module load boost/1.81.0 + +# optional: for openPMD and PSATD+RZ support +module load cray-hdf5-parallel/1.12.2.3 +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-master:$CMAKE_PREFIX_PATH + +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH + +export PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3/bin:${PATH} + +# optional: for Python bindings or libEnsemble +module load cray-python/3.9.13.1 + +if [ -d "/home/${USER}/sw/polaris/gpu/venvs/warpx" ] +then + source /home/${USER}/sw/polaris/gpu/venvs/warpx/bin/activate +fi + +# necessary to use CUDA-Aware MPI and run a job +export CRAY_ACCEL_TARGET=nvidia80 + +# optimize CUDA compilation for A100 +export AMREX_CUDA_ARCH=8.0 + +# optimize CPU microarchitecture for AMD EPYC 3rd Gen (Milan/Zen3) +# note: the cc/CC/ftn wrappers below add those +# export CXXFLAGS="-march=znver3" +# export CFLAGS="-march=znver3" + +# compiler environment hints +export CC=nvc +export CXX=nvc++ +export CUDACXX=nvcc +export CUDAHOSTCXX=nvc++ From 60bf00039522925870b86f3db6c4ac7d12c3a04b Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Thu, 1 Feb 2024 20:16:19 -0800 Subject: [PATCH 134/176] Implemented Villasenor and Buneman deposition routine for implicit solver (#4623) * added infrastracture for doing the VillasenorAndBuneman current deposition scheme. Set algo.current_deposition = 3 to use. Right now, the actual algorithm is identical to doChargeConsevingCurrentDepositionImplicit(). * small modification to message when asserting that the evolve scheme is implicit when using villasenor-and-buneman current deposition. * added new interpolation.angus_scheme as intermediate step for adding the CC1 deposit/gather routine from PICNIC. Seems to pass numerical energy conservation test in 2D right now, but the gather/deposit are actually just CIC. * Created doGatherPicnicShapeN function. Does CIC right now, same as the VandB current deposition. * vandb current deposition works in 2D. * fixed bug in vandb deposition. Exact charge conservation is now obtained. Made same fix to corresonding gather routine. * streamlined vandb deposition and corresponding gather routine. * using dxp/dt in place of vp when doing current deposition for vandb scheme gives better charge conservation. * added a new templated function to ShapeFactors.H that is used to compute the deposition weights tranverse to the current direction for a segment in the villesanor and buneman current deposition. * significant streamlining of new vandb current deposition. * applied same streamlining to vandb deposition from previous commit to the corresponding gather routine. * working on generalzation of vandb deposit and corresponding gather routine to work for 1D, 3D, and for depos_order = 1. WIP * removed interolation.angus_scheme flag that was used to do the Picnic gather when doing Implicit. Now, the gather routine is chosen based on the CurrentDepositionAlgo enum value. * galerkin_interpolation flag is no longer used with the Implicit solver. Removed some assertions related to this flag and the implicit solver. * VillasenorAndBuneman ==> Villasenor * added ability to use shape_factor 3 with villasenor deposit. However, initial tests do not give exact energy conservation. It is good, but not exact. Don't know why, so for now we still assert to shape_factor < 3 when using villasenor deposit. * villasenor deposition now works with shape_factor = 3. * villasenor deposition, and the corresponding gather, now works in 1D. * intermediate checkin. Working on getting villasenor working in 3D. WIP. * added a new function to ShapeFactors.H that is similar to the average function, but instead of returning the average weights using old and new positions it returns both old and new weights along with the shared left index. This is needed for the transverse interpolation for villasenor in 3D. * added coding to compute the cell-crossings in 3D. It runs in 3D, but charge is not conserved. WIP. * fixed bug. villasenor deposition and corresponding gather work in 3D for shape factor = 2. Cleaned up logic for cell crossings setting in 3D. * cleaning things up. * villasenor deposition and corresponding gather work in 3D with shape_factor = 1,2, and 3. * code compiles in RZ. * small tune up. * fixed bug in how Xcell is set in villasenor deposition/gather. Previous implementation only worked when the left domain boundary was zero. * refactoring to avoid roundoff issue in charge conservation for shape factor 1 with villasenor deposition. Roundoff issue solved in 1D. Still need to do 2D and 3D. * fixed roundoff issue with shape = 1 when using villasenor for 2D. * fixed roundoff issue with shape = 3 when using villasenor in 3D. * added proper briefs to the villasenor deposition and the picnic-like gather. * added a few comments. * removed duplicate query for galerkin_scheme in WarpX.cpp. Why didn't merge catch this? * minor cleanup. * added 4th order shape factor routines to ShapeFactors.H as needed for using shape factor = 4 with villasenor deposition. * generalized villasenor deposition routine and the picnic gather to work shape factor orders higher than 3. * added ability to parse in shape factor 4 when using villasenor deposition and added the ability to call shape factor 4 for the current density and charge density deposits. * fixed a few small bugs that prevented compile in 3D. * villasenor deposition now uses villasenor for out-of-plane J as well. Same for out-of-plane E in corresponding Picnic gather. * slight refactoring and cleaning up some comments. * cleaning up some comments. * added regression tests. * removed blank spaces. * fixed bug * cleaning up tabs. * commented out std::cout lines. * initializing potentially uninitialized variables. * fixed indentation issue. * changed some types to avoid narrowing conversion issue. * removed cout comment lines. * ions renamed to protons in new implicit example input deck, consistent with what is assumed in the benchmark data in the benchmarks_json/ folder. Updated the benchmark data using simulation ran on quartz LC. * had to static_cast some quantities in new gather routine. * more static casting. * clang-tidy told me to use auto. * fixed indentation. * changed tolerance for new 2D CI test. max delta energy error on Azure is 2X higher than on LC. * added particle y-position the json file for new 2D implicit ci test. * updated values in json file for new 2D implicit ci test to have the sum of the absolute value of the fields rather than just the sum. * if ==> else if. This should help avoid a potential bug in the future. * adjusted some lines had an incorrect number of white spaces. * significant simplification of code to determine segment values at cell crossings in 2D and 3D. By way of Dave Grote. * slight refactoring for style and performance. * more refactoring. No longer pre-defining slopes. Added logical check for zero dxp values before division. * made list of params in brief for doVillasenorDepositionShapeNImplicit consistent with the input parameters. * all variables used in computing the shape factor for the VB deposition, and corresponding gather, now have type double. --- Examples/Tests/Implicit/analysis_vandb_2d.py | 68 ++ Examples/Tests/Implicit/inputs_vandb_2d | 92 +++ .../ImplicitPicard_VandB_2d.json | 31 + Regression/WarpX-tests.ini | 17 + Source/Diagnostics/WarpXOpenPMD.cpp | 2 + Source/Initialization/WarpXInitData.cpp | 3 + .../Particles/Deposition/CurrentDeposition.H | 664 ++++++++++++++++ Source/Particles/Gather/FieldGather.H | 710 +++++++++++++++++- .../Particles/PhysicalParticleContainer.cpp | 6 +- Source/Particles/ShapeFactors.H | 105 +++ Source/Particles/WarpXParticleContainer.cpp | 74 +- Source/Utils/WarpXAlgorithmSelection.H | 3 +- Source/Utils/WarpXAlgorithmSelection.cpp | 9 +- Source/WarpX.H | 2 +- Source/WarpX.cpp | 57 +- Source/ablastr/particles/DepositCharge.H | 5 + 16 files changed, 1810 insertions(+), 38 deletions(-) create mode 100755 Examples/Tests/Implicit/analysis_vandb_2d.py create mode 100644 Examples/Tests/Implicit/inputs_vandb_2d create mode 100644 Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json diff --git a/Examples/Tests/Implicit/analysis_vandb_2d.py b/Examples/Tests/Implicit/analysis_vandb_2d.py new file mode 100755 index 00000000000..fa3299925a8 --- /dev/null +++ b/Examples/Tests/Implicit/analysis_vandb_2d.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Justin Angus +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from the script `inputs_vandb_2d`. +# This simulates a 2D periodic plasma using the implicit solver +# with the Villasenor deposition using shape factor 2. +import os +import sys + +import numpy as np +from scipy.constants import e, epsilon_0 +import yt + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +field_energy = np.loadtxt('diags/reducedfiles/field_energy.txt', skiprows=1) +particle_energy = np.loadtxt('diags/reducedfiles/particle_energy.txt', skiprows=1) + +total_energy = field_energy[:,2] + particle_energy[:,2] + +delta_E = (total_energy - total_energy[0])/total_energy[0] +max_delta_E = np.abs(delta_E).max() + +# This case should have near machine precision conservation of energy +tolerance_rel_energy = 2.e-14 +tolerance_rel_charge = 2.e-15 + +print(f"max change in energy: {max_delta_E}") +print(f"tolerance: {tolerance_rel_energy}") + +assert( max_delta_E < tolerance_rel_energy ) + +# check for machine precision conservation of charge density +n0 = 1.e30 + +pltdir = sys.argv[1] +ds = yt.load(pltdir) +data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) + +divE = data['boxlib', 'divE'].value +rho = data['boxlib', 'rho'].value + +# compute local error in Gauss's law +drho = (rho - epsilon_0*divE)/e/n0 + +# compute RMS on in error on the grid +nX = drho.shape[0] +nZ = drho.shape[1] +drho2_avg = (drho**2).sum()/(nX*nZ) +drho_rms = np.sqrt(drho2_avg) + +print(f"rms error in charge conservation: {drho_rms}") +print(f"tolerance: {tolerance_rel_charge}") + +assert( drho_rms < tolerance_rel_charge ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/Implicit/inputs_vandb_2d b/Examples/Tests/Implicit/inputs_vandb_2d new file mode 100644 index 00000000000..2dc57323efe --- /dev/null +++ b/Examples/Tests/Implicit/inputs_vandb_2d @@ -0,0 +1,92 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.n0 = 1.e30 # m^-3 +my_constants.Ti = 100. # eV +my_constants.Te = 100. # eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) +my_constants.de0 = clight/wpe +my_constants.nppcz = 10 # number of particles/cell in z +my_constants.dt = 0.1/wpe # s + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 20 +amr.n_cell = 40 40 +amr.max_grid_size = 8 +amr.blocking_factor = 8 +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 # physical domain +geometry.prob_hi = 10.0*de0 10.0*de0 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = dt +#warpx.cfl = 0.5656 +warpx.use_filter = 0 + +algo.maxwell_solver = Yee +algo.evolve_scheme = "implicit_picard" +algo.require_picard_convergence = 0 +algo.max_picard_iterations = 25 +algo.picard_iteration_tolerance = 0.0 #1.0e-12 +algo.particle_pusher = "boris" +#algo.particle_pusher = "higuera" + +algo.particle_shape = 2 +#algo.current_deposition = "direct" +#algo.current_deposition = "esirkepov" +algo.current_deposition = "villasenor" + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons protons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz nppcz +electrons.profile = constant +electrons.density = 1.e30 # number per m^3 +electrons.momentum_distribution_type = "gaussian" +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.charge = q_e +protons.mass = m_p +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz nppcz +protons.profile = constant +protons.density = 1.e30 # number per m^3 +protons.momentum_distribution_type = "gaussian" +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 20 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = w ux uy uz +diag1.protons.variables = w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json b/Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json new file mode 100644 index 00000000000..d97eb04883f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 72730.70321925254, + "By": 89276.6097395453, + "Bz": 66911.00019634314, + "Ex": 92036838733000.64, + "Ey": 15583500940725.84, + "Ez": 89163420502164.97, + "divE": 8.998871921763322e+22, + "jx": 2.7748639888523993e+19, + "jy": 2.9501400595579277e+19, + "jz": 2.6976140199337787e+19, + "rho": 796777020986.2787 + }, + "protons": { + "particle_momentum_x": 2.0873315539608036e-17, + "particle_momentum_y": 2.0858882907322405e-17, + "particle_momentum_z": 2.0877345477243595e-17, + "particle_position_x": 0.004251275869323399, + "particle_position_y": 0.0042512738905209615, + "particle_weight": 2823958719279159.5 + }, + "electrons": { + "particle_momentum_x": 4.882673707817137e-19, + "particle_momentum_y": 4.879672470952739e-19, + "particle_momentum_z": 4.872329687213274e-19, + "particle_position_x": 0.004251641684258687, + "particle_position_y": 0.004251751978637919, + "particle_weight": 2823958719279159.5 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index ae22eba499d..3310e642dd3 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4515,6 +4515,23 @@ doVis = 0 compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py +[ImplicitPicard_VandB_2d] +buildDir = . +inputFile = Examples/Tests/Implicit/inputs_vandb_2d +runtime_params = warpx.abort_on_warning_threshold=high +dim = 2 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=2 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_vandb_2d.py + [SemiImplicitPicard_1d] buildDir = . inputFile = Examples/Tests/Implicit/inputs_1d_semiimplicit diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 64411ecf6e4..7cc9f571a4a 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -1130,6 +1130,8 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( return "Esirkepov"; case CurrentDepositionAlgo::Vay : return "Vay"; + case CurrentDepositionAlgo::Villasenor : + return "Villasenor"; default: return "directMorseNielson"; } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 169453a6e99..eee3012ab4d 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -229,6 +229,9 @@ WarpX::PrintMainPICparameters () else if (current_deposition_algo == CurrentDepositionAlgo::Esirkepov){ amrex::Print() << "Current Deposition: | Esirkepov \n"; } + else if (current_deposition_algo == CurrentDepositionAlgo::Villasenor){ + amrex::Print() << "Current Deposition: | Villasenor \n"; + } // Print type of particle pusher if (particle_pusher_algo == ParticlePusherAlgo::Vay){ amrex::Print() << "Particle Pusher: | Vay \n"; diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 5d1055278b2..18df09c3b43 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -1535,6 +1535,670 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #endif } +/** + * \brief Villasenor and Buneman Current Deposition for thread thread_num for implicit scheme. + * The specifics for the implicit scheme are in how gamma is determined. This is a charge- + * conserving deposition. The difference from Esirkepov is that the deposit is done segment + * by segment, where the segments are determined by cell crossings. In general, this results + * in a tighter stencil. The implementation is valid for an arbitrary number of cell crossings. + * + * \param depos_order deposition order + * \param xp_n,yp_n,zp_n Pointer to arrays of particle position at time level n. + * \param GetPosition A functor for returning the particle position. + * \param wp Pointer to array of particle weights. + * \param uxp_n,uyp_n,uzp_n Pointer to arrays of particle momentum at time level n. + * \param uxp_nph,uyp_nph,uzp_nph Pointer to arrays of particle momentum at time level n + 1/2. + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. + * \param np_to_deposit Number of particles for which current is deposited. + * \param dt Time step for particle level + * \param dx 3D cell size + * \param xyzmin Physical lower bounds of domain. + * \param lo Index lower bounds of domain. + * \param q species charge. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. + * \param load_balance_costs_update_algo Selected method for updating load balance costs. + */ +template +void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, + const amrex::ParticleReal * const yp_n, + const amrex::ParticleReal * const zp_n, + const GetParticlePosition& GetPosition, + const amrex::ParticleReal * const wp, + [[maybe_unused]]const amrex::ParticleReal * const uxp_n, + [[maybe_unused]]const amrex::ParticleReal * const uyp_n, + [[maybe_unused]]const amrex::ParticleReal * const uzp_n, + [[maybe_unused]]const amrex::ParticleReal * const uxp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uyp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uzp_nph, + const int * const ion_lev, + const amrex::Array4& Jx_arr, + const amrex::Array4& Jy_arr, + const amrex::Array4& Jz_arr, + const long np_to_deposit, + const amrex::Real dt, + const std::array& dx, + const std::array xyzmin, + const amrex::Dim3 lo, + const amrex::Real q, + const int n_rz_azimuthal_modes, + amrex::Real * const cost, + const long load_balance_costs_update_algo) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(AMREX_USE_GPU) + amrex::ignore_unused(cost, load_balance_costs_update_algo); +#endif + + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) + bool const do_ionization = ion_lev; +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if defined(WARPX_DIM_3D) + Real const invvol = 1.0_rt / (dx[0]*dx[1]*dx[2]); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + Real const invvol = 1.0_rt / (dx[0]*dx[2]); +#elif defined(WARPX_DIM_1D_Z) + Real const invvol = 1.0_rt / (dx[2]); +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + + // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr +#if defined(WARPX_USE_GPUCLOCK) + amrex::Real* cost_real = nullptr; + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); + *cost_real = 0._rt; + } +#endif + amrex::ParallelFor( + np_to_deposit, + [=] AMREX_GPU_DEVICE (long const ip) { +#if defined(WARPX_USE_GPUCLOCK) + const auto KernelTimer = ablastr::parallelization::KernelTimer( + cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), + cost_real); +#endif + +#if !defined(WARPX_DIM_3D) + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The uxp,uyp,uzp are the velocities at time level n+1/2 + const amrex::ParticleReal uxp_np1 = 2._prt*uxp_nph[ip] - uxp_n[ip]; + const amrex::ParticleReal uyp_np1 = 2._prt*uyp_nph[ip] - uyp_n[ip]; + const amrex::ParticleReal uzp_np1 = 2._prt*uzp_nph[ip] - uzp_n[ip]; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (uxp_n[ip]*uxp_n[ip] + uyp_n[ip]*uyp_n[ip] + uzp_n[ip]*uzp_n[ip])*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (uxp_np1*uxp_np1 + uyp_np1*uyp_np1 + uzp_np1*uzp_np1)*inv_c2); + const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); +#endif + + // wqx, wqy wqz are particle current in each direction + Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; + } + + ParticleReal xp_nph, yp_nph, zp_nph; + GetPosition(ip, xp_nph, yp_nph, zp_nph); + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; +#else + ignore_unused(xp_n); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; +#else + ignore_unused(yp_n); +#endif + ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n[ip]; + amrex::Real const yp_old = yp_n[ip]; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; + amrex::Real const vx = (rp_new - rp_old)/dt; + amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; +#elif defined(WARPX_DIM_XZ) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n[ip] - xmin)*dxi; + amrex::Real const vx = (xp_np1 - xp_n[ip])/dt; + amrex::Real const vy = uyp_nph[ip]*gaminv; +#elif defined(WARPX_DIM_1D_Z) + amrex::Real const vx = uxp_nph[ip]*gaminv; + amrex::Real const vy = uyp_nph[ip]*gaminv; +#elif defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n[ip] - xmin)*dxi; + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n[ip] - ymin)*dyi; + amrex::Real const vx = (xp_np1 - xp_n[ip])/dt; + amrex::Real const vy = (yp_np1 - yp_n[ip])/dt; +#endif + + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n[ip] - zmin)*dzi; + amrex::Real const vz = (zp_np1 - zp_n[ip])/dt; + + // Define velocity kernals to deposit + amrex::Real const wqx = wq*vx*invvol; + amrex::Real const wqy = wq*vy*invvol; + amrex::Real const wqz = wq*vz*invvol; + + // 1) Determine the number of segments. + // 2) Loop over segments and deposit current. + + // cell crossings are defined at cell edges if depos_order is odd + // cell crossings are defined at cell centers if depos_order is even + + int num_segments = 1; + double shift = 0.0; + if ( (depos_order % 2) == 0 ) { shift = 0.5; } + +#if defined(WARPX_DIM_3D) + + // compute cell crossings in X-direction + const auto i_old = static_cast(x_old-shift); + const auto i_new = static_cast(x_new-shift); + int cell_crossings_x = std::abs(i_new-i_old); + num_segments += cell_crossings_x; + + // compute cell crossings in Y-direction + const auto j_old = static_cast(y_old-shift); + const auto j_new = static_cast(y_new-shift); + int cell_crossings_y = std::abs(j_new-j_old); + num_segments += cell_crossings_y; + + // compute cell crossings in Z-direction + const auto k_old = static_cast(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 7) ... + + // compute total change in particle position and the initial cell + // locations in each direction used to find the position at cell crossings. + const double dxp = x_new - x_old; + const double dyp = y_new - y_old; + const double dzp = z_new - z_old; + const auto dirX_sign = static_cast(dxp < 0. ? -1. : 1.); + const auto dirY_sign = static_cast(dyp < 0. ? -1. : 1.); + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Xcell = 0., Ycell = 0., Zcell = 0.; + if (num_segments > 1) { + Xcell = static_cast(i_old) + shift + 0.5*(1.-dirX_sign); + Ycell = static_cast(j_old) + shift + 0.5*(1.-dirY_sign); + Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + } + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dxp_seg, dyp_seg, dzp_seg; + double x0_new, y0_new, z0_new; + double x0_old = x_old; + double y0_old = y_old; + double z0_old = z_old; + + for (int ns=0; ns(dxp == 0. ? 1. : dxp_seg/dxp); + const auto seg_factor_y = static_cast(dyp == 0. ? 1. : dyp_seg/dyp); + const auto seg_factor_z = static_cast(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sx_cell[depos_order] = {0.}; + double sy_cell[depos_order] = {0.}; + double sz_cell[depos_order] = {0.}; + double const x0_bar = (x0_new + x0_old)/2.0; + double const y0_bar = (y0_new + y0_old)/2.0; + double const z0_bar = (z0_new + z0_old)/2.0; + const int i0_cell = compute_shape_factor_cell( sx_cell, x0_bar-0.5 ); + const int j0_cell = compute_shape_factor_cell( sy_cell, y0_bar-0.5 ); + const int k0_cell = compute_shape_factor_cell( sz_cell, z0_bar-0.5 ); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sx_old_cell[depos_order] = {0.}; + double sx_new_cell[depos_order] = {0.}; + double sy_old_cell[depos_order] = {0.}; + double sy_new_cell[depos_order] = {0.}; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int i0_cell_2 = compute_shape_factors_cell( sx_old_cell, sx_new_cell, x0_old-0.5, x0_new-0.5 ); + const int j0_cell_2 = compute_shape_factors_cell( sy_old_cell, sy_new_cell, y0_old-0.5, y0_new-0.5 ); + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(i0_cell_2, j0_cell_2, k0_cell_2); + for (int m=0; m(x_old-shift); + const auto i_new = static_cast(x_new-shift); + int cell_crossings_x = std::abs(i_new-i_old); + num_segments += cell_crossings_x; + + // compute cell crossings in Z-direction + const auto k_old = static_cast(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 5) ... + + // compute total change in particle position and the initial cell + // locations in each direction used to find the position at cell crossings. + const double dxp = x_new - x_old; + const double dzp = z_new - z_old; + const auto dirX_sign = static_cast(dxp < 0. ? -1. : 1.); + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Xcell = 0., Zcell = 0.; + if (num_segments > 1) { + Xcell = static_cast(i_old) + shift + 0.5*(1.-dirX_sign); + Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + } + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dxp_seg, dzp_seg; + double x0_new, z0_new; + double x0_old = x_old; + double z0_old = z_old; + + for (int ns=0; ns(dxp == 0. ? 1. : dxp_seg/dxp); + const auto seg_factor_z = static_cast(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sx_cell[depos_order] = {0.}; + double sz_cell[depos_order] = {0.}; + double const x0_bar = (x0_new + x0_old)/2.0; + double const z0_bar = (z0_new + z0_old)/2.0; + const int i0_cell = compute_shape_factor_cell( sx_cell, x0_bar-0.5 ); + const int k0_cell = compute_shape_factor_cell( sz_cell, z0_bar-0.5 ); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sx_old_cell[depos_order] = {0.}; + double sx_new_cell[depos_order] = {0.}; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int i0_cell_2 = compute_shape_factors_cell( sx_old_cell, sx_new_cell, x0_old-0.5, x0_new-0.5 ); + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(i0_cell_2, k0_cell_2); + for (int m=0; m(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 3) ... + + // compute dzp and the initial cell location used to find the cell crossings. + double const dzp = z_new - z_old; + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dzp_seg; + double z0_new; + double z0_old = z_old; + + for (int ns=0; ns(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sz_cell[depos_order] = {0.}; + double const z0_bar = (z0_new + z0_old)/2.0; + const int k0_cell = compute_shape_factor_cell( sz_cell, z0_bar-0.5 ); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(k0_cell_2); + for (int m=0; mfree(cost_real); + } +#endif +} + /** * \brief Vay current deposition * ( Vay et al, 2013) diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index dd6b7276681..670d95014a0 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -880,6 +880,681 @@ void doGatherShapeNEsirkepovStencilImplicit ( #endif } +/** + * \brief Energy conserving field gather for thread thread_num for the implicit scheme + * This uses the same stencil for the gather that is used for Villasenor current deposition. + * The magnetic field is deposited using direct deposition. + * + * \tparam depos_order Particle shape order + * \param xp_n,yp_n,zp_n Particle position coordinates at start of step + * \param xp_nph,yp_nph,zp_nph Particle position coordinates at half step + * \param Exp,Eyp,Ezp Electric field on particles. + * \param Bxp,Byp,Bzp Magnetic field on particles. + * \param Ex_arr,Ey_arr,Ez_arr Array4 of the electric field, either full array or tile. + * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. + * \param Ex_type,Ey_type,Ez_type IndexType of the electric field + * \param Bx_type,By_type,Bz_type IndexType of the magnetic field + * \param dx 3D cell spacing + * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + */ +template +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void doGatherPicnicShapeN ( + [[maybe_unused]] const amrex::ParticleReal xp_n, + [[maybe_unused]] const amrex::ParticleReal yp_n, + const amrex::ParticleReal zp_n, + [[maybe_unused]] const amrex::ParticleReal xp_nph, + [[maybe_unused]] const amrex::ParticleReal yp_nph, + const amrex::ParticleReal zp_nph, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp, + amrex::Array4 const& Ex_arr, + amrex::Array4 const& Ey_arr, + amrex::Array4 const& Ez_arr, + amrex::Array4 const& Bx_arr, + amrex::Array4 const& By_arr, + amrex::Array4 const& Bz_arr, + [[maybe_unused]] const amrex::IndexType Ex_type, + [[maybe_unused]] const amrex::IndexType Ey_type, + [[maybe_unused]] const amrex::IndexType Ez_type, + [[maybe_unused]] const amrex::IndexType Bx_type, + [[maybe_unused]] const amrex::IndexType By_type, + [[maybe_unused]] const amrex::IndexType Bz_type, + const amrex::GpuArray& dx, + const amrex::GpuArray& xyzmin, + const amrex::Dim3& lo, + const int n_rz_azimuthal_modes) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal xp_np1 = 2._prt*xp_nph - xp_n; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal yp_np1 = 2._prt*yp_nph - yp_n; +#endif + ParticleReal zp_np1 = 2._prt*zp_nph - zp_n; + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n; + amrex::Real const yp_old = yp_n; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; + double const x_bar = (rp_mid - xmin)*dxi; +#elif !defined(WARPX_DIM_1D_Z) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n - xmin)*dxi; + double const x_bar = (xp_nph - xmin)*dxi; +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n - ymin)*dyi; + double const y_bar = (yp_nph - ymin)*dyi; +#endif + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n - zmin)*dzi; + double const z_bar = (zp_nph - zmin)*dzi; + + // 1) Determine the number of segments. + // 2) Loop over segments and gather electric field. + // 3) Gather magnetic field. + + // cell crossings are defined at cell edges if depos_order is odd + // cell crossings are defined at cell centers if depos_order is even + + int num_segments = 1; + double shift = 0.0; + if ( (depos_order % 2) == 0 ) { shift = 0.5; } + +#if defined(WARPX_DIM_3D) + + // compute cell crossings in X-direction + const auto i_old = static_cast(x_old-shift); + const auto i_new = static_cast(x_new-shift); + int cell_crossings_x = std::abs(i_new-i_old); + num_segments += cell_crossings_x; + + // compute cell crossings in Y-direction + const auto j_old = static_cast(y_old-shift); + const auto j_new = static_cast(y_new-shift); + int cell_crossings_y = std::abs(j_new-j_old); + num_segments += cell_crossings_y; + + // compute cell crossings in Z-direction + const auto k_old = static_cast(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 7) ... + + // compute total change in particle position and the initial cell + // locations in each direction used to find the position at cell crossings. + const double dxp = x_new - x_old; + const double dyp = y_new - y_old; + const double dzp = z_new - z_old; + const auto dirX_sign = static_cast(dxp < 0. ? -1. : 1.); + const auto dirY_sign = static_cast(dyp < 0. ? -1. : 1.); + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Xcell = 0., Ycell = 0., Zcell = 0.; + if (num_segments > 1) { + Xcell = static_cast(i_old) + shift + 0.5*(1.-dirX_sign); + Ycell = static_cast(j_old) + shift + 0.5*(1.-dirY_sign); + Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + } + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dxp_seg, dyp_seg, dzp_seg; + double x0_new, y0_new, z0_new; + double x0_old = x_old; + double y0_old = y_old; + double z0_old = z_old; + + for (int ns=0; ns(dxp == 0. ? 1. : dxp_seg/dxp); + const auto seg_factor_y = static_cast(dyp == 0. ? 1. : dyp_seg/dyp); + const auto seg_factor_z = static_cast(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sx_cell[depos_order] = {0.}; + double sy_cell[depos_order] = {0.}; + double sz_cell[depos_order] = {0.}; + double const x0_bar = (x0_new + x0_old)/2.0; + double const y0_bar = (y0_new + y0_old)/2.0; + double const z0_bar = (z0_new + z0_old)/2.0; + const int i0_cell = compute_shape_factor_cell( sx_cell, x0_bar-0.5 ); + const int j0_cell = compute_shape_factor_cell( sy_cell, y0_bar-0.5 ); + const int k0_cell = compute_shape_factor_cell( sz_cell, z0_bar-0.5 ); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sx_old_cell[depos_order] = {0.}; + double sx_new_cell[depos_order] = {0.}; + double sy_old_cell[depos_order] = {0.}; + double sy_new_cell[depos_order] = {0.}; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int i0_cell_2 = compute_shape_factors_cell( sx_old_cell, sx_new_cell, x0_old-0.5, x0_new-0.5 ); + const int j0_cell_2 = compute_shape_factors_cell( sy_old_cell, sy_new_cell, y0_old-0.5, y0_new-0.5 ); + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(i0_cell_2, j0_cell_2, k0_cell_2); + for (int m=0; m compute_shape_factor_B; + double sz_bar_node[depos_order_B+1] = {0.}; + double sz_bar_cell[depos_order_B+1] = {0.}; + const int k_bar_node = compute_shape_factor_B(sz_bar_node, z_bar); + const int k_bar_cell = compute_shape_factor_B(sz_bar_cell, z_bar-0.5); + double sy_bar_node[depos_order_B+1] = {0.}; + double sy_bar_cell[depos_order_B+1] = {0.}; + const int j_bar_node = compute_shape_factor_B(sy_bar_node, y_bar); + const int j_bar_cell = compute_shape_factor_B(sy_bar_cell, y_bar-0.5); + double sx_bar_node[depos_order_B+1] = {0.}; + double sx_bar_cell[depos_order_B+1] = {0.}; + const int i_bar_node = compute_shape_factor_B(sx_bar_node, x_bar); + const int i_bar_cell = compute_shape_factor_B(sx_bar_cell, x_bar-0.5); + + amrex::Real weight; + for (int i=0; i<=depos_order_B; i++) { + for (int j=0; j<=depos_order_B; j++) { + for (int k=0; k<=depos_order_B; k++) { + weight = static_cast(sx_bar_node[i]*sy_bar_cell[j]*sz_bar_cell[k]); + Bxp += Bx_arr(lo.x+i_bar_node+i, lo.y+j_bar_cell+j, lo.z+k_bar_cell+k)*weight; + // + weight = static_cast(sx_bar_cell[i]*sy_bar_node[j]*sz_bar_cell[k]); + Byp += By_arr(lo.x+i_bar_cell+i, lo.y+j_bar_node+j, lo.z+k_bar_cell+k)*weight; + // + weight = static_cast(sx_bar_cell[i]*sy_bar_cell[j]*sz_bar_node[k]); + Bzp += Bz_arr(lo.x+i_bar_cell+i, lo.y+j_bar_cell+j, lo.z+k_bar_node+k)*weight; + } + } + } + +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + + // compute cell crossings in X-direction + const auto i_old = static_cast(x_old-shift); + const auto i_new = static_cast(x_new-shift); + int cell_crossings_x = std::abs(i_new-i_old); + num_segments += cell_crossings_x; + + // compute cell crossings in Z-direction + const auto k_old = static_cast(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 5) ... + + // compute total change in particle position and the initial cell + // locations in each direction used to find the position at cell crossings. + const double dxp = x_new - x_old; + const double dzp = z_new - z_old; + const auto dirX_sign = static_cast(dxp < 0. ? -1. : 1.); + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Xcell = 0., Zcell = 0.; + if (num_segments > 1) { + Xcell = static_cast(i_old) + shift + 0.5*(1.-dirX_sign); + Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + } + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dxp_seg, dzp_seg; + double x0_new, z0_new; + double x0_old = x_old; + double z0_old = z_old; + + for (int ns=0; ns(dxp == 0. ? 1. : dxp_seg/dxp); + const auto seg_factor_z = static_cast(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sx_cell[depos_order] = {0.}; + double sz_cell[depos_order] = {0.}; + double const x0_bar = (x0_new + x0_old)/2.0; + double const z0_bar = (z0_new + z0_old)/2.0; + const int i0_cell = compute_shape_factor_cell(sx_cell, x0_bar-0.5); + const int k0_cell = compute_shape_factor_cell(sz_cell, z0_bar-0.5); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sx_old_cell[depos_order] = {0.}; + double sx_new_cell[depos_order] = {0.}; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int i0_cell_2 = compute_shape_factors_cell( sx_old_cell, sx_new_cell, x0_old-0.5, x0_new-0.5 ); + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(i0_cell_2, k0_cell_2); + for (int m=0; m compute_shape_factor_B; + double sz_bar_node[depos_order_B+1] = {0.}; + double sz_bar_cell[depos_order_B+1] = {0.}; + const int k_bar_node = compute_shape_factor_B(sz_bar_node, z_bar); + const int k_bar_cell = compute_shape_factor_B(sz_bar_cell, z_bar-0.5); + double sx_bar_node[depos_order_B+1] = {0.}; + double sx_bar_cell[depos_order_B+1] = {0.}; + const int i_bar_node = compute_shape_factor_B(sx_bar_node, x_bar); + const int i_bar_cell = compute_shape_factor_B(sx_bar_cell, x_bar-0.5); + + for (int i=0; i<=depos_order_B; i++) { + for (int k=0; k<=depos_order_B; k++) { + const auto weight_Bz = static_cast(sx_bar_cell[i]*sz_bar_node[k]); + Bzp += Bz_arr(lo.x+i_bar_cell+i, lo.y+k_bar_node+k, 0, 0)*weight_Bz; + // + const auto weight_Bx = static_cast(sx_bar_node[i]*sz_bar_cell[k]); + Bxp += Bx_arr(lo.x+i_bar_node+i, lo.y+k_bar_cell+k, 0, 0)*weight_Bx; + // + const auto weight_By = static_cast(sx_bar_cell[i]*sz_bar_cell[k]); + Byp += By_arr(lo.x+i_bar_cell+i, lo.y+k_bar_cell+k, 0, 0)*weight_By; +#if defined(WARPX_DIM_RZ) + Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + const auto dBx = (+ Bx_arr(lo.x+i_bar_node+i, lo.y+k_bar_cell+k, 0, 2*imode-1)*xy_mid.real() + - Bx_arr(lo.x+i_bar_node+i, lo.y+k_bar_cell+k, 0, 2*imode)*xy_mid.imag()); + const auto dBy = (+ By_arr(lo.x+i_bar_cell+i, lo.y+k_bar_cell+k, 0, 2*imode-1)*xy_mid.real() + - By_arr(lo.x+i_bar_cell+i, lo.y+k_bar_cell+k, 0, 2*imode)*xy_mid.imag()); + const auto dBz = (+ Bz_arr(lo.x+i_bar_cell+i, lo.y+k_bar_node+k, 0, 2*imode-1)*xy_mid.real() + - Bz_arr(lo.x+i_bar_cell+i, lo.y+k_bar_node+k, 0, 2*imode)*xy_mid.imag()); + Bxp += weight_Bx*dBx; + Byp += weight_By*dBy; + Bzp += weight_Bz*dBz; + xy_mid = xy_mid*xy_mid0; + } +#endif + } + } + +#ifdef WARPX_DIM_RZ + + // Convert Exp and Eyp (which are actually Er and Etheta) to Ex and Ey + const amrex::Real Exp_save = Exp; + Exp = costheta_mid*Exp - sintheta_mid*Eyp; + Eyp = costheta_mid*Eyp + sintheta_mid*Exp_save; + const amrex::Real Bxp_save = Bxp; + Bxp = costheta_mid*Bxp - sintheta_mid*Byp; + Byp = costheta_mid*Byp + sintheta_mid*Bxp_save; + +#endif + +#elif defined(WARPX_DIM_1D_Z) + + // compute cell crossings in Z-direction + const auto k_old = static_cast(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 3) ... + + // compute dzp and the initial cell location used to find the cell crossings. + double const dzp = z_new - z_old; + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dzp_seg; + double z0_new; + double z0_old = z_old; + + for (int ns=0; ns(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sz_cell[depos_order] = {0.}; + double const z0_bar = (z0_new + z0_old)/2.0; + const int k0_cell = compute_shape_factor_cell( sz_cell, z0_bar-0.5 ); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(k0_cell_2); + for (int m=0; m compute_shape_factor_B; + double sz_bar_node[depos_order_B+1] = {0.}; + double sz_bar_cell[depos_order_B+1] = {0.}; + const int k_bar_node = compute_shape_factor_B(sz_bar_node, z_bar); + const int k_bar_cell = compute_shape_factor_B(sz_bar_cell, z_bar-0.5_rt); + + amrex::Real weight; + for (int k=0; k<=depos_order_B; k++) { + weight = static_cast(sz_bar_node[k]); + Bzp += Bz_arr(lo.x+k_bar_node+k, 0, 0)*weight; + // + weight = static_cast(sz_bar_cell[k]); + Bxp += Bx_arr(lo.x+k_bar_cell+k, 0, 0)*weight; + Byp += By_arr(lo.x+k_bar_cell+k, 0, 0)*weight; + } + +#endif +} + /** * \brief Field gather for particles * @@ -1052,6 +1727,7 @@ void doGatherShapeN (const amrex::ParticleReal xp, * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry * \param nox order of the particle shape function + * \param gather_type integer identifier for which algorithm to use * \param galerkin_interpolation whether to use lower order in v */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -1085,9 +1761,9 @@ void doGatherShapeNImplicit ( const amrex::Dim3& lo, const int n_rz_azimuthal_modes, const int nox, - const bool galerkin_interpolation) + const int depos_type ) { - if (galerkin_interpolation) { + if (depos_type==0) { // CurrentDepositionAlgo::Esirkepov if (nox == 1) { doGatherShapeNEsirkepovStencilImplicit<1>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, @@ -1107,7 +1783,35 @@ void doGatherShapeNImplicit ( ex_type, ey_type, ez_type, bx_type, by_type, bz_type, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); } - } else { + } + else if (depos_type==3) { // CurrentDepositionAlgo::Villasenor + if (nox == 1) { + doGatherPicnicShapeN<1>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherPicnicShapeN<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherPicnicShapeN<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 4) { + doGatherPicnicShapeN<4>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } + else if (depos_type==1) { // CurrentDepositionAlgo::Direct if (nox == 1) { doGatherShapeN<1,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index e39cd79b55d..929c3c26649 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2984,7 +2984,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const Dim3 lo = lbound(box); - bool galerkin_interpolation = WarpX::galerkin_interpolation; + int depos_type = WarpX::current_deposition_algo; int nox = WarpX::nox; int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; @@ -3107,8 +3107,8 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, doGatherShapeNImplicit(xp_n, yp_n, zp_n, xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, - nox, galerkin_interpolation); + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, nox, + depos_type ); } // Externally applied E and B-field in Cartesian co-ordinates diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index a0b4ed63a30..73e8f7243bb 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -64,6 +64,19 @@ struct Compute_shape_factor // index of the leftmost cell where particle deposits return j-1; } + else if constexpr (depos_order == 4){ + const auto j = static_cast(xmid + T(0.5)); + const T xint = xmid - T(j); + const T xint_p1 = xint + T(1.0); + const T xint_m1 = xint - T(1.0); + sx[0] = T(1.0)/T(384.0)*(T(1.0) - T(2.0)*xint)*(T(1.0) - T(2.0)*xint)*(T(1.0) - T(2.0)*xint)*(T(1.0) - T(2.0)*xint); + sx[1] = T(1.0)/T(96.0)*(T(55.0) + T(4.0)*xint_p1*(T(5.0) - T(2.0)*xint_p1*(T(15.0) + T(2.0)*xint_p1*(xint_p1 - T(5.0))))); + sx[2] = T(115.0)/T(192.0) + xint*xint*(xint*xint/T(4.0) - T(5.0)/T(8.0)); + sx[3] = T(1.0)/T(96.0)*(T(55.0) - T(4.0)*xint_m1*(T(5.0) + T(2.0)*xint_m1*(T(15.0) - T(2.0)*xint_m1*(-xint_m1 - T(5.0))))); + sx[4] = T(1.0)/T(384.0)*(T(1.0) + T(2.0)*xint)*(T(1.0) + T(2.0)*xint)*(T(1.0) + T(2.0)*xint)*(T(1.0) + T(2.0)*xint); + // index of the leftmost cell where particle deposits + return j-2; + } else{ WARPX_ABORT_WITH_MESSAGE("Unknown particle shape selected in Compute_shape_factor"); amrex::ignore_unused(sx, xmid); @@ -132,4 +145,96 @@ struct Compute_shifted_shape_factor } }; +/** + * Compute shape factors for two positions that are within + * half a grid cell of the same cell interface and return the common + * index of the leftmost cell where particle writes, which is correctly + * determined by the average of the positions. + * This is used for computing the segment weights transverse to the + * current density direction in the Villasenor deposition algorithm. + */ +template +struct Compute_shape_factor_pair +{ + template< typename T > + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + int operator()( + T* const sx_old, + T* const sx_new, + T xold, + T xnew) const + { + const T xmid = T(0.5)*(xnew + xold); + if constexpr (depos_order == 1){ + const auto j = static_cast(xmid); + const T xint_old = xold - T(j); + sx_old[0] = T(1.0) - xint_old; + sx_old[1] = xint_old; + // + const T xint_new = xnew - T(j); + sx_new[0] = T(1.0) - xint_new; + sx_new[1] = xint_new; + return j; + } + else if constexpr (depos_order == 2){ + const auto j = static_cast(xmid + T(0.5)); + const T xint_old = xold - T(j); + sx_old[0] = T(0.5)*(T(0.5) - xint_old)*(T(0.5) - xint_old); + sx_old[1] = T(0.75) - xint_old*xint_old; + sx_old[2] = T(0.5)*(T(0.5) + xint_old)*(T(0.5) + xint_old); + // + const T xint_new = xnew - T(j); + sx_new[0] = T(0.5)*(T(0.5) - xint_new)*(T(0.5) - xint_new); + sx_new[1] = T(0.75) - xint_new*xint_new; + sx_new[2] = T(0.5)*(T(0.5) + xint_new)*(T(0.5) + xint_new); + // index of the leftmost cell where particle deposits + return j-1; + } + else if constexpr (depos_order == 3){ + const auto j = static_cast(xmid); + const T xint_old = xold - T(j); + sx_old[0] = T(1.0)/T(6.0)*(T(1.0) - xint_old)*(T(1.0) - xint_old)*(T(1.0) - xint_old); + sx_old[1] = T(2.0)/T(3.0) - xint_old*xint_old*(T(1.0) - xint_old/(T(2.0))); + sx_old[2] = T(2.0)/T(3.0) - (T(1.0) - xint_old)*(T(1.0) - xint_old)*(T(1.0) - T(0.5)*(T(1.0) - xint_old)); + sx_old[3] = T(1.0)/T(6.0)*xint_old*xint_old*xint_old; + // + const T xint_new = xnew - T(j); + sx_new[0] = T(1.0)/T(6.0)*(T(1.0) - xint_new)*(T(1.0) - xint_new)*(T(1.0) - xint_new); + sx_new[1] = T(2.0)/T(3.0) - xint_new*xint_new*(T(1.0) - xint_new/(T(2.0))); + sx_new[2] = T(2.0)/T(3.0) - (T(1.0) - xint_new)*(T(1.0) - xint_new)*(T(1.0) - T(0.5)*(T(1.0) - xint_new)); + sx_new[3] = T(1.0)/T(6.0)*xint_new*xint_new*xint_new; + // index of the leftmost cell where particle deposits + return j-1; + } + else if constexpr (depos_order == 4){ + const auto j = static_cast(xmid + T(0.5)); + const T xint_old = xold - T(j); + T xint_p1 = xint_old + T(1.0); + T xint_m1 = xint_old - T(1.0); + sx_old[0] = T(1.0)/T(384.0)*(T(1.0) - T(2.0)*xint_old)*(T(1.0) - T(2.0)*xint_old)*(T(1.0) - T(2.0)*xint_old)*(T(1.0) - T(2.0)*xint_old); + sx_old[1] = T(1.0)/T(96.0)*(T(55.0) + T(4.0)*xint_p1*(T(5.0) - T(2.0)*xint_p1*(T(15.0) + T(2.0)*xint_p1*(xint_p1 - T(5.0))))); + sx_old[2] = T(115.0)/T(192.0) + xint_old*xint_old*(xint_old*xint_old/T(4.0) - T(5.0)/T(8.0)); + sx_old[3] = T(1.0)/T(96.0)*(T(55.0) - T(4.0)*xint_m1*(T(5.0) + T(2.0)*xint_m1*(T(15.0) - T(2.0)*xint_m1*(-xint_m1 - T(5.0))))); + sx_old[4] = T(1.0)/T(384.0)*(T(1.0) + T(2.0)*xint_old)*(T(1.0) + T(2.0)*xint_old)*(T(1.0) + T(2.0)*xint_old)*(T(1.0) + T(2.0)*xint_old); + // + const T xint_new = xnew - T(j); + xint_p1 = xint_new + T(1.0); + xint_m1 = xint_new - T(1.0); + sx_new[0] = T(1.0)/T(384.0)*(T(1.0) - T(2.0)*xint_new)*(T(1.0) - T(2.0)*xint_new)*(T(1.0) - T(2.0)*xint_new)*(T(1.0) - T(2.0)*xint_new); + sx_new[1] = T(1.0)/T(96.0)*(T(55.0) + T(4.0)*xint_p1*(T(5.0) - T(2.0)*xint_p1*(T(15.0) + T(2.0)*xint_p1*(xint_p1 - T(5.0))))); + sx_new[2] = T(115.0)/T(192.0) + xint_new*xint_new*(xint_new*xint_new/T(4.0) - T(5.0)/T(8.0)); + sx_new[3] = T(1.0)/T(96.0)*(T(55.0) - T(4.0)*xint_m1*(T(5.0) + T(2.0)*xint_m1*(T(15.0) - T(2.0)*xint_m1*(-xint_m1 - T(5.0))))); + sx_new[4] = T(1.0)/T(384.0)*(T(1.0) + T(2.0)*xint_new)*(T(1.0) + T(2.0)*xint_new)*(T(1.0) + T(2.0)*xint_new)*(T(1.0) + T(2.0)*xint_new); + // + // index of the leftmost cell where particle deposits + return j-2; + } + else{ + WARPX_ABORT_WITH_MESSAGE("Unknown particle shape selected in Compute_shape_factor_pair"); + amrex::ignore_unused(sx_old, sx_new, xold, xnew); + } + return 0; + } +}; + #endif // SHAPEFACTORS_H_ diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 85bb1c3f4b8..a395198e361 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -452,9 +452,10 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, // Take into account Galilean shift const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev, 0.5_rt*dt); - if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { + if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov || + WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { if (WarpX::grid_type == GridType::Collocated) { - WARPX_ABORT_WITH_MESSAGE("The Esirkepov algorithm cannot be used with a collocated grid."); + WARPX_ABORT_WITH_MESSAGE("Charge-conserving current depositions (Esirkepov and Villasenor) cannot be used with a collocated grid."); } } @@ -517,6 +518,9 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { WARPX_ABORT_WITH_MESSAGE("Cannot do shared memory deposition with Esirkepov algorithm"); } + else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { + WARPX_ABORT_WITH_MESSAGE("Cannot do shared memory deposition with Villasenor algorithm"); + } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { WARPX_ABORT_WITH_MESSAGE("Cannot do shared memory deposition with Vay algorithm"); } @@ -525,21 +529,21 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, if (WarpX::nox == 1){ doDepositionSharedShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 2){ doDepositionSharedShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 3){ doDepositionSharedShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); @@ -620,6 +624,66 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, WarpX::load_balance_costs_update_algo); } } + } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { + if (push_type == PushType::Implicit) { +#if (AMREX_SPACEDIM >= 2) + auto& xp_n = pti.GetAttribs(particle_comps["x_n"]); + const ParticleReal* xp_n_data = xp_n.dataPtr() + offset; +#else + const ParticleReal* xp_n_data = nullptr; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + auto& yp_n = pti.GetAttribs(particle_comps["y_n"]); + const ParticleReal* yp_n_data = yp_n.dataPtr() + offset; +#else + const ParticleReal* yp_n_data = nullptr; +#endif + auto& zp_n = pti.GetAttribs(particle_comps["z_n"]); + const ParticleReal* zp_n_data = zp_n.dataPtr() + offset; + auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); + auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); + auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + if (WarpX::nox == 1){ + doVillasenorDepositionShapeNImplicit<1>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doVillasenorDepositionShapeNImplicit<2>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doVillasenorDepositionShapeNImplicit<3>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doVillasenorDepositionShapeNImplicit<4>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } + } + else { + WARPX_ABORT_WITH_MESSAGE("The Villasenor algorithm can only be used with implicit algorithm."); + } } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { if (push_type == PushType::Implicit) { WARPX_ABORT_WITH_MESSAGE("The Vay algorithm cannot be used with implicit algorithm."); diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 87db0ae9b9b..735fc7993f1 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -87,7 +87,8 @@ struct CurrentDepositionAlgo { enum { Esirkepov = 0, Direct = 1, - Vay = 2 + Vay = 2, + Villasenor = 3 }; }; diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 75c488134e0..abaf17f0a2c 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -63,10 +63,11 @@ const std::map particle_pusher_algo_to_int = { }; const std::map current_deposition_algo_to_int = { - {"esirkepov", CurrentDepositionAlgo::Esirkepov }, - {"direct", CurrentDepositionAlgo::Direct }, - {"vay", CurrentDepositionAlgo::Vay }, - {"default", CurrentDepositionAlgo::Esirkepov } // NOTE: overwritten for PSATD and Hybrid-PIC below + {"esirkepov", CurrentDepositionAlgo::Esirkepov }, + {"direct", CurrentDepositionAlgo::Direct }, + {"vay", CurrentDepositionAlgo::Vay }, + {"villasenor", CurrentDepositionAlgo::Villasenor }, + {"default", CurrentDepositionAlgo::Esirkepov } // NOTE: overwritten for PSATD and Hybrid-PIC below }; const std::map charge_deposition_algo_to_int = { diff --git a/Source/WarpX.H b/Source/WarpX.H index e8c7ae79f7e..d3900c91593 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -156,7 +156,7 @@ public: int maxlevel_extEMfield_init; // Algorithms - //! Integer that corresponds to the current deposition algorithm (Esirkepov, direct, Vay) + //! Integer that corresponds to the current deposition algorithm (Esirkepov, direct, Vay, Villasenor) static short current_deposition_algo; //! Integer that corresponds to the charge deposition algorithm (only standard deposition) static short charge_deposition_algo; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 92623bc2a06..5031381561f 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1157,6 +1157,12 @@ WarpX::ReadParameters () "Current centering (nodal deposition) cannot be used with Esirkepov deposition." "Please set warpx.do_current_centering = 0 or algo.current_deposition = direct."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + current_deposition_algo != CurrentDepositionAlgo::Villasenor || + !do_current_centering, + "Current centering (nodal deposition) cannot be used with Villasenor deposition." + "Please set warpx.do_current_centering = 0 or algo.current_deposition = direct."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( WarpX::current_deposition_algo != CurrentDepositionAlgo::Vay || !do_current_centering, @@ -1179,6 +1185,14 @@ WarpX::ReadParameters () "Vay deposition not implemented with multi-J algorithm"); } + if (current_deposition_algo == CurrentDepositionAlgo::Villasenor) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard, + "Villasenor current deposition can only" + "be used with Implicit evolve schemes."); + } + // Query algo.field_gathering from input, set field_gathering_algo to // "default" if not found (default defined in Utils/WarpXAlgorithmSelection.cpp) field_gathering_algo = static_cast(GetAlgorithmInteger(pp_algo, "field_gathering")); @@ -1243,8 +1257,9 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo == CurrentDepositionAlgo::Esirkepov || + current_deposition_algo == CurrentDepositionAlgo::Villasenor || current_deposition_algo == CurrentDepositionAlgo::Direct, - "Only Esirkepov or Direct current deposition supported with the implicit and semi-implicit schemes"); + "Only Esirkepov, Villasenor, or Direct current deposition supported with the implicit and semi-implicit schemes"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || @@ -1259,18 +1274,6 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( field_gathering_algo != GatheringAlgo::MomentumConserving, "With implicit and semi-implicit schemes, the momentum conserving field gather is not supported as it would not conserve energy"); - - if (current_deposition_algo == CurrentDepositionAlgo::Direct) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - !galerkin_interpolation, - "With implicit and semi-implicit schemes and direct deposition, the Galerkin field gathering must be turned off in order to conserve energy"); - } - - if (current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - galerkin_interpolation, - "With implicit and semi-implicit schemes and Esirkepov deposition, the Galerkin field gathering must be turned on in order to conserve energy"); - } } // Load balancing parameters @@ -1325,10 +1328,18 @@ WarpX::ReadParameters () if (!species_names.empty() || !lasers_names.empty()) { if (utils::parser::queryWithParser(pp_algo, "particle_shape", particle_shape)){ - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (particle_shape >= 1) && (particle_shape <=3), - "algo.particle_shape can be only 1, 2, or 3" - ); + if(current_deposition_algo == CurrentDepositionAlgo::Villasenor) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (particle_shape >= 1) && (particle_shape <=4), + "algo.particle_shape can be only 1, 2, 3, or 4 with villasenor deposition" + ); + } + else { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (particle_shape >= 1) && (particle_shape <=3), + "algo.particle_shape can be only 1, 2, or 3" + ); + } nox = particle_shape; noy = particle_shape; @@ -1337,7 +1348,8 @@ WarpX::ReadParameters () else{ WARPX_ABORT_WITH_MESSAGE( "algo.particle_shape must be set in the input file:" - " please set algo.particle_shape to 1, 2, or 3"); + " please set algo.particle_shape to 1, 2, or 3." + " if using the villasenor deposition, can use 4 also."); } if ((maxLevel() > 0) && (particle_shape > 1) && (do_pml_j_damping == 1)) @@ -1481,6 +1493,7 @@ WarpX::ReadParameters () // are used current_correction = true; if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov || + WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor || WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay || WarpX::do_dive_cleaning) { @@ -1495,6 +1508,7 @@ WarpX::ReadParameters () if (!current_correction && current_deposition_algo != CurrentDepositionAlgo::Esirkepov && + current_deposition_algo != CurrentDepositionAlgo::Villasenor && current_deposition_algo != CurrentDepositionAlgo::Vay) { ablastr::warn_manager::WMRecordWarning( @@ -1585,14 +1599,15 @@ WarpX::ReadParameters () ); - if (current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { + if (current_deposition_algo == CurrentDepositionAlgo::Esirkepov || + current_deposition_algo == CurrentDepositionAlgo::Villasenor) { // The comoving PSATD algorithm is not implemented nor tested with Esirkepov current deposition WARPX_ALWAYS_ASSERT_WITH_MESSAGE(v_comoving_is_zero, - "Esirkepov current deposition cannot be used with the comoving PSATD algorithm"); + "charge-conserving current depositions (Esirkepov and Villasenor) cannot be used with the comoving PSATD algorithm"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(v_galilean_is_zero, - "Esirkepov current deposition cannot be used with the Galilean algorithm."); + "charge-conserving current depositions (Esirkepov and Villasenor) cannot be used with the Galilean algorithm."); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index ad01ba4a213..f43e35c6b0b 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -195,6 +195,11 @@ deposit_charge (typename T_PC::ParIterType& pti, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo); + } else if (nox == 4){ + doChargeDepositionShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, + rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + n_rz_azimuthal_modes, cost, + load_balance_costs_update_algo); } ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd, do_device_synchronize); From 6e332e9479baa2769ff0ac22adb51c25c67627da Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 2 Feb 2024 14:21:51 -0800 Subject: [PATCH 135/176] Particle Container to Pure SoA Again (#4653) * AMReX & pyAMReX: Latest `development` More pure SoA and id handling goodness. * Particle Container to Pure SoA Again Transition to new, purely SoA particle containers. This was originally merged in #3850 and reverted in #4652, since we discovered issues loosing particles & laser particles on GPU. * Modernize `idcpu` Treatment - faster: less emitted operations, no jumps - cheaper: less used registers - safer: no read-before-write warnings - cooler: no explanation needed --- .github/workflows/cuda.yml | 2 +- Docs/source/developers/amrex_basics.rst | 2 +- Docs/source/developers/dimensionality.rst | 4 +- Docs/source/developers/particles.rst | 18 +- Docs/source/usage/workflows/python_extend.rst | 28 +- .../particle_data_python/PICMI_inputs_2d.py | 6 +- .../PICMI_inputs_prev_pos_2d.py | 6 +- .../PICMI_inputs_runtime_component_analyze.py | 6 +- Python/pywarpx/_libwarpx.py | 6 +- Python/pywarpx/particle_containers.py | 255 ++++++++---------- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- .../BackTransformParticleFunctor.H | 16 +- .../FlushFormats/FlushFormatAscent.cpp | 3 - .../FlushFormats/FlushFormatCheckpoint.cpp | 9 +- .../FlushFormats/FlushFormatPlotfile.cpp | 13 +- Source/Diagnostics/ParticleIO.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 6 +- .../FieldProbeParticleContainer.H | 20 +- .../FieldProbeParticleContainer.cpp | 34 +-- .../ReducedDiags/LoadBalanceCosts.cpp | 3 +- Source/Diagnostics/WarpXOpenPMD.H | 4 +- Source/Diagnostics/WarpXOpenPMD.cpp | 139 ++-------- .../ParticleBoundaryProcess.H | 7 +- Source/EmbeddedBoundary/ParticleScraper.H | 7 +- .../BinaryCollision/BinaryCollision.H | 16 +- .../Coulomb/PairWiseCoulombCollisionFunc.H | 7 +- .../Collision/BinaryCollision/DSMC/DSMC.H | 3 +- .../DSMC/SplitAndScatterFunc.H | 28 +- .../NuclearFusion/NuclearFusionFunc.H | 14 +- .../ProtonBoronFusionInitializeMomentum.H | 10 +- .../TwoProductFusionInitializeMomentum.H | 4 +- .../BinaryCollision/ParticleCreationFunc.H | 59 ++-- .../BinaryCollision/ShuffleFisherYates.H | 2 +- .../Particles/Deposition/ChargeDeposition.H | 2 +- .../Particles/Deposition/CurrentDeposition.H | 2 +- .../ElementaryProcess/QEDPairGeneration.H | 2 +- .../ElementaryProcess/QEDPhotonEmission.H | 16 +- Source/Particles/LaserParticleContainer.H | 7 +- .../NamedComponentParticleContainer.H | 52 ++-- Source/Particles/ParticleBoundaryBuffer.cpp | 4 +- Source/Particles/ParticleCreation/SmartCopy.H | 5 +- .../Particles/ParticleCreation/SmartCreate.H | 21 +- .../Particles/ParticleCreation/SmartUtils.H | 8 +- Source/Particles/PhysicalParticleContainer.H | 8 +- .../Particles/PhysicalParticleContainer.cpp | 131 ++++----- Source/Particles/Pusher/GetAndSetPosition.H | 152 +++++++---- .../Particles/Resampling/LevelingThinning.cpp | 5 +- Source/Particles/Sorting/Partition.cpp | 4 +- Source/Particles/Sorting/SortingUtils.H | 41 ++- Source/Particles/Sorting/SortingUtils.cpp | 2 +- Source/Particles/WarpXParticleContainer.H | 27 +- Source/Particles/WarpXParticleContainer.cpp | 90 +++---- .../Particles/ParticleBoundaryBuffer.cpp | 10 +- .../PinnedMemoryParticleContainer.cpp | 2 +- .../Particles/WarpXParticleContainer.cpp | 18 +- Source/Utils/ParticleUtils.H | 7 +- Source/Utils/ParticleUtils.cpp | 26 +- Source/ablastr/particles/IndexHandling.H | 41 --- Source/ablastr/particles/ParticleMoments.H | 25 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 63 files changed, 703 insertions(+), 754 deletions(-) delete mode 100644 Source/ablastr/particles/IndexHandling.H diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 79916c455d1..5e9f43f639d 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.02 && cd - + cd ../amrex && git checkout --detach 296ed40e16ae1877640f5b78e9162dbd4ba1c279 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 ccache -s diff --git a/Docs/source/developers/amrex_basics.rst b/Docs/source/developers/amrex_basics.rst index 577a6547bb5..64ad71af06c 100644 --- a/Docs/source/developers/amrex_basics.rst +++ b/Docs/source/developers/amrex_basics.rst @@ -13,7 +13,7 @@ WarpX is built on the Adaptive Mesh Refinement (AMR) library `AMReX & particle_di // get names of real comps std::map real_comps_map = pc->getParticleComps(); - // WarpXParticleContainer compile-time extra AoS attributes (Real): 0 - // WarpXParticleContainer compile-time extra AoS attributes (int): 0 - // WarpXParticleContainer compile-time extra SoA attributes (Real): PIdx::nattribs // not an efficient search, but N is small... for(int j = 0; j < PIdx::nattribs; ++j) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index d77437fb931..b083e60529f 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -178,8 +178,8 @@ FlushFormatCheckpoint::CheckpointParticles ( Vector real_names; Vector int_names; + // note: positions skipped here, since we reconstruct a plotfile SoA from them real_names.push_back("weight"); - real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); @@ -189,9 +189,12 @@ FlushFormatCheckpoint::CheckpointParticles ( #endif // get the names of the real comps - real_names.resize(pc->NumRealComps()); + // note: skips the mandatory AMREX_SPACEDIM positions for pure SoA + real_names.resize(pc->NumRealComps() - AMREX_SPACEDIM); auto runtime_rnames = pc->getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) { real_names[x.second+PIdx::nattribs] = x.first; } + for (auto const& x : runtime_rnames) { + real_names[x.second + PIdx::nattribs - AMREX_SPACEDIM] = x.first; + } // and the int comps int_names.resize(pc->NumIntComps()); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 970d9a504d2..880e2df01ff 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -355,8 +355,8 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, Vector int_flags; Vector real_flags; + // note: positions skipped here, since we reconstruct a plotfile SoA from them real_names.push_back("weight"); - real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); @@ -366,14 +366,21 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, #endif // get the names of the real comps - real_names.resize(tmp.NumRealComps()); + + // note: skips the mandatory AMREX_SPACEDIM positions for pure SoA + real_names.resize(tmp.NumRealComps() - AMREX_SPACEDIM); auto runtime_rnames = tmp.getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) { real_names[x.second+PIdx::nattribs] = x.first; } + for (auto const& x : runtime_rnames) { + real_names[x.second + PIdx::nattribs - AMREX_SPACEDIM] = x.first; + } // plot any "extra" fields by default real_flags = part_diag.m_plot_flags; real_flags.resize(tmp.NumRealComps(), 1); + // note: skip the mandatory AMREX_SPACEDIM positions for pure SoA + real_flags.erase(real_flags.begin(), real_flags.begin() + AMREX_SPACEDIM); + // and the names int_names.resize(tmp.NumIntComps()); auto runtime_inames = tmp.getParticleRuntimeiComps(); diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index 7ca5e6541d7..a8bb9303fe1 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -160,7 +160,7 @@ MultiParticleContainer::Restart (const std::string& dir) ); } - for (int j = PIdx::nattribs; j < nr; ++j) { + for (int j = PIdx::nattribs-AMREX_SPACEDIM; j < nr; ++j) { const auto& comp_name = real_comp_names[j]; auto current_comp_names = pc->getParticleComps(); auto search = current_comp_names.find(comp_name); diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 9f45392bb0a..24ad0e64ea8 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -431,8 +431,6 @@ void FieldProbe::ComputeDiags (int step) { const auto getPosition = GetParticlePosition(pti); auto setPosition = SetParticlePosition(pti); - const auto& aos = pti.GetArrayOfStructs(); - const auto* AMREX_RESTRICT m_structs = aos().dataPtr(); auto const np = pti.numParticles(); if (update_particles_moving_window) @@ -482,6 +480,8 @@ void FieldProbe::ComputeDiags (int step) ParticleReal* const AMREX_RESTRICT part_Bz = attribs[FieldProbePIdx::Bz].dataPtr(); ParticleReal* const AMREX_RESTRICT part_S = attribs[FieldProbePIdx::S].dataPtr(); + auto * const AMREX_RESTRICT idcpu = pti.GetStructOfArrays().GetIdCPUData().data(); + const auto &xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const std::array &dx = WarpX::CellSize(lev); @@ -556,7 +556,7 @@ void FieldProbe::ComputeDiags (int step) amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); long idx = ip*noutputs; - dvp[idx++] = m_structs[ip].id(); + dvp[idx++] = amrex::ParticleIDWrapper{idcpu[ip]}; // all particles created on IO cpu dvp[idx++] = xp; dvp[idx++] = yp; dvp[idx++] = zp; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H index c85bf8fd541..7d59ade5dc6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H @@ -24,7 +24,14 @@ struct FieldProbePIdx { enum { - Ex = 0, Ey, Ez, +#if !defined (WARPX_DIM_1D_Z) + x, +#endif +#if defined (WARPX_DIM_3D) + y, +#endif + z, + Ex, Ey, Ez, Bx, By, Bz, S, //!< the Poynting vector #ifdef WARPX_DIM_RZ @@ -40,9 +47,14 @@ struct FieldProbePIdx * nattribs tells the particle container to allot 7 SOA values. */ class FieldProbeParticleContainer - : public amrex::ParticleContainer<0, 0, FieldProbePIdx::nattribs> + : public amrex::ParticleContainerPureSoA { public: + static constexpr int NStructReal = 0; + static constexpr int NStructInt = 0; + static constexpr int NReal = FieldProbePIdx::nattribs; + static constexpr int NInt = 0; + FieldProbeParticleContainer (amrex::AmrCore* amr_core); ~FieldProbeParticleContainer() override = default; @@ -52,9 +64,9 @@ public: FieldProbeParticleContainer& operator= ( FieldProbeParticleContainer&& ) = default; //! amrex iterator for our number of attributes - using iterator = amrex::ParIter<0, 0, FieldProbePIdx::nattribs, 0>; + using iterator = amrex::ParIterSoA; //! amrex iterator for our number of attributes (read-only) - using const_iterator = amrex::ParConstIter<0, 0, FieldProbePIdx::nattribs, 0>; + using const_iterator = amrex::ParConstIterSoA; //! similar to WarpXParticleContainer::AddNParticles but does not include u(x,y,z) void AddNParticles (int lev, amrex::Vector const & x, amrex::Vector const & y, amrex::Vector const & z); diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp index 1fd741ddc47..7e7aecb9167 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp @@ -59,7 +59,7 @@ using namespace amrex; FieldProbeParticleContainer::FieldProbeParticleContainer (AmrCore* amr_core) - : ParticleContainer<0, 0, FieldProbePIdx::nattribs>(amr_core->GetParGDB()) + : ParticleContainerPureSoA(amr_core->GetParGDB()) { SetParticleSize(); } @@ -89,33 +89,15 @@ FieldProbeParticleContainer::AddNParticles (int lev, * is then coppied to the permament tile which is stored on the particle * (particle_tile). */ + using PinnedTile = typename ContainerLike::ParticleTileType; - using PinnedTile = ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>; PinnedTile pinned_tile; pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps()); for (int i = 0; i < np; i++) { - ParticleType p; - p.id() = ParticleType::NextID(); - p.cpu() = ParallelDescriptor::MyProc(); -#if defined(WARPX_DIM_3D) - p.pos(0) = x[i]; - p.pos(1) = y[i]; - p.pos(2) = z[i]; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(y); - p.pos(0) = x[i]; - p.pos(1) = z[i]; -#elif defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(x, y); - p.pos(0) = z[i]; -#endif - - // write position, cpu id, and particle id to particle - pinned_tile.push_back(p); + auto & idcpu_data = pinned_tile.GetStructOfArrays().GetIdCPUData(); + idcpu_data.push_back(amrex::SetParticleIDandCPU(ParticleType::NextID(), ParallelDescriptor::MyProc())); } // write Real attributes (SoA) to particle initialized zero @@ -125,7 +107,13 @@ FieldProbeParticleContainer::AddNParticles (int lev, #ifdef WARPX_DIM_RZ pinned_tile.push_back_real(FieldProbePIdx::theta, np, 0.0); #endif - +#if !defined (WARPX_DIM_1D_Z) + pinned_tile.push_back_real(FieldProbePIdx::x, x); +#endif +#if defined (WARPX_DIM_3D) + pinned_tile.push_back_real(FieldProbePIdx::y, y); +#endif + pinned_tile.push_back_real(FieldProbePIdx::z, z); pinned_tile.push_back_real(FieldProbePIdx::Ex, np, 0.0); pinned_tile.push_back_real(FieldProbePIdx::Ey, np, 0.0); pinned_tile.push_back_real(FieldProbePIdx::Ez, np, 0.0); diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 893b00a5f00..b4e07b51982 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -56,8 +56,7 @@ namespace auto const & plev = pc.GetParticles(lev); auto const & ptile = plev.at(box_index); - auto const & aos = ptile.GetArrayOfStructs(); - auto const np = aos.numParticles(); + auto const np = ptile.numParticles(); num_macro_particles += np; } diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 4597dacd9ae..6c904790e15 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -41,7 +41,7 @@ class WarpXParticleCounter { public: using ParticleContainer = typename WarpXParticleContainer::ContainerLike; - using ParticleIter = typename amrex::ParIter<0, 0, PIdx::nattribs, 0, amrex::PinnedArenaAllocator>; + using ParticleIter = typename amrex::ParIterSoA; WarpXParticleCounter (ParticleContainer* pc); [[nodiscard]] unsigned long GetTotalNumParticles () const {return m_Total;} @@ -77,7 +77,7 @@ class WarpXOpenPMDPlot { public: using ParticleContainer = typename WarpXParticleContainer::ContainerLike; - using ParticleIter = typename amrex::ParConstIter<0, 0, PIdx::nattribs, 0, amrex::PinnedArenaAllocator>; + using ParticleIter = typename amrex::ParConstIterSoA; /** Initialize openPMD I/O routines * diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 7cc9f571a4a..39717ef6ec5 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -18,11 +18,9 @@ #include "WarpX.H" #include "OpenPMDHelpFunction.H" -#include #include #include -#include #include #include #include @@ -550,6 +548,13 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { // see openPMD ED-PIC extension for namings // note: an underscore separates the record name from its component // for non-scalar records +#if !defined (WARPX_DIM_1D_Z) + real_names.push_back("position_x"); +#endif +#if defined (WARPX_DIM_3D) + real_names.push_back("position_y"); +#endif + real_names.push_back("position_z"); real_names.push_back("weighting"); real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); @@ -722,77 +727,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, contributed_particles = true; - // get position and particle ID from aos - // note: this implementation iterates the AoS 4x... - // if we flush late as we do now, we can also copy out the data in one go - const auto &aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile - { - // Save positions -#if defined(WARPX_DIM_RZ) - { - const std::shared_ptr z( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (auto i = 0; i < numParticleOnTile; i++) { - z.get()[i] = aos[i].pos(1); // {0: "r", 1: "z"} - } - std::string const positionComponent = "z"; - currSpecies["position"]["z"].storeChunk(z, {offset}, {numParticleOnTile64}); - } - - // reconstruct x and y from polar coordinates r, theta - auto const& soa = pti.GetStructOfArrays(); - amrex::ParticleReal const* theta = soa.GetRealData(PIdx::theta).dataPtr(); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(theta != nullptr, "openPMD: invalid theta pointer."); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(int(soa.GetRealData(PIdx::theta).size()) == numParticleOnTile, - "openPMD: theta and tile size do not match"); - { - const std::shared_ptr< amrex::ParticleReal > x( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - const std::shared_ptr< amrex::ParticleReal > y( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - for (auto i=0; i curr( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (auto i = 0; i < numParticleOnTile; i++) { - curr.get()[i] = aos[i].pos(currDim); - } - std::string const positionComponent = positionComponents[currDim]; - currSpecies["position"][positionComponent].storeChunk(curr, {offset}, - {numParticleOnTile64}); - } -#endif - - // save particle ID after converting it to a globally unique ID - const std::shared_ptr ids( - new uint64_t[numParticleOnTile], - [](uint64_t const *p) { delete[] p; } - ); - for (auto i = 0; i < numParticleOnTile; i++) { - ids.get()[i] = ablastr::particles::localIDtoGlobal(static_cast(aos[i].id()), static_cast(aos[i].cpu())); - } - const auto *const scalar = openPMD::RecordComponent::SCALAR; - currSpecies["id"][scalar].storeChunk(ids, {offset}, {numParticleOnTile64}); - - } - // save "extra" particle properties in AoS and SoA + // save particle properties SaveRealProperty(pti, currSpecies, offset, @@ -893,10 +828,9 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, std::set< std::string > addedRecords; // add meta-data per record only once for (auto idx=0; idxNumRealComps(); idx++) { - auto ii = ParticleContainer::NStructReal + idx; // jump over extra AoS names - if (write_real_comp[ii]) { + if (write_real_comp[idx]) { // handle scalar and non-scalar records by name - const auto [record_name, component_name] = detail::name2openPMD(real_comp_names[ii]); + const auto [record_name, component_name] = detail::name2openPMD(real_comp_names[idx]); auto currRecord = currSpecies[record_name]; // meta data for ED-PIC extension @@ -917,10 +851,9 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, } } for (auto idx=0; idx( numParticleOnTile ); - auto const& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile + auto const numParticleOnTile64 = static_cast(numParticleOnTile); auto const& soa = pti.GetStructOfArrays(); - // first we concatenate the AoS into contiguous arrays - { - // note: WarpX does not yet use extra AoS Real attributes - for( auto idx=0; idx d( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - - for( auto kk=0; kk #include #include #include + namespace ParticleBoundaryProcess { struct NoOp { @@ -25,12 +27,11 @@ struct NoOp { struct Absorb { template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator() (const PData& ptd, int i, + void operator() (PData& ptd, int i, const amrex::RealVect& /*pos*/, const amrex::RealVect& /*normal*/, amrex::RandomEngine const& /*engine*/) const noexcept { - auto& p = ptd.m_aos[i]; - p.id() = -p.id(); + amrex::ParticleIDWrapper{ptd.m_idcpu[i]}.make_invalid(); } }; } diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index d6196c35f44..a175fe23133 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -38,7 +38,7 @@ * passed in to this function as an argument. This function can access the * position at which the particle hit the boundary, and also the associated * normal vector. Particles can be `absorbed` by setting their ids to negative - * to flag them for removal. Likewise, the can be reflected back into the domain + * to flag them for removal. Likewise, they can be reflected back into the domain * by modifying their data appropriately and leaving their ids alone. * * This version operates only at the specified level. @@ -82,7 +82,7 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t * passed in to this function as an argument. This function can access the * position at which the particle hit the boundary, and also the associated * normal vector. Particles can be `absorbed` by setting their ids to negative - * to flag them for removal. Likewise, the can be reflected back into the domain + * to flag them for removal. Likewise, they can be reflected back into the domain * by modifying their data appropriately and leaving their ids alone. * * This version operates over all the levels in the pc. @@ -170,13 +170,12 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t auto& tile = pti.GetParticleTile(); auto ptd = tile.getParticleTileData(); const auto np = tile.numParticles(); - amrex::Particle<0,0> * const particles = tile.GetArrayOfStructs()().data(); auto phi = (*distance_to_eb[lev])[pti].array(); // signed distance function amrex::ParallelForRNG( np, [=] AMREX_GPU_DEVICE (const int ip, amrex::RandomEngine const& engine) noexcept { // skip particles that are already flagged for removal - if (particles[ip].id() < 0) return; + if (!amrex::ParticleIDWrapper{ptd.m_idcpu[ip]}.is_valid()) return; amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 5c90dab25e6..c69f07acdb2 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -72,7 +72,8 @@ class BinaryCollision final // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = amrex::DenseBins; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using index_type = ParticleBins::index_type; @@ -261,9 +262,6 @@ public: const amrex::ParticleReal q1 = species_1.getCharge(); const amrex::ParticleReal m1 = species_1.getMass(); auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); - // Needed to access the particle id - ParticleType * AMREX_RESTRICT - particle_ptr_1 = ptile_1.GetArrayOfStructs()().data(); amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); #if defined WARPX_DIM_1D_Z @@ -371,7 +369,7 @@ public: soa_1, soa_1, product_species_vector, tile_products_data, - particle_ptr_1, particle_ptr_1, m1, m1, + m1, m1, products_mass, p_mask, products_np, copy_species1, copy_species2, p_pair_indices_1, p_pair_indices_2, @@ -403,9 +401,6 @@ public: const amrex::ParticleReal q1 = species_1.getCharge(); const amrex::ParticleReal m1 = species_1.getMass(); auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); - // Needed to access the particle id - ParticleType * AMREX_RESTRICT - particle_ptr_1 = ptile_1.GetArrayOfStructs()().data(); // - Species 2 const auto soa_2 = ptile_2.getParticleTileData(); index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); @@ -413,9 +408,6 @@ public: const amrex::ParticleReal q2 = species_2.getCharge(); const amrex::ParticleReal m2 = species_2.getMass(); auto get_position_2 = GetParticlePosition(ptile_2, getpos_offset); - // Needed to access the particle id - ParticleType * AMREX_RESTRICT - particle_ptr_2 = ptile_2.GetArrayOfStructs()().data(); amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); #if defined WARPX_DIM_1D_Z @@ -535,7 +527,7 @@ public: soa_1, soa_2, product_species_vector, tile_products_data, - particle_ptr_1, particle_ptr_2, m1, m2, + m1, m2, products_mass, p_mask, products_np, copy_species1, copy_species2, p_pair_indices_1, p_pair_indices_2, diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index feb7acf81d3..cfdc36d3c50 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -23,10 +23,13 @@ * \brief This functor performs pairwise Coulomb collision on a single cell by calling the function * ElasticCollisionPerez. It also reads and contains the Coulomb logarithm. */ -class PairWiseCoulombCollisionFunc{ +class PairWiseCoulombCollisionFunc +{ // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H index c1be307b811..ab01eba2c81 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H @@ -38,7 +38,8 @@ class DSMC final // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = amrex::DenseBins; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using index_type = ParticleBins::index_type; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index c1fb7ee7e38..f684b60da78 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -10,6 +10,9 @@ #define SPLIT_AND_SCATTER_FUNC_H_ #include "Particles/Collision/ScatteringProcess.H" +#include "Particles/NamedComponentParticleContainer.H" + +#include /** * \brief Function that performs the particle scattering and injection due @@ -55,8 +58,6 @@ int splitScatteringParticles ( const auto ptile1_data = ptile1.getParticleTileData(); const auto ptile2_data = ptile2.getParticleTileData(); - const Long minus_one_long = -1; - ParallelForRNG(n_total_pairs, [=] AMREX_GPU_DEVICE (int i, RandomEngine const& engine) noexcept { @@ -70,20 +71,35 @@ int splitScatteringParticles ( // starting with the parent particles auto& w1 = ptile1_data.m_rdata[PIdx::w][p_pair_indices_1[i]]; auto& w2 = ptile2_data.m_rdata[PIdx::w][p_pair_indices_2[i]]; + uint64_t* AMREX_RESTRICT idcpu1 = ptile1_data.m_idcpu; + uint64_t* AMREX_RESTRICT idcpu2 = ptile2_data.m_idcpu; + + // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX + // to replace the following lambda. + auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + { +#if defined(AMREX_USE_OMP) +#pragma omp atomic write + idcpu = amrex::ParticleIdCpus::Invalid; +#else + amrex::Gpu::Atomic::Exch( + (unsigned long long *)&idcpu, + (unsigned long long)amrex::ParticleIdCpus::Invalid + ); +#endif + }; // Remove p_pair_reaction_weight[i] from the colliding particles' weights. // If the colliding particle weight decreases to zero, remove particle by // setting its id to -1. Gpu::Atomic::AddNoRet(&w1, -p_pair_reaction_weight[i]); if (w1 <= 0._prt) { - auto& p = ptile1_data.m_aos[p_pair_indices_1[i]]; - p.atomicSetID(minus_one_long); + atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); } Gpu::Atomic::AddNoRet(&w2, -p_pair_reaction_weight[i]); if (w2 <= 0._prt) { - auto& p = ptile2_data.m_aos[p_pair_indices_2[i]]; - p.atomicSetID(minus_one_long); + atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); } // Set the child particle properties appropriately diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index 397536b67bf..b2a2112ca68 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -33,10 +33,13 @@ * creation functor. * This functor also reads and contains the fusion multiplier. */ -class NuclearFusionFunc{ +class NuclearFusionFunc +{ // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; @@ -154,12 +157,13 @@ public: // other species and we need to decrease their weight accordingly. // c1 corresponds to the minimum number of times a particle of species 1 will be paired // with a particle of species 2. Same for c2. - const index_type c1 = amrex::max(NI2/NI1,1u); - const index_type c2 = amrex::max(NI1/NI2,1u); + // index_type(1): https://github.com/AMReX-Codes/amrex/pull/3684 + const index_type c1 = amrex::max(NI2/NI1, index_type(1)); + const index_type c2 = amrex::max(NI1/NI2, index_type(1)); // multiplier ratio to take into account unsampled pairs const auto multiplier_ratio = static_cast( - (m_isSameSpecies)?(2u*max_N - 1):(max_N)); + m_isSameSpecies ? 2*max_N - 1 : max_N); #if (defined WARPX_DIM_RZ) amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 7b29267ec32..0b51d6b4b61 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -22,10 +22,12 @@ namespace { // Define shortcuts for frequently-used type names - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; - using index_type = ParticleBins::index_type; + using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; /** * \brief This function initializes the momentum of the alpha particles produced from diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H index be3f5b2d957..52e9db8aa94 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H @@ -24,7 +24,9 @@ namespace { // Define shortcuts for frequently-used type names using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; /** diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index dc830b477df..7a2853e3db5 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -30,13 +30,15 @@ * \brief This functor creates particles produced from a binary collision and sets their initial * properties (position, momentum, weight). */ -class ParticleCreationFunc{ +class ParticleCreationFunc +{ // Define shortcuts for frequently-used type names - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = amrex::DenseBins; - using index_type = ParticleBins::index_type; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; + using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; public: /** @@ -69,12 +71,6 @@ public: * @param[in, out] soa_1 struct of array data of the first colliding particle species * @param[in, out] soa_2 struct of array data of the second colliding particle species * @param[out] tile_products array containing tile data of the product particles. - * @param[out] particle_ptr_1 pointer to data of the first colliding particle species. Is - * needed to set the id of a particle to -1 in order to delete it when its weight - * reaches 0. - * @param[out] particle_ptr_2 pointer to data of the second colliding particle species. Is - * needed to set the id of a particle to -1 in order to delete it when its weight - * reaches 0. * @param[in] m1 mass of the first colliding particle species * @param[in] m2 mass of the second colliding particle species * @param[in] products_mass array storing the mass of product particles @@ -102,7 +98,6 @@ public: const SoaData_type& soa_1, const SoaData_type& soa_2, const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, - ParticleType* particle_ptr_1, ParticleType* particle_ptr_2, const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, const amrex::Vector& products_mass, const index_type* AMREX_RESTRICT p_mask, @@ -137,6 +132,8 @@ public: amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; + uint64_t* AMREX_RESTRICT idcpu2 = soa_2.m_idcpu; // Create necessary GPU vectors, that will be used in the kernel below amrex::Vector soa_products; @@ -205,16 +202,31 @@ public: amrex::Gpu::Atomic::AddNoRet(&w2[p_pair_indices_2[i]], -p_pair_reaction_weight[i]); + // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX + // to replace the following lambda. + auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + { +#if defined(AMREX_USE_OMP) +#pragma omp atomic write + idcpu = amrex::ParticleIdCpus::Invalid; +#else + amrex::Gpu::Atomic::Exch( + (unsigned long long *)&idcpu, + (unsigned long long)amrex::ParticleIdCpus::Invalid + ); +#endif + }; + // If the colliding particle weight decreases to zero, remove particle by // setting its id to -1 - constexpr amrex::Long minus_one_long = -1; if (w1[p_pair_indices_1[i]] <= amrex::ParticleReal(0.)) { - particle_ptr_1[p_pair_indices_1[i]].atomicSetID(minus_one_long); + atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); + } if (w2[p_pair_indices_2[i]] <= amrex::ParticleReal(0.)) { - particle_ptr_2[p_pair_indices_2[i]].atomicSetID(minus_one_long); + atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); } // Initialize the product particles' momentum, using a function depending on the @@ -294,12 +306,14 @@ private: * \brief This class does nothing and is used as second template parameter for binary collisions * that do not create particles. */ -class NoParticleCreationFunc{ - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = amrex::DenseBins; - using index_type = ParticleBins::index_type; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; +class NoParticleCreationFunc +{ + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; + using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; public: NoParticleCreationFunc () = default; @@ -313,7 +327,6 @@ public: const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, amrex::Vector& /*pc_products*/, ParticleTileType** /*tile_products*/, - ParticleType* /*particle_ptr_1*/, ParticleType* /*particle_ptr_2*/, const amrex::ParticleReal& /*m1*/, const amrex::ParticleReal& /*m2*/, const amrex::Vector& /*products_mass*/, const index_type* /*p_mask*/, const amrex::Vector& /*products_np*/, diff --git a/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H b/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H index 42259512b0d..3b8f72f4b84 100644 --- a/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H +++ b/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H @@ -12,7 +12,7 @@ /* \brief Shuffle array according to Fisher-Yates algorithm. * Only shuffle the part between is <= i < ie, n = ie-is. * T_index shall be - * amrex::DenseBins::index_type + * amrex::DenseBins::index_type */ template diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index d0db678dfda..d0822789015 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -252,7 +252,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio const int n_rz_azimuthal_modes, amrex::Real* cost, const long load_balance_costs_update_algo, - const amrex::DenseBins& a_bins, + const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, const amrex::IntVect& a_tbox_max_size, diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 18df09c3b43..2252a63fd07 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -592,7 +592,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, int n_rz_azimuthal_modes, amrex::Real* cost, long load_balance_costs_update_algo, - const amrex::DenseBins& a_bins, + const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, const amrex::IntVect& a_tbox_max_size) diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index 5abc9282d4f..fb723f0b79a 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -167,7 +167,7 @@ public: p_ux, p_uy, p_uz, engine); - src.m_aos[i_src].id() = -1; //destroy photon after pair generation + src.m_idcpu[i_src] = amrex::ParticleIdCpus::Invalid; // destroy photon after pair generation } private: diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index 8ba5c63ad57..567b260d0e4 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -237,12 +238,11 @@ void cleanLowEnergyPhotons( const int old_size, const int num_added, const amrex::ParticleReal energy_threshold) { - auto pp = ptile.GetArrayOfStructs()().data() + old_size; - - const auto& soa = ptile.GetStructOfArrays(); + auto& soa = ptile.GetStructOfArrays(); + auto p_idcpu = soa.GetIdCPUData().data() + old_size; const auto p_ux = soa.GetRealData(PIdx::ux).data() + old_size; - const auto p_uy = soa.GetRealData(PIdx::uy).data() + old_size; - const auto p_uz = soa.GetRealData(PIdx::uz).data() + old_size; + const auto p_uy = soa.GetRealData(PIdx::uy).data() + old_size; + const auto p_uz = soa.GetRealData(PIdx::uz).data() + old_size; //The square of the energy threshold const auto energy_threshold2 = std::max( @@ -251,8 +251,6 @@ void cleanLowEnergyPhotons( amrex::ParallelFor(num_added, [=] AMREX_GPU_DEVICE (int ip) noexcept { - auto& p = pp[ip]; - const auto ux = p_ux[ip]; const auto uy = p_uy[ip]; const auto uz = p_uz[ip]; @@ -262,8 +260,8 @@ void cleanLowEnergyPhotons( constexpr amrex::ParticleReal me_c = PhysConst::m_e*PhysConst::c; const auto phot_energy2 = (ux*ux + uy*uy + uz*uz)*me_c*me_c; - if (phot_energy2 < energy_threshold2){ - p.id() = - 1; + if (phot_energy2 < energy_threshold2) { + p_idcpu[ip] = amrex::ParticleIdCpus::Invalid; } }); } diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index e6fa308431c..fac94ff20a3 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -56,10 +56,9 @@ public: * \brief Method to initialize runtime attributes. Does nothing for LaserParticleContainer. */ void DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& /*pinned_tile*/, - const int /*n_external_attr_real*/, - const int /*n_external_attr_int*/) final {} + typename ContainerLike::ParticleTileType& /*pinned_tile*/, + int /*n_external_attr_real*/, + int /*n_external_attr_int*/) final {} void ReadHeader (std::istream& is) final; diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index 3be0886425d..e7a7a20fad5 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -18,24 +18,39 @@ #include -/** Particle Attributes stored in amrex::ParticleContainer's struct of array +/** Real Particle Attributes stored in amrex::ParticleContainer's struct of array */ struct PIdx { enum { - w = 0, ///< weight +#if !defined (WARPX_DIM_1D_Z) + x, +#endif +#if defined (WARPX_DIM_3D) + y, +#endif + z, + w, ///< weight ux, uy, uz, #ifdef WARPX_DIM_RZ theta, ///< RZ needs all three position components #endif - nattribs ///< number of attributes + nattribs ///< number of compile-time attributes + }; +}; + +/** Integer Particle Attributes stored in amrex::ParticleContainer's struct of array + */ +struct PIdxInt +{ + enum { + nattribs ///< number of compile-time attributes }; }; /** Particle Container class that allows to add/access particle components * with a name (string) instead of doing so with an integer index. - * (The "components" are all the particle quantities - except those - * that are stored in an AoS by amrex, i.e. the particle positions and ID) + * (The "components" are all the particle amrex::Real quantities.) * * This is done by storing maps that give the index of the component * that corresponds to a given string. @@ -45,11 +60,11 @@ struct PIdx */ template class T_Allocator=amrex::DefaultAllocator> class NamedComponentParticleContainer : -public amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator> +public amrex::ParticleContainerPureSoA { public: /** Construct an empty NamedComponentParticleContainer **/ - NamedComponentParticleContainer () : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>() {} + NamedComponentParticleContainer () : amrex::ParticleContainerPureSoA() {} /** Construct a NamedComponentParticleContainer from an AmrParGDB object * @@ -61,8 +76,15 @@ public: * AMR hierarchy. Usually, this is generated by an AmrCore or AmrLevel object. */ NamedComponentParticleContainer (amrex::AmrParGDB* amr_pgdb) - : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>(amr_pgdb) { + : amrex::ParticleContainerPureSoA(amr_pgdb) { // build up the map of string names to particle component numbers +#if !defined (WARPX_DIM_1D_Z) + particle_comps["x"] = PIdx::x; +#endif +#if defined (WARPX_DIM_3D) + particle_comps["y"] = PIdx::y; +#endif + particle_comps["z"] = PIdx::z; particle_comps["w"] = PIdx::w; particle_comps["ux"] = PIdx::ux; particle_comps["uy"] = PIdx::uy; @@ -85,12 +107,12 @@ public: * @param p_ricomps name-to-index map for run-time integer components */ NamedComponentParticleContainer( - amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator> && pc, + amrex::ParticleContainerPureSoA && pc, std::map p_comps, std::map p_icomps, std::map p_rcomps, std::map p_ricomps) - : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>(std::move(pc)), + : amrex::ParticleContainerPureSoA(std::move(pc)), particle_comps(std::move(p_comps)), particle_icomps(std::move(p_icomps)), particle_runtime_comps(std::move(p_rcomps)), @@ -118,7 +140,7 @@ public: NamedComponentParticleContainer make_alike () const { auto tmp = NamedComponentParticleContainer( - amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::template make_alike(), + amrex::ParticleContainerPureSoA::template make_alike(), particle_comps, particle_icomps, particle_runtime_comps, @@ -127,10 +149,10 @@ public: return tmp; } - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::NumRealComps; - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::NumIntComps; - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::AddRealComp; - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::AddIntComp; + using amrex::ParticleContainerPureSoA::NumRealComps; + using amrex::ParticleContainerPureSoA::NumIntComps; + using amrex::ParticleContainerPureSoA::AddRealComp; + using amrex::ParticleContainerPureSoA::AddIntComp; /** Allocate a new run-time real component * diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 54c4396379d..88304bd8a9c 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -50,7 +50,7 @@ struct CopyAndTimestamp { void operator() (const DstData& dst, const SrcData& src, int src_i, int dst_i) const noexcept { - dst.m_aos[dst_i] = src.m_aos[src_i]; + dst.m_idcpu[dst_i] = src.m_idcpu[src_i]; for (int j = 0; j < SrcData::NAR; ++j) { dst.m_rdata[j][dst_i] = src.m_rdata[j][src_i]; } @@ -222,7 +222,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles"); - using PIter = amrex::ParConstIter<0,0,PIdx::nattribs>; + using PIter = amrex::ParConstIterSoA; const auto& warpx_instance = WarpX::GetInstance(); const amrex::Geometry& geom = warpx_instance.Geom(0); auto plo = geom.ProbLoArray(); diff --git a/Source/Particles/ParticleCreation/SmartCopy.H b/Source/Particles/ParticleCreation/SmartCopy.H index 2c04baa18bb..6a6ceb3d290 100644 --- a/Source/Particles/ParticleCreation/SmartCopy.H +++ b/Source/Particles/ParticleCreation/SmartCopy.H @@ -26,7 +26,7 @@ * type. Second, if a given component name is found in both the src * and the dst, then the src value is copied. * - * Particle structs - positions and id numbers - are always copied. + * Particle positions and id numbers are always copied. * * You don't create this directly - use the SmartCopyFactory object below. */ @@ -48,9 +48,6 @@ struct SmartCopy void operator() (DstData& dst, const SrcData& src, int i_src, int i_dst, amrex::RandomEngine const& engine) const noexcept { - // the particle struct is always copied over - dst.m_aos[i_dst] = src.m_aos[i_src]; - // initialize the real components for (int j = 0; j < DstData::NAR; ++j) { dst.m_rdata[j][i_dst] = initializeRealValue(m_policy_real[j], engine); diff --git a/Source/Particles/ParticleCreation/SmartCreate.H b/Source/Particles/ParticleCreation/SmartCreate.H index 67d7767a5d3..b4f25d5daad 100644 --- a/Source/Particles/ParticleCreation/SmartCreate.H +++ b/Source/Particles/ParticleCreation/SmartCreate.H @@ -14,6 +14,8 @@ #include #include #include +#include +#include /** * \brief This is a functor for performing a "smart create" that works @@ -47,23 +49,22 @@ struct SmartCreate const int id = 0) const noexcept { #if defined(WARPX_DIM_3D) - prt.m_aos[i_prt].pos(0) = x; - prt.m_aos[i_prt].pos(1) = y; - prt.m_aos[i_prt].pos(2) = z; + prt.m_rdata[PIdx::x][i_prt] = x; + prt.m_rdata[PIdx::y][i_prt] = y; + prt.m_rdata[PIdx::z][i_prt] = z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - prt.m_aos[i_prt].pos(0) = x; - prt.m_aos[i_prt].pos(1) = z; + prt.m_rdata[PIdx::x][i_prt] = x; + prt.m_rdata[PIdx::z][i_prt] = z; amrex::ignore_unused(y); #else - prt.m_aos[i_prt].pos(0) = z; + prt.m_rdata[PIdx::z][i_prt] = z; amrex::ignore_unused(x,y); #endif - prt.m_aos[i_prt].cpu() = cpu; - prt.m_aos[i_prt].id() = id; + prt.m_idcpu[i_prt] = amrex::SetParticleIDandCPU(id, cpu); - // initialize the real components - for (int j = 0; j < PartData::NAR; ++j) { + // initialize the real components after position + for (int j = AMREX_SPACEDIM; j < PartData::NAR; ++j) { prt.m_rdata[j][i_prt] = initializeRealValue(m_policy_real[j], engine); } for (int j = 0; j < prt.m_num_runtime_real; ++j) { diff --git a/Source/Particles/ParticleCreation/SmartUtils.H b/Source/Particles/ParticleCreation/SmartUtils.H index 732a12bb729..f84734308fb 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.H +++ b/Source/Particles/ParticleCreation/SmartUtils.H @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -60,12 +61,11 @@ void setNewParticleIDs (PTile& ptile, int old_size, int num_added) } const int cpuid = amrex::ParallelDescriptor::MyProc(); - auto pp = ptile.GetArrayOfStructs()().data() + old_size; + auto ptd = ptile.getParticleTileData(); amrex::ParallelFor(num_added, [=] AMREX_GPU_DEVICE (int ip) noexcept { - auto& p = pp[ip]; - p.id() = pid+ip; - p.cpu() = cpuid; + auto const new_id = ip + old_size; + ptd.m_idcpu[new_id] = amrex::SetParticleIDandCPU(pid+ip, cpuid); }); } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index a12ae75f629..edf91a84526 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -268,11 +268,9 @@ public: * @param[in] engine the random engine, used in initialization of QED optical depths */ void DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>& pinned_tile, - int n_external_attr_real, - int n_external_attr_int) final; + typename ContainerLike::ParticleTileType& pinned_tile, + int n_external_attr_real, + int n_external_attr_int) final; /** * \brief Apply NCI Godfrey filter to all components of E and B before gather diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 929c3c26649..08c784709fa 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -198,8 +198,8 @@ namespace * and avoid any possible undefined behavior before the next call to redistribute) and sets * the particle id to -1 so that it can be effectively deleted. * - * \param p particle aos data - * \param pa particle soa data + * \param idcpu particle id soa data + * \param pa particle real soa data * \param ip index for soa data * \param do_field_ionization whether species has ionization * \param pi ionization level data @@ -210,20 +210,21 @@ namespace */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void ZeroInitializeAndSetNegativeID ( - ParticleType& p, const GpuArray& pa, long& ip, + uint64_t * AMREX_RESTRICT idcpu, + const GpuArray& pa, long& ip, const bool& do_field_ionization, int* pi #ifdef WARPX_QED - ,const bool& has_quantum_sync, amrex::ParticleReal* p_optical_depth_QSR - ,const bool& has_breit_wheeler, amrex::ParticleReal* p_optical_depth_BW + ,const bool& has_quantum_sync, amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR + ,const bool& has_breit_wheeler, amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_BW #endif ) noexcept { - p.pos(0) = 0._rt; + pa[PIdx::z][ip] = 0._rt; #if (AMREX_SPACEDIM >= 2) - p.pos(1) = 0._rt; + pa[PIdx::x][ip] = 0._rt; #endif #if defined(WARPX_DIM_3D) - p.pos(2) = 0._rt; + pa[PIdx::y][ip] = 0._rt; #endif pa[PIdx::w ][ip] = 0._rt; pa[PIdx::ux][ip] = 0._rt; @@ -238,7 +239,7 @@ namespace if (has_breit_wheeler) {p_optical_depth_BW[ip] = 0._rt;} #endif - p.id() = -1; + idcpu[ip] = amrex::ParticleIdCpus::Invalid; } } @@ -780,11 +781,9 @@ PhysicalParticleContainer::AddPlasmaFromFile(PlasmaInjector & plasma_injector, void PhysicalParticleContainer::DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>& pinned_tile, - const int n_external_attr_real, - const int n_external_attr_int) + typename ContainerLike::ParticleTileType& pinned_tile, + int n_external_attr_real, + int n_external_attr_int) { ParticleCreation::DefaultInitializeRuntimeAttributes(pinned_tile, n_external_attr_real, n_external_attr_int, @@ -1084,7 +1083,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - Long pid; + int pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1093,7 +1092,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid + max_new_particles) < LastParticleID, + static_cast(pid) + static_cast(max_new_particles) < LongParticleIds::LastParticleID, "ERROR: overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); @@ -1104,16 +1103,16 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int DefineAndReturnParticleTile(lev, grid_id, tile_id); } - auto old_size = particle_tile.GetArrayOfStructs().size(); + auto old_size = particle_tile.size(); auto new_size = old_size + max_new_particles; particle_tile.resize(new_size); - ParticleType* pp = particle_tile.GetArrayOfStructs()().data() + old_size; auto& soa = particle_tile.GetStructOfArrays(); GpuArray pa; for (int ia = 0; ia < PIdx::nattribs; ++ia) { pa[ia] = soa.GetRealData(ia).data() + old_size; } + uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); @@ -1226,9 +1225,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int for (int i_part = 0; i_part < pcounts[index]; ++i_part) { long ip = poffset[index] + i_part; - ParticleType& p = pp[ip]; - p.id() = pid+ip; - p.cpu() = cpuid; + pa_idcpu[ip] = amrex::SetParticleIDandCPU(pid+ip, cpuid); const XDim3 r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? // In the refined injection region: use refinement ratio `lrrfac` inj_pos->getPositionUnitBox(i_part, lrrfac, engine) : @@ -1238,7 +1235,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int #if defined(WARPX_DIM_3D) if (!tile_realbox.contains(XDim3{pos.x,pos.y,pos.z})) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1249,7 +1246,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(k); if (!tile_realbox.contains(XDim3{pos.x,pos.z,0.0_rt})) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1260,7 +1257,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int #else amrex::ignore_unused(j,k); if (!tile_realbox.contains(XDim3{pos.z,0.0_rt,0.0_rt})) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1299,7 +1296,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const Real z0 = applyBallisticCorrection(pos, inj_mom, gamma_boost, beta_boost, t); if (!inj_pos->insideBounds(xb, yb, z0)) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1313,7 +1310,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // Remove particle if density below threshold if ( dens < density_min ){ - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1331,7 +1328,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // If the particle is not within the lab-frame zmin, zmax, etc. // go to the next generated particle. if (!inj_pos->insideBounds(xb, yb, z0_lab)) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1343,7 +1340,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int dens = inj_rho->getDensity(pos.x, pos.y, z0_lab); // Remove particle if density below threshold if ( dens < density_min ){ - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1410,17 +1407,17 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int pa[PIdx::uz][ip] = u.z; #if defined(WARPX_DIM_3D) - p.pos(0) = pos.x; - p.pos(1) = pos.y; - p.pos(2) = pos.z; + pa[PIdx::x][ip] = pos.x; + pa[PIdx::y][ip] = pos.y; + pa[PIdx::z][ip] = pos.z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) #ifdef WARPX_DIM_RZ pa[PIdx::theta][ip] = theta; #endif - p.pos(0) = xb; - p.pos(1) = pos.z; + pa[PIdx::x][ip] = xb; + pa[PIdx::z][ip] = pos.z; #else - p.pos(0) = pos.z; + pa[PIdx::z][ip] = pos.z; #endif } }); @@ -1645,7 +1642,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - Long pid; + int pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1654,23 +1651,23 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid + max_new_particles) < LastParticleID, + static_cast(pid) + static_cast(max_new_particles) < LongParticleIds::LastParticleID, "overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); auto& particle_tile = tmp_pc.DefineAndReturnParticleTile(0, grid_id, tile_id); - auto old_size = particle_tile.GetArrayOfStructs().size(); + auto old_size = particle_tile.size(); auto new_size = old_size + max_new_particles; particle_tile.resize(new_size); - ParticleType* pp = particle_tile.GetArrayOfStructs()().data() + old_size; auto& soa = particle_tile.GetStructOfArrays(); GpuArray pa; for (int ia = 0; ia < PIdx::nattribs; ++ia) { pa[ia] = soa.GetRealData(ia).data() + old_size; } + uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); @@ -1768,9 +1765,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, for (int i_part = 0; i_part < pcounts[index]; ++i_part) { const long ip = poffset[index] + i_part; - ParticleType& p = pp[ip]; - p.id() = pid+ip; - p.cpu() = cpuid; + pa_idcpu[ip] = amrex::SetParticleIDandCPU(pid+ip, cpuid); // This assumes the flux_pos is of type InjectorPositionRandomPlane const XDim3 r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? @@ -1795,19 +1790,19 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // the particles will be within the domain. #if defined(WARPX_DIM_3D) if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.x,ppos.y,ppos.z})) { - p.id() = -1; + pa_idcpu[ip] = amrex::ParticleIdCpus::Invalid; continue; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(k); if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.x,ppos.z,0.0_prt})) { - p.id() = -1; + pa_idcpu[ip] = amrex::ParticleIdCpus::Invalid; continue; } #else amrex::ignore_unused(j,k); if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.z,0.0_prt,0.0_prt})) { - p.id() = -1; + pa_idcpu[ip] = amrex::ParticleIdCpus::Invalid; continue; } #endif @@ -1815,7 +1810,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // If the particle's initial position is not within or on the species's // xmin, xmax, ymin, ymax, zmin, zmax, go to the next generated particle. if (!flux_pos->insideBoundsInclusive(ppos.x, ppos.y, ppos.z)) { - p.id() = -1; + pa_idcpu[ip] = amrex::ParticleIdCpus::Invalid; continue; } @@ -1848,8 +1843,8 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, #endif Real flux = inj_flux->getFlux(ppos.x, ppos.y, ppos.z, t); // Remove particle if flux is negative or 0 - if ( flux <=0 ){ - p.id() = -1; + if (flux <= 0) { + pa_idcpu[ip] = amrex::ParticleIdCpus::Invalid; continue; } @@ -1858,7 +1853,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, } #ifdef WARPX_QED - if(loc_has_quantum_sync){ + if (loc_has_quantum_sync) { p_optical_depth_QSR[ip] = quantum_sync_get_opt(engine); } @@ -1908,18 +1903,18 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, UpdatePosition(ppos.x, ppos.y, ppos.z, pu.x, pu.y, pu.z, t_fract); #if defined(WARPX_DIM_3D) - p.pos(0) = ppos.x; - p.pos(1) = ppos.y; - p.pos(2) = ppos.z; + pa[PIdx::x][ip] = ppos.x; + pa[PIdx::y][ip] = ppos.y; + pa[PIdx::z][ip] = ppos.z; #elif defined(WARPX_DIM_RZ) pa[PIdx::theta][ip] = std::atan2(ppos.y, ppos.x); - p.pos(0) = std::sqrt(ppos.x*ppos.x + ppos.y*ppos.y); - p.pos(1) = ppos.z; + pa[PIdx::x][ip] = std::sqrt(ppos.x*ppos.x + ppos.y*ppos.y); + pa[PIdx::z][ip] = ppos.z; #elif defined(WARPX_DIM_XZ) - p.pos(0) = ppos.x; - p.pos(1) = ppos.z; + pa[PIdx::x][ip] = ppos.x; + pa[PIdx::z][ip] = ppos.z; #else - p.pos(0) = ppos.z; + pa[PIdx::z][ip] = ppos.z; #endif } }); @@ -2342,20 +2337,22 @@ PhysicalParticleContainer::SplitParticles (int lev) split_offset[1] /= ppc_nd[1]; split_offset[2] /= ppc_nd[2]; } - // particle Array Of Structs data - auto& particles = pti.GetArrayOfStructs(); // particle Struct Of Arrays data auto& attribs = pti.GetAttribs(); auto& wp = attribs[PIdx::w ]; auto& uxp = attribs[PIdx::ux]; auto& uyp = attribs[PIdx::uy]; auto& uzp = attribs[PIdx::uz]; + + ParticleTileType& ptile = ParticlesAt(lev, pti); + auto& soa = ptile.GetStructOfArrays(); + uint64_t * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); + const long np = pti.numParticles(); for(int i=0; i> attr_int; pctmp_split.AddNParticles(lev, np_split_to_add, - xp, yp, zp, uxp, uyp, uzp, - 1, attr, + xp, + yp, + zp, + uxp, + uyp, + uzp, + 1, + attr, 0, attr_int, - 1, NoSplitParticleID); + 1, LongParticleIds::NoSplitParticleID); // Copy particles from tmp to current particle container constexpr bool local_flag = true; addParticles(pctmp_split,local_flag); diff --git a/Source/Particles/Pusher/GetAndSetPosition.H b/Source/Particles/Pusher/GetAndSetPosition.H index e4477a2a60d..44641557756 100644 --- a/Source/Particles/Pusher/GetAndSetPosition.H +++ b/Source/Particles/Pusher/GetAndSetPosition.H @@ -30,24 +30,26 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, amrex::ParticleReal& y, amrex::ParticleReal& z) noexcept { -#ifdef WARPX_DIM_RZ - const amrex::ParticleReal theta = p.rdata(T_PIdx::theta); - const amrex::ParticleReal r = p.pos(0); + using namespace amrex::literals; + +#if defined(WARPX_DIM_RZ) + amrex::ParticleReal const theta = p.rdata(T_PIdx::theta); + amrex::ParticleReal const r = p.pos(T_PIdx::x); x = r*std::cos(theta); y = r*std::sin(theta); - z = p.pos(1); -#elif WARPX_DIM_3D - x = p.pos(0); - y = p.pos(1); - z = p.pos(2); -#elif WARPX_DIM_XZ - x = p.pos(0); - y = amrex::ParticleReal(0.0); - z = p.pos(1); + z = p.pos(PIdx::z); +#elif defined(WARPX_DIM_3D) + x = p.pos(PIdx::x); + y = p.pos(PIdx::y); + z = p.pos(PIdx::z); +#elif defined(WARPX_DIM_XZ) + x = p.pos(PIdx::x); + y = 0_prt; + z = p.pos(PIdx::z); #else - x = amrex::ParticleReal(0.0); - y = amrex::ParticleReal(0.0); - z = p.pos(0); + x = 0_prt; + y = 0_prt; + z = p.pos(PIdx::z); #endif } @@ -59,10 +61,19 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, template struct GetParticlePosition { - using PType = WarpXParticleContainer::ParticleType; using RType = amrex::ParticleReal; - const PType* AMREX_RESTRICT m_structs = nullptr; +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + const RType* AMREX_RESTRICT m_x = nullptr; + const RType* AMREX_RESTRICT m_z = nullptr; +#elif defined(WARPX_DIM_3D) + const RType* AMREX_RESTRICT m_x = nullptr; + const RType* AMREX_RESTRICT m_y = nullptr; + const RType* AMREX_RESTRICT m_z = nullptr; +#elif defined(WARPX_DIM_1D_Z) + const RType* AMREX_RESTRICT m_z = nullptr; +#endif + #if defined(WARPX_DIM_RZ) const RType* m_theta = nullptr; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) @@ -84,10 +95,19 @@ struct GetParticlePosition template GetParticlePosition (const ptiType& a_pti, long a_offset = 0) noexcept { - const auto& aos = a_pti.GetArrayOfStructs(); - m_structs = aos().dataPtr() + a_offset; -#if defined(WARPX_DIM_RZ) const auto& soa = a_pti.GetStructOfArrays(); + +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_3D) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_y = soa.GetRealData(PIdx::y).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_1D_Z) + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#endif +#if defined(WARPX_DIM_RZ) m_theta = soa.GetRealData(T_PIdx::theta).dataPtr() + a_offset; #endif } @@ -98,24 +118,23 @@ struct GetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void operator() (const long i, RType& x, RType& y, RType& z) const noexcept { - const PType& p = m_structs[i]; #ifdef WARPX_DIM_RZ - const RType r = p.pos(0); + RType const r = m_x[i]; x = r*std::cos(m_theta[i]); y = r*std::sin(m_theta[i]); - z = p.pos(1); + z = m_z[i]; #elif WARPX_DIM_3D - x = p.pos(0); - y = p.pos(1); - z = p.pos(2); + x = m_x[i]; + y = m_y[i]; + z = m_z[i]; #elif WARPX_DIM_XZ - x = p.pos(0); + x = m_x[i]; y = m_y_default; - z = p.pos(1); + z = m_z[i]; #else x = m_x_default; y = m_y_default; - z = p.pos(0); + z = m_z[i]; #endif } @@ -127,23 +146,22 @@ struct GetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void AsStored (const long i, RType& x, RType& y, RType& z) const noexcept { - const PType& p = m_structs[i]; #ifdef WARPX_DIM_RZ - x = p.pos(0); + x = m_x[i]; y = m_theta[i]; - z = p.pos(1); + z = m_z[i]; #elif WARPX_DIM_3D - x = p.pos(0); - y = p.pos(1); - z = p.pos(2); + x = m_x[i]; + y = m_y[i]; + z = m_z[i]; #elif WARPX_DIM_XZ - x = p.pos(0); + x = m_x[i]; y = m_y_default; - z = p.pos(1); + z = m_z[i]; #else x = m_x_default; y = m_y_default; - z = p.pos(0); + z = m_z[i]; #endif } }; @@ -158,10 +176,18 @@ struct GetParticlePosition template struct SetParticlePosition { - using PType = WarpXParticleContainer::ParticleType; using RType = amrex::ParticleReal; - PType* AMREX_RESTRICT m_structs; +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + RType* AMREX_RESTRICT m_x; + RType* AMREX_RESTRICT m_z; +#elif defined(WARPX_DIM_3D) + RType* AMREX_RESTRICT m_x; + RType* AMREX_RESTRICT m_y; + RType* AMREX_RESTRICT m_z; +#elif defined(WARPX_DIM_1D_Z) + RType* AMREX_RESTRICT m_z; +#endif #if defined(WARPX_DIM_RZ) RType* AMREX_RESTRICT m_theta; #endif @@ -169,10 +195,18 @@ struct SetParticlePosition template SetParticlePosition (const ptiType& a_pti, long a_offset = 0) noexcept { - auto& aos = a_pti.GetArrayOfStructs(); - m_structs = aos().dataPtr() + a_offset; -#if defined(WARPX_DIM_RZ) auto& soa = a_pti.GetStructOfArrays(); +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_3D) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_y = soa.GetRealData(PIdx::y).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_1D_Z) + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#endif +#if defined(WARPX_DIM_RZ) m_theta = soa.GetRealData(T_PIdx::theta).dataPtr() + a_offset; #endif } @@ -190,17 +224,17 @@ struct SetParticlePosition #endif #ifdef WARPX_DIM_RZ m_theta[i] = std::atan2(y, x); - m_structs[i].pos(0) = std::sqrt(x*x + y*y); - m_structs[i].pos(1) = z; + m_x[i] = std::sqrt(x*x + y*y); + m_z[i] = z; #elif WARPX_DIM_3D - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = y; - m_structs[i].pos(2) = z; + m_x[i] = x; + m_y[i] = y; + m_z[i] = z; #elif WARPX_DIM_XZ - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = z; + m_x[i] = x; + m_z[i] = z; #else - m_structs[i].pos(0) = z; + m_z[i] = z; #endif } @@ -218,18 +252,18 @@ struct SetParticlePosition amrex::ignore_unused(x,y); #endif #ifdef WARPX_DIM_RZ - m_structs[i].pos(0) = x; + m_x[i] = x; m_theta[i] = y; - m_structs[i].pos(1) = z; + m_z[i] = z; #elif WARPX_DIM_3D - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = y; - m_structs[i].pos(2) = z; + m_x[i] = x; + m_y[i] = y; + m_z[i] = z; #elif WARPX_DIM_XZ - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = z; + m_x[i] = x; + m_z[i] = z; #else - m_structs[i].pos(0) = z; + m_z[i] = z; #endif } }; diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index 680e33ebe6a..5dc6a458f97 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -60,8 +60,7 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, auto& ptile = pc->ParticlesAt(lev, pti); auto& soa = ptile.GetStructOfArrays(); amrex::ParticleReal * const AMREX_RESTRICT w = soa.GetRealData(PIdx::w).data(); - WarpXParticleContainer::ParticleType * const AMREX_RESTRICT - particle_ptr = ptile.GetArrayOfStructs()().data(); + auto * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); // Using this function means that we must loop over the cells in the ParallelFor. In the case // of the leveling thinning algorithm, it would have possibly been more natural and more @@ -114,7 +113,7 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, // Remove particle with probability 1 - particle_weight/level_weight if (random_number > w[indices[i]]/level_weight) { - particle_ptr[indices[i]].id() = -1; + idcpu[indices[i]] = amrex::ParticleIdCpus::Invalid; } // Set particle weight to level weight otherwise else diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp index 58511cfd5e7..58e3450f47d 100644 --- a/Source/Particles/Sorting/Partition.cpp +++ b/Source/Particles/Sorting/Partition.cpp @@ -61,7 +61,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // Initialize temporary arrays Gpu::DeviceVector inexflag; inexflag.resize(np); - Gpu::DeviceVector pid; + Gpu::DeviceVector pid; pid.resize(np); // First, partition particles into the larger buffer @@ -109,7 +109,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // - For each particle in the large buffer, find whether it is in // the smaller buffer, by looking up the mask. Store the answer in `inexflag`. amrex::ParallelFor( np - n_fine, - fillBufferFlagRemainingParticles(pti, bmasks, inexflag, Geom(lev), pid, n_fine) ); + fillBufferFlagRemainingParticles(pti, bmasks, inexflag, Geom(lev), pid, int(n_fine)) ); auto *const sep2 = stablePartition( sep, pid.end(), inexflag ); if (bmasks == gather_masks) { diff --git a/Source/Particles/Sorting/SortingUtils.H b/Source/Particles/Sorting/SortingUtils.H index ac2c63e88f8..ba7761bf48a 100644 --- a/Source/Particles/Sorting/SortingUtils.H +++ b/Source/Particles/Sorting/SortingUtils.H @@ -12,6 +12,7 @@ #include #include +#include /** \brief Fill the elements of the input vector with consecutive integer, @@ -19,7 +20,7 @@ * * \param[inout] v Vector of integers, to be filled by this routine */ -void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ); +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ); /** \brief Find the indices that would reorder the elements of `predicate` * so that the elements with non-zero value precede the other elements @@ -41,7 +42,7 @@ ForwardIterator stablePartition(ForwardIterator const index_begin, int const* AMREX_RESTRICT predicate_ptr = predicate.dataPtr(); int N = static_cast(std::distance(index_begin, index_end)); auto num_true = amrex::StablePartition(&(*index_begin), N, - [predicate_ptr] AMREX_GPU_DEVICE (long i) { return predicate_ptr[i]; }); + [predicate_ptr] AMREX_GPU_DEVICE (int i) { return predicate_ptr[i]; }); ForwardIterator sep = index_begin; std::advance(sep, num_true); @@ -49,7 +50,7 @@ ForwardIterator stablePartition(ForwardIterator const index_begin, // On CPU: Use std library ForwardIterator const sep = std::stable_partition( index_begin, index_end, - [&predicate](long i) { return predicate[i]; } + [&predicate](int i) { return predicate[i]; } ); #endif return sep; @@ -88,7 +89,7 @@ class fillBufferFlag // Extract simple structure that can be used directly on the GPU m_domain{geom.Domain()}, m_inexflag_ptr{inexflag.dataPtr()}, - m_particles{pti.GetArrayOfStructs().data()}, + m_ptd{pti.GetParticleTile().getConstParticleTileData()}, m_buffer_mask{(*bmasks)[pti].array()} { for (int idim=0; idim m_buffer_mask; amrex::GpuArray m_prob_lo; amrex::GpuArray m_inv_cell_size; @@ -141,12 +140,12 @@ class fillBufferFlagRemainingParticles amrex::iMultiFab const* bmasks, amrex::Gpu::DeviceVector& inexflag, amrex::Geometry const& geom, - amrex::Gpu::DeviceVector const& particle_indices, - long const start_index ) : + amrex::Gpu::DeviceVector const& particle_indices, + int start_index ) : m_domain{geom.Domain()}, // Extract simple structure that can be used directly on the GPU m_inexflag_ptr{inexflag.dataPtr()}, - m_particles{pti.GetArrayOfStructs().data()}, + m_ptd{pti.GetParticleTile().getConstParticleTileData()}, m_buffer_mask{(*bmasks)[pti].array()}, m_start_index{start_index}, m_indices_ptr{particle_indices.dataPtr()} @@ -159,11 +158,11 @@ class fillBufferFlagRemainingParticles AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator()( const long i ) const { + void operator()( const int i ) const { // Select a particle - auto const& p = m_particles[m_indices_ptr[i+m_start_index]]; + auto const j = m_indices_ptr[i+m_start_index]; // Find the index of the cell where this particle is located - amrex::IntVect const iv = amrex::getParticleCell( p, + amrex::IntVect const iv = amrex::getParticleCell( m_ptd, j, m_prob_lo, m_inv_cell_size, m_domain ); // Find the value of the buffer flag in this cell and // store it at the corresponding particle position in the array `inexflag` @@ -175,10 +174,10 @@ class fillBufferFlagRemainingParticles amrex::GpuArray m_inv_cell_size; amrex::Box m_domain; int* m_inexflag_ptr; - WarpXParticleContainer::ParticleType const* m_particles; + WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType const m_ptd; amrex::Array4 m_buffer_mask; - long const m_start_index; - long const* m_indices_ptr; + int const m_start_index; + int const* m_indices_ptr; }; /** \brief Functor that copies the elements of `src` into `dst`, @@ -195,7 +194,7 @@ class copyAndReorder copyAndReorder( amrex::Gpu::DeviceVector const& src, amrex::Gpu::DeviceVector& dst, - amrex::Gpu::DeviceVector const& indices ): + amrex::Gpu::DeviceVector const& indices ): // Extract simple structure that can be used directly on the GPU m_src_ptr{src.dataPtr()}, m_dst_ptr{dst.dataPtr()}, @@ -203,14 +202,14 @@ class copyAndReorder {} AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void operator()( const long ip ) const { + void operator()( const int ip ) const { m_dst_ptr[ip] = m_src_ptr[ m_indices_ptr[ip] ]; } private: T const* m_src_ptr; T* m_dst_ptr; - long const* m_indices_ptr; + int const* m_indices_ptr; }; #endif // WARPX_PARTICLES_SORTING_SORTINGUTILS_H_ diff --git a/Source/Particles/Sorting/SortingUtils.cpp b/Source/Particles/Sorting/SortingUtils.cpp index 699119e8e18..cd4b6a13c76 100644 --- a/Source/Particles/Sorting/SortingUtils.cpp +++ b/Source/Particles/Sorting/SortingUtils.cpp @@ -8,7 +8,7 @@ #include "SortingUtils.H" -void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) { #ifdef AMREX_USE_GPU // On GPU: Use amrex diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 33aa71d1c7d..7d2d5619da9 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -49,10 +49,10 @@ class WarpXParIter - : public amrex::ParIter<0,0,PIdx::nattribs> + : public amrex::ParIterSoA { public: - using amrex::ParIter<0,0,PIdx::nattribs>::ParIter; + using amrex::ParIterSoA::ParIterSoA; WarpXParIter (ContainerType& pc, int level); @@ -89,13 +89,14 @@ public: * particle container classes (that store a collection of particles) derive. Derived * classes can be used for plasma particles, photon particles, or non-physical * particles (e.g., for the laser antenna). - * It derives from amrex::ParticleContainer<0,0,PIdx::nattribs>, where the - * template arguments stand for the number of int and amrex::Real SoA and AoS - * data in amrex::Particle. - * - AoS amrex::Real: x, y, z (default), 0 additional (first template - * parameter) - * - AoS int: id, cpu (default), 0 additional (second template parameter) - * - SoA amrex::Real: PIdx::nattribs (third template parameter), see PIdx for + * It derives from amrex::ParticleContainerPureSoA, where the + * template arguments stand for the number of int and amrex::Real SoA + * data in amrex::SoAParticle. + * - SoA amrex::Real: positions x, y, z, momentum ux, uy, uz, ... see PIdx for details; + * more can be added at runtime + * - SoA int: 0 attributes by default, but can be added at runtime + * - SoA uint64_t: idcpu, a global 64bit index, with a 40bit local id and a 24bit cpu id + * (both set at creation) * the list. * * WarpXParticleContainer contains the main functions for initialization, @@ -164,11 +165,9 @@ public: * class. */ virtual void DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>& pinned_tile, - int n_external_attr_real, - int n_external_attr_int) = 0; + typename ContainerLike::ParticleTileType& pinned_tile, + int n_external_attr_real, + int n_external_attr_int) = 0; /// /// This pushes the particle positions by one half time step. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index a395198e361..0d565c039e6 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -75,13 +75,13 @@ using namespace amrex; WarpXParIter::WarpXParIter (ContainerType& pc, int level) - : amrex::ParIter<0,0,PIdx::nattribs>(pc, level, + : amrex::ParIterSoA(pc, level, MFItInfo().SetDynamic(WarpX::do_dynamic_scheduling)) { } WarpXParIter::WarpXParIter (ContainerType& pc, int level, MFItInfo& info) - : amrex::ParIter<0,0,PIdx::nattribs>(pc, level, + : amrex::ParIterSoA(pc, level, info.SetDynamic(WarpX::do_dynamic_scheduling)) { } @@ -198,52 +198,53 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, // Redistribute() will move them to proper places. auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); - using PinnedTile = amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>; + using PinnedTile = typename ContainerLike::ParticleTileType; PinnedTile pinned_tile; pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps()); const std::size_t np = iend-ibegin; #ifdef WARPX_DIM_RZ + amrex::Vector r(np); amrex::Vector theta(np); #endif for (auto i = ibegin; i < iend; ++i) { - ParticleType p; - if (id==-1) - { - p.id() = ParticleType::NextID(); - } else { - p.id() = id; + auto & idcpu_data = pinned_tile.GetStructOfArrays().GetIdCPUData(); + + amrex::Long current_id = id; // copy input + if (id == -1) { + current_id = ParticleType::NextID(); } - p.cpu() = amrex::ParallelDescriptor::MyProc(); + idcpu_data.push_back(amrex::SetParticleIDandCPU(current_id, ParallelDescriptor::MyProc())); + +#ifdef WARPX_DIM_RZ + r[i-ibegin] = std::sqrt(x[i]*x[i] + y[i]*y[i]); + theta[i-ibegin] = std::atan2(y[i], x[i]); +#endif + } + + if (np > 0) + { #if defined(WARPX_DIM_3D) - p.pos(0) = x[i]; - p.pos(1) = y[i]; - p.pos(2) = z[i]; + pinned_tile.push_back_real(PIdx::x, x.data() + ibegin, x.data() + iend); + pinned_tile.push_back_real(PIdx::y, y.data() + ibegin, y.data() + iend); + pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(y); #ifdef WARPX_DIM_RZ - theta[i-ibegin] = std::atan2(y[i], x[i]); - p.pos(0) = std::sqrt(x[i]*x[i] + y[i]*y[i]); + pinned_tile.push_back_real(PIdx::x, r.data(), r.data() + np); #else - p.pos(0) = x[i]; + pinned_tile.push_back_real(PIdx::x, x.data() + ibegin, x.data() + iend); #endif - p.pos(1) = z[i]; + pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); #else //AMREX_SPACEDIM == 1 amrex::ignore_unused(x,y); - p.pos(0) = z[i]; + pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); #endif - pinned_tile.push_back(p); - } - - if (np > 0) - { - pinned_tile.push_back_real(PIdx::w , attr_real[0].data() + ibegin, attr_real[0].data() + iend); + pinned_tile.push_back_real(PIdx::w, attr_real[0].data() + ibegin, attr_real[0].data() + iend); pinned_tile.push_back_real(PIdx::ux, ux.data() + ibegin, ux.data() + iend); pinned_tile.push_back_real(PIdx::uy, uy.data() + ibegin, uy.data() + iend); pinned_tile.push_back_real(PIdx::uz, uz.data() + ibegin, uz.data() + iend); @@ -476,15 +477,14 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, //sort particles by bin WARPX_PROFILE_VAR_START(blp_sort); - amrex::DenseBins bins; + amrex::DenseBins bins; { auto& ptile = ParticlesAt(lev, pti); - auto& aos = ptile.GetArrayOfStructs(); - auto *pstruct_ptr = aos().dataPtr(); + auto ptd = ptile.getParticleTileData(); const int ntiles = numTilesInBox(box, true, bin_size); - bins.build(ptile.numParticles(), pstruct_ptr, ntiles, + bins.build(ptile.numParticles(), ptd, ntiles, [=] AMREX_GPU_HOST_DEVICE (const ParticleType& p) -> unsigned int { Box tbox; @@ -947,7 +947,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // HACK - sort particles by bin here. WARPX_PROFILE_VAR_START(blp_sort); - amrex::DenseBins bins; + amrex::DenseBins bins; { const Geometry& geom = Geom(lev); const auto dxi = geom.InvCellSizeArray(); @@ -955,16 +955,15 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const auto domain = geom.Domain(); auto& ptile = ParticlesAt(lev, pti); - auto& aos = ptile.GetArrayOfStructs(); - auto *pstruct_ptr = aos().dataPtr(); + auto ptd = ptile.getParticleTileData(); Box box = pti.validbox(); box.grow(ng_rho); const amrex::IntVect bin_size = WarpX::shared_tilesize; const int ntiles = numTilesInBox(box, true, bin_size); - bins.build(ptile.numParticles(), pstruct_ptr, ntiles, - [=] AMREX_GPU_HOST_DEVICE (const ParticleType& p) -> unsigned int + bins.build(ptile.numParticles(), ptd, ntiles, + [=] AMREX_GPU_HOST_DEVICE (ParticleType const & p) -> unsigned int { Box tbx; auto iv = getParticleCell(p, plo, dxi, domain); @@ -984,8 +983,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const auto domain = geom.Domain(); auto& ptile = ParticlesAt(lev, pti); - auto& aos = ptile.GetArrayOfStructs(); - auto *pstruct_ptr = aos().dataPtr(); + auto ptd = ptile.getParticleTileData(); Box box = pti.validbox(); box.grow(ng_rho); @@ -999,9 +997,10 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const auto bin_start = offsets_ptr[ibin]; const auto bin_stop = offsets_ptr[ibin+1]; if (bin_start < bin_stop) { - auto p = pstruct_ptr[permutation[bin_start]]; + // static_cast until https://github.com/AMReX-Codes/amrex/pull/3684 + auto const i = static_cast(permutation[bin_start]); Box tbx; - auto iv = getParticleCell(p, plo, dxi, domain); + auto iv = getParticleCell(ptd, i, plo, dxi, domain); AMREX_ASSERT(box.contains(iv)); [[maybe_unused]] auto tid = getTileIndex(iv, box, true, bin_size, tbx); AMREX_ASSERT(tid == ibin); @@ -1490,10 +1489,10 @@ WarpXParticleContainer::particlePostLocate(ParticleType& p, // Tag particle if goes to higher level. // It will be split later in the loop if (pld.m_lev == lev+1 - and p.id() != NoSplitParticleID + and p.id() != amrex::LongParticleIds::NoSplitParticleID and p.id() >= 0) { - p.id() = DoSplitParticleID; + p.id() = amrex::LongParticleIds::DoSplitParticleID; } if (pld.m_lev == lev-1){ @@ -1532,9 +1531,9 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ const Real zmax = Geom(lev).ProbHi(WARPX_ZINDEX); ParticleTileType& ptile = ParticlesAt(lev, pti); - ParticleType * const pp = ptile.GetArrayOfStructs()().data(); auto& soa = ptile.GetStructOfArrays(); + uint64_t * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); amrex::ParticleReal * const AMREX_RESTRICT ux = soa.GetRealData(PIdx::ux).data(); amrex::ParticleReal * const AMREX_RESTRICT uy = soa.GetRealData(PIdx::uy).data(); amrex::ParticleReal * const AMREX_RESTRICT uz = soa.GetRealData(PIdx::uz).data(); @@ -1543,10 +1542,9 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ amrex::ParallelForRNG( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i, amrex::RandomEngine const& engine) { - ParticleType& p = pp[i]; - // skip particles that are already flagged for removal - if (p.id() < 0) { return; } + auto pidw = amrex::ParticleIDWrapper{idcpu[i]}; + if (!pidw.is_valid()) { return; } ParticleReal x, y, z; GetPosition.AsStored(i, x, y, z); @@ -1568,7 +1566,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ boundary_conditions, engine); if (particle_lost) { - p.id() = -p.id(); + pidw.make_invalid(); } else { SetPosition.AsStored(i, x, y, z); } diff --git a/Source/Python/Particles/ParticleBoundaryBuffer.cpp b/Source/Python/Particles/ParticleBoundaryBuffer.cpp index 2a35faece9b..b04ac75e600 100644 --- a/Source/Python/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Python/Particles/ParticleBoundaryBuffer.cpp @@ -10,13 +10,13 @@ namespace warpx { class BoundaryBufferParIter - : public amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> + : public amrex::ParIterSoA { public: - using amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator>::ParIter; + using amrex::ParIterSoA::ParIterSoA; BoundaryBufferParIter(ContainerType& pc, int level) : - amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator>(pc, level) {} + amrex::ParIterSoA(pc, level) {} }; } @@ -24,9 +24,9 @@ void init_BoundaryBufferParIter (py::module& m) { py::class_< warpx::BoundaryBufferParIter, - amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> + amrex::ParIterSoA >(m, "BoundaryBufferParIter") - .def(py::init::ContainerType&, int>(), + .def(py::init::ContainerType&, int>(), py::arg("particle_container"), py::arg("level") ) ; diff --git a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp index 600d56a62c9..d4f6a422dbe 100644 --- a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp +++ b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp @@ -13,6 +13,6 @@ void init_PinnedMemoryParticleContainer (py::module& m) { py::class_< PinnedMemoryParticleContainer, - amrex::ParticleContainer<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> + amrex::ParticleContainerPureSoA > pmpc (m, "PinnedMemoryParticleContainer"); } diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index 1473a750941..07793a373f3 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -12,11 +12,11 @@ void init_WarpXParIter (py::module& m) { py::class_< - WarpXParIter, amrex::ParIter<0,0,PIdx::nattribs> + WarpXParIter, amrex::ParIterSoA >(m, "WarpXParIter") - .def(py::init::ContainerType&, int>(), + .def(py::init::ContainerType&, int>(), py::arg("particle_container"), py::arg("level")) - .def(py::init::ContainerType&, int, amrex::MFItInfo&>(), + .def(py::init::ContainerType&, int, amrex::MFItInfo&>(), py::arg("particle_container"), py::arg("level"), py::arg("info")) ; @@ -26,11 +26,11 @@ void init_WarpXParticleContainer (py::module& m) { py::class_< WarpXParticleContainer, - amrex::ParticleContainer<0, 0, PIdx::nattribs, 0> + amrex::ParticleContainerPureSoA > wpc (m, "WarpXParticleContainer"); wpc .def("add_real_comp", - [](WarpXParticleContainer& pc, const std::string& name, bool const comm) { pc.AddRealComp(name, comm); }, + [](WarpXParticleContainer& pc, const std::string& name, bool comm) { pc.AddRealComp(name, comm); }, py::arg("name"), py::arg("comm") ) .def("add_n_particles", @@ -93,6 +93,14 @@ void init_WarpXParticleContainer (py::module& m) }, py::arg("comp_name") ) + .def("get_icomp_index", + [](WarpXParticleContainer& pc, std::string comp_name) + { + auto particle_comps = pc.getParticleiComps(); + return particle_comps.at(comp_name); + }, + py::arg("comp_name") + ) .def("num_local_tiles_at_level", &WarpXParticleContainer::numLocalTilesAtLevel, py::arg("level") diff --git a/Source/Utils/ParticleUtils.H b/Source/Utils/ParticleUtils.H index b04176d4d83..7e3c89228ea 100644 --- a/Source/Utils/ParticleUtils.H +++ b/Source/Utils/ParticleUtils.H @@ -28,9 +28,10 @@ namespace ParticleUtils { * @param[in] mfi the MultiFAB iterator. * @param[in] ptile the particle tile. */ - amrex::DenseBins - findParticlesInEachCell(int lev, amrex::MFIter const& mfi, - WarpXParticleContainer::ParticleTileType const& ptile); + amrex::DenseBins + findParticlesInEachCell (int lev, + amrex::MFIter const & mfi, + WarpXParticleContainer::ParticleTileType & ptile); /** * \brief Return (relativistic) particle energy given velocity and mass. diff --git a/Source/Utils/ParticleUtils.cpp b/Source/Utils/ParticleUtils.cpp index 60e04f12b86..b8207b61fa0 100644 --- a/Source/Utils/ParticleUtils.cpp +++ b/Source/Utils/ParticleUtils.cpp @@ -22,24 +22,28 @@ #include #include -namespace ParticleUtils { +namespace ParticleUtils +{ using namespace amrex; + // Define shortcuts for frequently-used type names - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = DenseBins; - using index_type = ParticleBins::index_type; + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = DenseBins; + using index_type = typename ParticleBins::index_type; /* Find the particles and count the particles that are in each cell. Note that this does *not* rearrange particle arrays */ ParticleBins - findParticlesInEachCell( int const lev, MFIter const& mfi, - ParticleTileType const& ptile) { + findParticlesInEachCell (int lev, + MFIter const & mfi, + ParticleTileType & ptile) { // Extract particle structures for this tile int const np = ptile.numParticles(); - ParticleType const* particle_ptr = ptile.GetArrayOfStructs()().data(); + auto ptd = ptile.getParticleTileData(); // Extract box properties Geometry const& geom = WarpX::GetInstance().Geom(lev); @@ -51,9 +55,9 @@ namespace ParticleUtils { // Find particles that are in each cell; // results are stored in the object `bins`. ParticleBins bins; - bins.build(np, particle_ptr, cbx, + bins.build(np, ptd, cbx, // Pass lambda function that returns the cell index - [=] AMREX_GPU_DEVICE (const ParticleType& p) noexcept + [=] AMREX_GPU_DEVICE (ParticleType const & p) noexcept -> amrex::IntVect { return IntVect{AMREX_D_DECL( static_cast((p.pos(0)-plo[0])*dxi[0] - lo.x), @@ -64,4 +68,4 @@ namespace ParticleUtils { return bins; } -} +} // namespace ParticleUtils diff --git a/Source/ablastr/particles/IndexHandling.H b/Source/ablastr/particles/IndexHandling.H deleted file mode 100644 index 0ad5ca60446..00000000000 --- a/Source/ablastr/particles/IndexHandling.H +++ /dev/null @@ -1,41 +0,0 @@ -/* Copyright 2019-2022 Axel Huebl - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#ifndef ABLASTR_INDEX_HANDLING_H -#define ABLASTR_INDEX_HANDLING_H - -#include - - -namespace ablastr::particles { - - /** A helper function to derive a globally unique particle ID - * - * @param[in] id AMReX particle ID (on local cpu/rank), AoS .id - * @param[in] cpu AMReX particle CPU (rank) at creation of the particle, AoS .cpu - * @return global particle ID that is unique and permanent in the whole simulation - */ - constexpr uint64_t - localIDtoGlobal (int const id, int const cpu) - { - static_assert(sizeof(int) * 2u <= sizeof(uint64_t), - "int size might cause collisions in global IDs"); - // implementation: - // - we cast both 32-bit (or smaller) ints to a 64bit unsigned int - // - this will leave half of the "upper range" bits in the 64bit unsigned int zeroed out - // because the corresponding (extended) value range was not part of the value range in - // the int representation - // - we bit-shift the cpu into the upper half of zero bits in the 64 bit unsigned int - // (imagine this step as "adding a per-cpu/rank offset to the local integers") - // - then we add this offset - // note: the add is expressed as bitwise OR (|) since this saves us from writing - // brackets for operator precedence between + and << - return uint64_t(id) | uint64_t(cpu) << 32u; - } - -} // namespace ablastr::particles - -#endif // ABLASTR_INDEX_HANDLING_H diff --git a/Source/ablastr/particles/ParticleMoments.H b/Source/ablastr/particles/ParticleMoments.H index e45fb574cce..b648ccb28aa 100644 --- a/Source/ablastr/particles/ParticleMoments.H +++ b/Source/ablastr/particles/ParticleMoments.H @@ -35,7 +35,7 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal> MinAndMaxPositions (T_PC const & pc) { - using PType = typename T_PC::SuperParticleType; + using ConstParticleTileDataType = typename T_PC::ParticleTileType::ConstParticleTileDataType; // Get min and max for the local rank amrex::ReduceOps< @@ -46,11 +46,11 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal, amrex::ParticleReal> >( pc, - [=] AMREX_GPU_DEVICE(PType const & p) noexcept + [=] AMREX_GPU_DEVICE(const ConstParticleTileDataType& ptd, const int i) noexcept { - amrex::ParticleReal const x = p.pos(0); - amrex::ParticleReal const y = p.pos(1); - amrex::ParticleReal const z = p.pos(2); + const amrex::ParticleReal x = ptd.rdata(0)[i]; + const amrex::ParticleReal y = ptd.rdata(1)[i]; + const amrex::ParticleReal z = ptd.rdata(2)[i]; return amrex::makeTuple(x, y, z, x, y, z); }, @@ -90,7 +90,8 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal> MeanAndStdPositions (T_PC const & pc) { - using PType = typename T_PC::SuperParticleType; + + using ConstParticleTileDataType = typename T_PC::ParticleTileType::ConstParticleTileDataType; amrex::ReduceOps< amrex::ReduceOpSum, amrex::ReduceOpSum, amrex::ReduceOpSum, @@ -103,12 +104,14 @@ namespace particles { amrex::ParticleReal> >( pc, - [=] AMREX_GPU_DEVICE(const PType& p) noexcept + [=] AMREX_GPU_DEVICE(const ConstParticleTileDataType& ptd, const int i) noexcept { - amrex::ParticleReal const x = p.pos(0); - amrex::ParticleReal const y = p.pos(1); - amrex::ParticleReal const z = p.pos(2); - amrex::ParticleReal const w = p.rdata(T_RealSoAWeight); + + const amrex::ParticleReal x = ptd.rdata(0)[i]; + const amrex::ParticleReal y = ptd.rdata(1)[i]; + const amrex::ParticleReal z = ptd.rdata(2)[i]; + + const amrex::ParticleReal w = ptd.rdata(T_RealSoAWeight)[i]; return amrex::makeTuple(x, x*x, y, y*y, z, z*z, w); }, diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 0f6a15a5ff4..81f5a533a76 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -269,7 +269,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.02" +set(WarpX_amrex_branch "296ed40e16ae1877640f5b78e9162dbd4ba1c279" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index b4cf9f3f9c1..8a9e35c6579 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.02" +set(WarpX_pyamrex_branch "defb663d74ef9f50183b31c5dc9731cf6adb447c" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index e1b45ab7c28..6d8a1ddb014 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.02 && cd - +cd amrex && git checkout --detach 296ed40e16ae1877640f5b78e9162dbd4ba1c279 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From a9d8126b500e1c7197eb0ed1e52fd50bb09cbdf4 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 5 Feb 2024 04:15:34 -0800 Subject: [PATCH 136/176] Fix: Pre-Installed AMReX w/ CUDA (#4668) Fix CMake language activation with pre-installed AMReX using the CUDA backend. --- cmake/dependencies/AMReX.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 81f5a533a76..6c7c9466dfd 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -257,6 +257,10 @@ macro(find_amrex) list(APPEND CMAKE_MODULE_PATH "${AMReX_DIR}/AMReXCMakeModules") message(STATUS "AMReX: Found version '${AMReX_VERSION}'") + + if(WarpX_COMPUTE STREQUAL CUDA) + enable_language(CUDA) + endif() endif() endmacro() From 7e368134be037599d8ed1983a04f400765dd719b Mon Sep 17 00:00:00 2001 From: "S. Eric Clark" <25495882+clarkse@users.noreply.github.com> Date: Mon, 5 Feb 2024 16:26:55 -0800 Subject: [PATCH 137/176] Add hybrid resistivity current term (#4661) * Adding total current magnitude dependence in hybrid resistivity. * Removed dead line of code. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Adding logic to only do current interpolation when the resistivity has J dependence. * Fixing staggering bug and changing how squares are computed. * Changing to using std::sqrt and adding _rt to value initialization for jtot_val. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/usage/parameters.rst | 2 +- Python/pywarpx/picmi.py | 2 +- .../HybridPICModel/HybridPICModel.H | 3 +- .../HybridPICModel/HybridPICModel.cpp | 8 ++- .../HybridPICSolveE.cpp | 68 +++++++++++++++++-- 5 files changed, 71 insertions(+), 12 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 493e8307037..d26c22e6dea 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2247,7 +2247,7 @@ Maxwell solver: kinetic-fluid hybrid If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the exponent used to calculate the electron pressure (see :ref:`here `). -* ``hybrid_pic_model.plasma_resistivity(rho)`` (`float` or `str`) optional (default ``0``) +* ``hybrid_pic_model.plasma_resistivity(rho,J)`` (`float` or `str`) optional (default ``0``) If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma resistivity in :math:`\Omega m`. * ``hybrid_pic_model.J[x/y/z]_external_grid_function(x, y, z, t)`` (`float` or `str`) optional (default ``0``) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 89bd5af2eab..f11ecb379f2 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1184,7 +1184,7 @@ def solver_initialize_inputs(self): pywarpx.hybridpicmodel.gamma = self.gamma pywarpx.hybridpicmodel.n_floor = self.n_floor pywarpx.hybridpicmodel.__setattr__( - 'plasma_resistivity(rho)', + 'plasma_resistivity(rho,J)', pywarpx.my_constants.mangle_expression(self.plasma_resistivity, self.mangle_dict) ) pywarpx.hybridpicmodel.substeps = self.substeps diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index d1931a71765..23ef49b58cb 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -172,7 +172,8 @@ public: /** Plasma resistivity */ std::string m_eta_expression = "0.0"; std::unique_ptr m_resistivity_parser; - amrex::ParserExecutor<1> m_eta; + amrex::ParserExecutor<2> m_eta; + bool m_resistivity_has_J_dependence = false; /** External current */ std::string m_Jx_ext_grid_function = "0.0"; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index fb7e90f21a1..034bb71efbc 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -37,7 +37,7 @@ void HybridPICModel::ReadParameters () Abort("hybrid_pic_model.n0_ref should be specified if hybrid_pic_model.gamma != 1"); } - pp_hybrid.query("plasma_resistivity(rho)", m_eta_expression); + pp_hybrid.query("plasma_resistivity(rho,J)", m_eta_expression); utils::parser::queryWithParser(pp_hybrid, "n_floor", m_n_floor); // convert electron temperature from eV to J @@ -123,8 +123,10 @@ void HybridPICModel::ClearLevel (int lev) void HybridPICModel::InitData () { m_resistivity_parser = std::make_unique( - utils::parser::makeParser(m_eta_expression, {"rho"})); - m_eta = m_resistivity_parser->compile<1>(); + utils::parser::makeParser(m_eta_expression, {"rho","J"})); + m_eta = m_resistivity_parser->compile<2>(); + const std::set resistivity_symbols = m_resistivity_parser->symbols(); + m_resistivity_has_J_dependence += resistivity_symbols.count("J"); m_J_external_parser[0] = std::make_unique( utils::parser::makeParser(m_Jx_ext_grid_function,{"x","y","z","t"})); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index 1a72fee53c2..5100eed0df3 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -433,6 +433,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // get hybrid model parameters const auto eta = hybrid_model->m_eta; const auto rho_floor = hybrid_model->m_n_floor * PhysConst::q_e; + const auto resistivity_has_J_dependence = hybrid_model->m_resistivity_has_J_dependence; // Index type required for interpolating fields from their respective // staggering to the Ex, Ey, Ez locations @@ -589,6 +590,15 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jr_val = Interp(Jr, Jr_stag, Er_stag, coarsen, i, j, 0, 0); + Real jt_val = Interp(Jt, Jt_stag, Er_stag, coarsen, i, j, 0, 0); + Real jz_val = Interp(Jz, Jz_stag, Er_stag, coarsen, i, j, 0, 0); + jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later if (rho_val < rho_floor) { rho_val = rho_floor; } @@ -601,7 +611,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Er(i, j, 0) = (enE_r - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Er(i, j, 0) += eta(rho_val) * Jr(i, j, 0); } + if (include_resistivity_term) { Er(i, j, 0) += eta(rho_val, jtot_val) * Jr(i, j, 0); } }, // Et calculation @@ -622,6 +632,15 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jr_val = Interp(Jr, Jr_stag, Et_stag, coarsen, i, j, 0, 0); + Real jt_val = Interp(Jt, Jt_stag, Et_stag, coarsen, i, j, 0, 0); + Real jz_val = Interp(Jz, Jz_stag, Et_stag, coarsen, i, j, 0, 0); + jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later if (rho_val < rho_floor) { rho_val = rho_floor; } @@ -635,7 +654,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Et(i, j, 0) = (enE_t - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Et(i, j, 0) += eta(rho_val) * Jt(i, j, 0); } + if (include_resistivity_term) { Et(i, j, 0) += eta(rho_val, jtot_val) * Jt(i, j, 0); } }, // Ez calculation @@ -647,6 +666,15 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jr_val = Interp(Jr, Jr_stag, Ez_stag, coarsen, i, j, 0, 0); + Real jt_val = Interp(Jt, Jt_stag, Ez_stag, coarsen, i, j, 0, 0); + Real jz_val = Interp(Jz, Jz_stag, Ez_stag, coarsen, i, j, 0, 0); + jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later if (rho_val < rho_floor) { rho_val = rho_floor; } @@ -659,7 +687,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); } + if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val, jtot_val) * Jz(i, j, k); } } ); @@ -699,6 +727,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // get hybrid model parameters const auto eta = hybrid_model->m_eta; const auto rho_floor = hybrid_model->m_n_floor * PhysConst::q_e; + const auto resistivity_has_J_dependence = hybrid_model->m_resistivity_has_J_dependence; // Index type required for interpolating fields from their respective // staggering to the Ex, Ey, Ez locations @@ -853,6 +882,15 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ex_stag, coarsen, i, j, k, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jx_val = Interp(Jx, Jx_stag, Ex_stag, coarsen, i, j, k, 0); + Real jy_val = Interp(Jy, Jy_stag, Ex_stag, coarsen, i, j, k, 0); + Real jz_val = Interp(Jz, Jz_stag, Ex_stag, coarsen, i, j, k, 0); + jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later if (rho_val < rho_floor) { rho_val = rho_floor; } @@ -865,7 +903,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ex(i, j, k) = (enE_x - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Ex(i, j, k) += eta(rho_val) * Jx(i, j, k); } + if (include_resistivity_term) { Ex(i, j, k) += eta(rho_val, jtot_val) * Jx(i, j, k); } }, // Ey calculation @@ -883,6 +921,15 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ey_stag, coarsen, i, j, k, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jx_val = Interp(Jx, Jx_stag, Ey_stag, coarsen, i, j, k, 0); + Real jy_val = Interp(Jy, Jy_stag, Ey_stag, coarsen, i, j, k, 0); + Real jz_val = Interp(Jz, Jz_stag, Ey_stag, coarsen, i, j, k, 0); + jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later if (rho_val < rho_floor) { rho_val = rho_floor; } @@ -895,7 +942,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ey(i, j, k) = (enE_y - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Ey(i, j, k) += eta(rho_val) * Jy(i, j, k); } + if (include_resistivity_term) { Ey(i, j, k) += eta(rho_val, jtot_val) * Jy(i, j, k); } }, // Ez calculation @@ -907,6 +954,15 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jx_val = Interp(Jx, Jx_stag, Ez_stag, coarsen, i, j, k, 0); + Real jy_val = Interp(Jy, Jy_stag, Ez_stag, coarsen, i, j, k, 0); + Real jz_val = Interp(Jz, Jz_stag, Ez_stag, coarsen, i, j, k, 0); + jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later if (rho_val < rho_floor) { rho_val = rho_floor; } @@ -919,7 +975,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); } + if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val, jtot_val) * Jz(i, j, k); } } ); From 83e16ee9a6875d4434bf9d9aff9bc51d1923c6e4 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 5 Feb 2024 18:24:51 -0800 Subject: [PATCH 138/176] CI: 4 Cores Linux/Win, 3 Cores macOS (#4673) Increase build and test parallelism according to new increased core limits on public GH hosted runners. --- .github/workflows/clang_tidy.yml | 6 +++--- .github/workflows/codeql.yml | 4 ++-- .github/workflows/cuda.yml | 6 +++--- .github/workflows/hip.yml | 4 ++-- .github/workflows/insitu.yml | 4 ++-- .github/workflows/intel.yml | 8 ++++---- .github/workflows/ubuntu.yml | 10 +++++----- .github/workflows/windows.yml | 4 ++-- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index b8b0053adaa..2a30696fb8c 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -37,7 +37,7 @@ jobs: cmake -S . -B build_clang_tidy \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DWarpX_DIMS="1;2;3;RZ" \ + -DWarpX_DIMS="1;2;RZ;3" \ -DWarpX_MPI=ON \ -DWarpX_COMPUTE=OMP \ -DWarpX_PSATD=ON \ @@ -47,10 +47,10 @@ jobs: -DWarpX_PRECISION=SINGLE \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - cmake --build build_clang_tidy -j 2 + cmake --build build_clang_tidy -j 4 ${{github.workspace}}/.github/workflows/source/makeMakefileForClangTidy.py --input ${{github.workspace}}/ccache.log.txt - make -j2 --keep-going -f clang-tidy-ccache-misses.mak \ + make -j4 --keep-going -f clang-tidy-ccache-misses.mak \ CLANG_TIDY=clang-tidy \ CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 436df798d3b..008d82af239 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -75,7 +75,7 @@ jobs: export CCACHE_MAXSIZE=100M ccache -z - $CMAKE --build build -j 2 + $CMAKE --build build -j 4 ccache -s du -hs ~/.cache/ccache @@ -83,7 +83,7 @@ jobs: # Make sure CodeQL has something to do touch Source/Utils/WarpXVersion.cpp export CCACHE_DISABLE=1 - $CMAKE --build build -j 2 + $CMAKE --build build -j 4 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 5e9f43f639d..ed8b315c4e9 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -73,7 +73,7 @@ jobs: -DWarpX_PSATD=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON - cmake --build build_sp -j 2 + cmake --build build_sp -j 4 python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools wheel @@ -116,7 +116,7 @@ jobs: git clone https://github.com/AMReX-Codes/amrex.git ../amrex cd ../amrex && git checkout --detach 296ed40e16ae1877640f5b78e9162dbd4ba1c279 && cd - - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 + make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s du -hs ~/.cache/ccache @@ -171,7 +171,7 @@ jobs: -DWarpX_PSATD=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON - cmake --build build -j 2 + cmake --build build -j 4 # work-around for mpi4py 3.1.1 build system issue with using # a GNU-built Python executable with non-GNU Python modules diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index f7378bfa775..51cadc89604 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -56,7 +56,7 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ -DWarpX_PSATD=ON - cmake --build build_sp -j 2 + cmake --build build_sp -j 4 export WARPX_MPI=OFF export PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ @@ -116,7 +116,7 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ -DWarpX_PSATD=ON - cmake --build build_2d -j 2 + cmake --build build_2d -j 4 export WARPX_MPI=OFF export PYWARPX_LIB_DIR=$PWD/build_2d/lib/site-packages/pywarpx/ diff --git a/.github/workflows/insitu.yml b/.github/workflows/insitu.yml index 57a25ce7629..6006c3e5c5b 100644 --- a/.github/workflows/insitu.yml +++ b/.github/workflows/insitu.yml @@ -28,7 +28,7 @@ jobs: -DWarpX_COMPUTE=NOACC - name: Build run: | - cmake --build build -j 2 + cmake --build build -j 4 ascent: name: Ascent @@ -51,7 +51,7 @@ jobs: - name: Build run: | . /ascent_docker_setup_env.sh - cmake --build build -j 2 + cmake --build build -j 4 - name: Test run: | cp Examples/Physics_applications/laser_acceleration/inputs_3d . diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 9124715fe18..1731f6e3723 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -53,7 +53,7 @@ jobs: -DWarpX_MPI=OFF \ -DWarpX_OPENPMD=ON \ -DWarpX_openpmd_internal=OFF - cmake --build build_dp -j 2 + cmake --build build_dp -j 4 cmake -S . -B build_sp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -64,7 +64,7 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_openpmd_internal=OFF \ -DWarpX_PRECISION=SINGLE - cmake --build build_sp -j 2 + cmake --build build_sp -j 4 cmake --build build_sp --target pip_install ccache -s @@ -120,7 +120,7 @@ jobs: -DWarpX_MPI=OFF \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE - cmake --build build_sp -j 2 + cmake --build build_sp -j 4 cmake --build build_sp --target pip_install ccache -s @@ -184,7 +184,7 @@ jobs: -DWarpX_MPI=OFF \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE - cmake --build build_sp -j 2 + cmake --build build_sp -j 4 ccache -s du -hs ~/.cache/ccache diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 6b8e26111b8..239da17be64 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -38,7 +38,7 @@ jobs: -DWarpX_EB=OFF \ -DWarpX_MPI=OFF \ -DWarpX_QED=OFF - cmake --build build -j 2 + cmake --build build -j 4 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz @@ -79,7 +79,7 @@ jobs: -DWarpX_EB=OFF \ -DWarpX_PSATD=ON \ -DWarpX_QED_TABLE_GEN=ON - cmake --build build -j 2 + cmake --build build -j 4 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d @@ -126,7 +126,7 @@ jobs: -DWarpX_PARTICLE_PRECISION=SINGLE \ -DWarpX_QED_TABLE_GEN=ON - cmake --build build -j 2 + cmake --build build -j 4 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz @@ -164,7 +164,7 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_APP=OFF \ -DWarpX_LIB=OFF - cmake --build build -j 2 + cmake --build build -j 4 ccache -s du -hs ~/.cache/ccache @@ -208,7 +208,7 @@ jobs: -DWarpX_PSATD=ON \ -DWarpX_PYTHON=ON \ -DWarpX_QED_TABLE_GEN=ON - cmake --build build -j 2 --target pip_install + cmake --build build -j 4 --target pip_install ccache -s du -hs ~/.cache/ccache diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 8e2bb00f1db..eee40e72965 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -38,7 +38,7 @@ jobs: -DWarpX_MPI=OFF ` -DWarpX_PYTHON=ON if(!$?) { Exit $LASTEXITCODE } - cmake --build build --config Debug --parallel 2 + cmake --build build --config Debug --parallel 4 if(!$?) { Exit $LASTEXITCODE } python3 -m pip install --upgrade pip @@ -96,7 +96,7 @@ jobs: -DWarpX_MPI=OFF ^ -DWarpX_OPENPMD=ON if errorlevel 1 exit 1 - cmake --build build --config Release --parallel 2 + cmake --build build --config Release --parallel 4 if errorlevel 1 exit 1 cmake --build build --config Release --target install From 6e87dd52995b6faf16a2bf76bd873c6447f76548 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Tue, 6 Feb 2024 13:01:58 -0800 Subject: [PATCH 139/176] Update GitHub Action versions (#4674) * Bump actions/upload-artifact from 3 to 4 * Bump github/codeql-action from 2 to 3 * Bump actions/checkout from 3 to 4 * Bump actions/setup-python from 4 to 5 * Bump actions/cache from 3 to 4 --- .github/workflows/clang_tidy.yml | 6 +++--- .github/workflows/cleanup-cache-postpr.yml | 2 +- .github/workflows/cleanup-cache.yml | 2 +- .github/workflows/codeql.yml | 14 +++++++------- .github/workflows/cuda.yml | 16 ++++++++-------- .github/workflows/hip.yml | 10 +++++----- .github/workflows/insitu.yml | 6 +++--- .github/workflows/intel.yml | 14 +++++++------- .github/workflows/macos.yml | 6 +++--- .github/workflows/post-pr.yml | 2 +- .github/workflows/source.yml | 2 +- .github/workflows/ubuntu.yml | 22 +++++++++++----------- .github/workflows/windows.yml | 12 ++++++------ 13 files changed, 57 insertions(+), 57 deletions(-) diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index 2a30696fb8c..6a1172802a8 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -12,12 +12,12 @@ jobs: runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/clang14.sh - name: set up cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -66,7 +66,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} run: | echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: pr_number path: pr_number.txt diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml index 978e9c28f04..9a2ffb0f61a 100644 --- a/.github/workflows/cleanup-cache-postpr.yml +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -16,7 +16,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Clean up ccache run: | gh extension install actions/gh-actions-cache diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml index 6421bbf4215..bd1a518acf4 100644 --- a/.github/workflows/cleanup-cache.yml +++ b/.github/workflows/cleanup-cache.yml @@ -16,7 +16,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Clean up ccache run: | gh extension install actions/gh-actions-cache diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 008d82af239..bc0bee545cc 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -29,7 +29,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Packages (C++) if: ${{ matrix.language == 'cpp' }} @@ -44,7 +44,7 @@ jobs: - name: Set Up Cache if: ${{ matrix.language == 'cpp' }} - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -57,14 +57,14 @@ jobs: $CMAKE -S . -B build -DWarpX_OPENPMD=ON - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: config-file: ./.github/codeql/warpx-codeql.yml languages: ${{ matrix.language }} queries: +security-and-quality - name: Build (py) - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 if: ${{ matrix.language == 'python' }} - name: Build (C++) @@ -86,7 +86,7 @@ jobs: $CMAKE --build build -j 4 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{ matrix.language }}" upload: False @@ -107,7 +107,7 @@ jobs: output: sarif-results/${{ matrix.language }}.sarif - name: Upload SARIF - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: sarif-results/${{ matrix.language }}.sarif @@ -120,7 +120,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} run: | echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: pr_number path: pr_number.txt diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index ed8b315c4e9..3546eb8e9eb 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -19,8 +19,8 @@ jobs: CXXFLAGS: "-Werror" CMAKE_GENERATOR: Ninja steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 name: Install Python with: python-version: '3.x' @@ -28,7 +28,7 @@ jobs: run: | .github/workflows/dependencies/nvcc11-3.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -92,12 +92,12 @@ jobs: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/nvcc11-8.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -129,11 +129,11 @@ jobs: # # For NVHPC, Ninja is slower than the default: # CMAKE_GENERATOR: Ninja steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Dependencies run: .github/workflows/dependencies/nvhpc.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -197,7 +197,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} run: | echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: pr_number path: pr_number.txt diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 51cadc89604..0e311f061ef 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -15,12 +15,12 @@ jobs: CMAKE_GENERATOR: Ninja if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies shell: bash run: .github/workflows/dependencies/hip.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -74,12 +74,12 @@ jobs: CMAKE_GENERATOR: Ninja if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies shell: bash run: .github/workflows/dependencies/hip.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -135,7 +135,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} run: | echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: pr_number path: pr_number.txt diff --git a/.github/workflows/insitu.yml b/.github/workflows/insitu.yml index 6006c3e5c5b..42923d3df8e 100644 --- a/.github/workflows/insitu.yml +++ b/.github/workflows/insitu.yml @@ -20,7 +20,7 @@ jobs: container: image: senseiinsitu/ci:fedora35-amrex-20220613 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure run: | cmake -S . -B build \ @@ -41,7 +41,7 @@ jobs: container: image: alpinedav/ascent:0.9.2 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure run: | . /ascent_docker_setup_env.sh @@ -61,7 +61,7 @@ jobs: max_step = 40 \ diag1.intervals = 30:40:10 \ diag1.format = ascent - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: ascent-test-artifacts path: | diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 1731f6e3723..3b1d6b546a4 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -17,12 +17,12 @@ jobs: #env: # CMAKE_GENERATOR: Ninja steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/icc.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -82,13 +82,13 @@ jobs: # CMAKE_GENERATOR: Ninja if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies shell: bash run: | .github/workflows/dependencies/dpcpp.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -146,13 +146,13 @@ jobs: # CMAKE_GENERATOR: Ninja if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies shell: bash run: | .github/workflows/dependencies/dpcpp.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -204,7 +204,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} run: | echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: pr_number path: pr_number.txt diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 0e8819032e3..f34f9f3534d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -17,7 +17,7 @@ jobs: # For macOS, Ninja is slower than the default: #CMAKE_GENERATOR: Ninja steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | set +e @@ -45,7 +45,7 @@ jobs: python3 -m pip install --upgrade build packaging setuptools wheel python3 -m pip install --upgrade mpi4py - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /Users/runner/Library/Caches/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -97,7 +97,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} run: | echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: pr_number path: pr_number.txt diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml index f5b914033b7..2768ef376cc 100644 --- a/.github/workflows/post-pr.yml +++ b/.github/workflows/post-pr.yml @@ -13,7 +13,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} run: | echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: pr_number path: pr_number.txt diff --git a/.github/workflows/source.yml b/.github/workflows/source.yml index 08050768894..a1c29416b3e 100644 --- a/.github/workflows/source.yml +++ b/.github/workflows/source.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Non-ASCII characters run: .github/workflows/source/hasNonASCII - name: TABs diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 239da17be64..cf4b375ce00 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -14,12 +14,12 @@ jobs: env: CXXFLAGS: "-Werror" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/gcc.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -54,12 +54,12 @@ jobs: CXX: "g++-12" CC: "gcc-12" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/gcc12.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -94,12 +94,12 @@ jobs: CXX: "g++-12" CC: "gcc-12" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/gcc12_blaspp_lapackpp.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -141,13 +141,13 @@ jobs: CMAKE_GENERATOR: Ninja CXXFLAGS: "-Werror" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/gcc.sh sudo apt-get install -y libopenmpi-dev openmpi-bin - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -179,12 +179,12 @@ jobs: # On CI for this test, Ninja is slower than the default: #CMAKE_GENERATOR: Ninja steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/pyfull.sh - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} @@ -227,7 +227,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} run: | echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: pr_number path: pr_number.txt diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index eee40e72965..2ef74cdb7f9 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -12,12 +12,12 @@ jobs: runs-on: windows-latest if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 # - once stored under a key, they become immutable (even if local cache path content changes) # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: @@ -63,13 +63,13 @@ jobs: runs-on: windows-2019 if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.8' - uses: seanmiddleditch/gha-setup-ninja@master - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 # - once stored under a key, they become immutable (even if local cache path content changes) # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: From 85326aea82fd141de0d9a463a1d52083786e872f Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 7 Feb 2024 16:08:23 -0800 Subject: [PATCH 140/176] Update Polaris install instructions (#4664) --- .../polaris-alcf/install_gpu_dependencies.sh | 10 +++++----- .../polaris_gpu_warpx.profile.example | 20 +++++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/Tools/machines/polaris-alcf/install_gpu_dependencies.sh b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh index e2cdca86fbc..dbc66b2ffff 100755 --- a/Tools/machines/polaris-alcf/install_gpu_dependencies.sh +++ b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh @@ -4,7 +4,7 @@ # # This file is part of WarpX. # -# Author: Axel Huebl (edited by Roelof Groenewald for Polaris) +# Authors: Axel Huebl, Roelof Groenewald # License: BSD-3-Clause-LBNL # Exit on first error encountered ############################################# @@ -98,7 +98,7 @@ python3 -m pip install --upgrade pip python3 -m pip install --upgrade virtualenv python3 -m pip cache purge rm -rf ${SW_DIR}/venvs/warpx -python3 -m venv --system-site-packages ${SW_DIR}/venvs/warpx +python3 -m venv ${SW_DIR}/venvs/warpx source ${SW_DIR}/venvs/warpx/bin/activate python3 -m pip install --upgrade pip python3 -m pip install --upgrade build @@ -109,15 +109,15 @@ python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas python3 -m pip install --upgrade scipy -# MPICC="cc -target-accel=nvidia80 -shared" python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +MPICC="CC -target-accel=nvidia80 -shared" python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -python3 -m pip install cupy-cuda11x # CUDA 11.7 compatible wheel +python3 -m pip install cupy-cuda11x # CUDA 11.8 compatible wheel # optional: for libEnsemble python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch # CUDA 11.7 compatible wheel +python3 -m pip install --upgrade torch # CUDA 11.8 compatible wheel python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example index d7a68bf16bb..d7635188141 100644 --- a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example +++ b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example @@ -1,12 +1,16 @@ # Set the project name export proj="" # change me! +# swap to GNU programming environment (with gcc 11.2) +module swap PrgEnv-nvhpc PrgEnv-gnu +module swap gcc/12.2.0 gcc/11.2.0 +module load nvhpc-mixed/22.11 + # swap to the Milan cray package -# module swap craype-x86-rome craype-x86-milan +module swap craype-x86-rome craype-x86-milan # required dependencies module load cmake/3.23.2 -module load cudatoolkit-standalone # optional: for QED support with detailed tables # module load boost/1.81.0 @@ -41,11 +45,11 @@ export AMREX_CUDA_ARCH=8.0 # optimize CPU microarchitecture for AMD EPYC 3rd Gen (Milan/Zen3) # note: the cc/CC/ftn wrappers below add those -# export CXXFLAGS="-march=znver3" -# export CFLAGS="-march=znver3" +export CXXFLAGS="-march=znver3" +export CFLAGS="-march=znver3" # compiler environment hints -export CC=nvc -export CXX=nvc++ -export CUDACXX=nvcc -export CUDAHOSTCXX=nvc++ +export CC=$(which gcc) +export CXX=$(which g++) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} From 9c082da749d19c0a82c50419d91c32066a743f32 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 7 Feb 2024 16:09:38 -0800 Subject: [PATCH 141/176] Minor fix in SLURM signal forwarding description (#4665) --- Docs/source/usage/parameters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index d26c22e6dea..387d219f11a 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -250,7 +250,7 @@ We follow the same naming, but remove the ``SIG`` prefix, e.g., the WarpX signal .. code-block:: bash - #SBATCH --signal=B:1@360 + #SBATCH --signal=1@360 srun ... \ warpx.break_signals=HUP \ From 94d48fd80e7f4b375306abac6e94ad4df2b69997 Mon Sep 17 00:00:00 2001 From: Marco Garten Date: Wed, 7 Feb 2024 16:10:15 -0800 Subject: [PATCH 142/176] ParticleHistogram2D change openPMD access type (#4659) * Change openPMD access type Changed openPMD access type for file creation in ParticleHistogram2D from APPEND to CREATE to avoid warnings on execution. Additionally: - make paths compatible with Windows - allow to set the runtime option `file_min_digits` * Add file_min_digits parameter to docs --- Docs/source/usage/parameters.rst | 3 +++ .../ReducedDiags/ParticleHistogram2D.H | 3 +++ .../ReducedDiags/ParticleHistogram2D.cpp | 17 +++++++++++++++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 387d219f11a..b02e6198213 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -3104,6 +3104,9 @@ Reduced Diagnostics A species name must be provided, such that the diagnostics are done for this species. + * ``.file_min_digits`` (`int`) optional (default `6`) + The minimum number of digits used for the iteration number appended to the diagnostic file names. + * ``.histogram_function_abs(t,x,y,z,ux,uy,uz,w)`` (`string`) A histogram function must be provided for the abscissa axis. `t` represents the physical time in seconds during the simulation. diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H index 98697380a4a..6ba85502819 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H @@ -35,6 +35,9 @@ public: /// File type std::string m_openpmd_backend {"default"}; + /// minimum number of digits for file suffix (file-based only supported now for now) */ + int m_file_min_digits = 6; + /// number of bins on the abscissa and ordinate int m_bin_num_abs; int m_bin_num_ord; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp index 7cac712b4fc..473a53715e6 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp @@ -60,6 +60,7 @@ ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) ParmParse pp_rd_name(rd_name); pp_rd_name.query("openpmd_backend", m_openpmd_backend); + pp_rd_name.query("file_min_digits", m_file_min_digits); // pick first available backend if default is chosen if( m_openpmd_backend == "default" ) { m_openpmd_backend = WarpXOpenPMDFileType(); @@ -260,10 +261,22 @@ void ParticleHistogram2D::WriteToFile (int step) const // only IO processor writes if ( !ParallelDescriptor::IOProcessor() ) { return; } + // TODO: support different filename templates + std::string filename = "openpmd"; + // TODO: support also group-based encoding + const std::string fileSuffix = std::string("_%0") + std::to_string(m_file_min_digits) + std::string("T"); + filename = filename.append(fileSuffix).append(".").append(m_openpmd_backend); + + std::string filepath = m_path + m_rd_name + "/" + filename; + // transform paths for Windows + #ifdef _WIN32 + filepath = openPMD::auxiliary::replace_all(filepath, "/", "\\"); + #endif + // Create the OpenPMD series auto series = io::Series( - m_path + m_rd_name + "/openpmd_%06T." + m_openpmd_backend, - io::Access::APPEND); + filepath, + io::Access::CREATE); auto i = series.iterations[step + 1]; // record auto f_mesh = i.meshes["data"]; From 11aabdca56335c5ae1cbb2257b8abd6c8f04a67c Mon Sep 17 00:00:00 2001 From: Marco Garten Date: Wed, 7 Feb 2024 20:26:41 -0800 Subject: [PATCH 143/176] Fix docs for checksumAPI terminal call (#4677) Changed `--format` to `--output-format` like the call signature demands --- Docs/source/developers/checksum.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/developers/checksum.rst b/Docs/source/developers/checksum.rst index 4412c3e3cc4..81d16809d50 100644 --- a/Docs/source/developers/checksum.rst +++ b/Docs/source/developers/checksum.rst @@ -32,7 +32,7 @@ You can execute ``checksumAPI.py`` as a Python script for that, and pass the plo .. code-block:: bash - ./checksumAPI.py --evaluate --output-file --format <'openpmd' or 'plotfile'> --test-name + ./checksumAPI.py --evaluate --output-file --output-format <'openpmd' or 'plotfile'> --test-name See additional options @@ -51,7 +51,7 @@ This is using ``checksumAPI.py`` as a Python script. .. code-block:: bash - ./checksumAPI.py --reset-benchmark --output-file --format <'openpmd' or 'plotfile'> --test-name + ./checksumAPI.py --reset-benchmark --output-file --output-format <'openpmd' or 'plotfile'> --test-name See additional options From c6875f4fe06e79db8b46bf10cbab7cd5a9385e4f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 8 Feb 2024 01:20:12 -0800 Subject: [PATCH 144/176] CI: NVHPC 24.1 (#4679) * CI: NVHPC 24.1 Update NVHPC from 21.11 to 24.1. * Python off: NVHPC Fails on pybind11 Reenable once compiler bug is fixed. --- .github/workflows/cuda.yml | 8 ++++---- .github/workflows/dependencies/nvhpc.sh | 10 +++++----- Source/Diagnostics/BTD_Plotfile_Header_Impl.H | 4 ++-- Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp | 4 +++- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 3546eb8e9eb..0e5b32c2de0 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -121,8 +121,8 @@ jobs: ccache -s du -hs ~/.cache/ccache - build_nvhpc21-11-nvcc: - name: NVHPC@21.11 NVCC/NVC++ Release [tests] + build_nvhpc24-1-nvcc: + name: NVHPC@24.1 NVCC/NVC++ Release [tests] runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false #env: @@ -147,7 +147,7 @@ jobs: ccache -z source /etc/profile.d/modules.sh - module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.11 + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/24.1 which nvcc || echo "nvcc not in PATH!" which nvc++ || echo "nvc++ not in PATH!" which nvc || echo "nvc not in PATH!" @@ -165,7 +165,7 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_COMPUTE=CUDA \ -DWarpX_EB=ON \ - -DWarpX_PYTHON=ON \ + -DWarpX_PYTHON=OFF \ -DAMReX_CUDA_ARCH=8.0 \ -DWarpX_OPENPMD=ON \ -DWarpX_PSATD=ON \ diff --git a/.github/workflows/dependencies/nvhpc.sh b/.github/workflows/dependencies/nvhpc.sh index be2ff29ca1e..3533d6ca9f6 100755 --- a/.github/workflows/dependencies/nvhpc.sh +++ b/.github/workflows/dependencies/nvhpc.sh @@ -31,17 +31,17 @@ $(dirname "$0")/ccache.sh echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | \ sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt update -y && \ -sudo apt install -y --no-install-recommends nvhpc-21-11 && \ +sudo apt install -y --no-install-recommends nvhpc-24-1 && \ sudo rm -rf /var/lib/apt/lists/* && \ - sudo rm -rf /opt/nvidia/hpc_sdk/Linux_x86_64/21.11/examples \ - /opt/nvidia/hpc_sdk/Linux_x86_64/21.11/profilers \ - /opt/nvidia/hpc_sdk/Linux_x86_64/21.11/math_libs/11.5/targets/x86_64-linux/lib/lib*_static*.a + sudo rm -rf /opt/nvidia/hpc_sdk/Linux_x86_64/24.1/examples \ + /opt/nvidia/hpc_sdk/Linux_x86_64/24.1/profilers \ + /opt/nvidia/hpc_sdk/Linux_x86_64/24.1/math_libs/11.5/targets/x86_64-linux/lib/lib*_static*.a # things should reside in /opt/nvidia/hpc_sdk now # activation via: # source /etc/profile.d/modules.sh -# module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.11 +# module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/24.1 # cmake-easyinstall # diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H index b87c54be9f0..0a22774bb8d 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H @@ -144,9 +144,9 @@ private: /** Number of Fabs in the plotfile. */ int m_numFabs; /** Lower corner physical coordinates of each fab in the plotfile. */ - amrex::Vector > m_glo {{AMREX_D_DECL(0.,0.,0.)}}; + amrex::Vector > m_glo; /** Upper corner physical coordinates of each fab in the plotfile. */ - amrex::Vector > m_ghi {{AMREX_D_DECL(1.,1.,1.)}}; + amrex::Vector > m_ghi; std::string m_CellPath; }; diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp index da4a3a79547..cd3eb962405 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp @@ -22,7 +22,9 @@ using namespace amrex::literals; BTDPlotfileHeaderImpl::BTDPlotfileHeaderImpl (std::string const & Headerfile_path) - : m_Header_path{Headerfile_path} + : m_Header_path{Headerfile_path}, + m_glo{{AMREX_D_DECL(0., 0., 0.)}}, + m_ghi{{AMREX_D_DECL(1., 1., 1.)}} { } From 58bc0ba55b46c466baed0ae662fe0e7dd447bbd5 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 8 Feb 2024 07:56:01 -0800 Subject: [PATCH 145/176] Update pyAMReX to `development` (#4680) * pyAMReX: Weekly Update * Update Python APIs pyAMReX changed some APIs to be more Pythonic (snake_case). --- Python/pywarpx/particle_containers.py | 10 +++++----- cmake/dependencies/pyAMReX.cmake | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index 72a675ec43c..393663bc8cf 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -223,7 +223,7 @@ def get_particle_real_arrays(self, comp_name, level, copy_to_host=False): data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): soa = pti.soa() - idx = soa.GetRealData(comp_idx) + idx = soa.get_real_data(comp_idx) if copy_to_host: data_array.append(idx.to_numpy(copy=True)) else: @@ -269,7 +269,7 @@ def get_particle_int_arrays(self, comp_name, level, copy_to_host=False): data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): soa = pti.soa() - idx = soa.GetIntData(comp_idx) + idx = soa.get_int_data(comp_idx) if copy_to_host: data_array.append(idx.to_numpy(copy=True)) else: @@ -309,7 +309,7 @@ def get_particle_idcpu_arrays(self, level, copy_to_host=False): data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): soa = pti.soa() - idx = soa.GetIdCPUData() + idx = soa.get_idcpu_data() if copy_to_host: data_array.append(idx.to_numpy(copy=True)) else: @@ -790,13 +790,13 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) comp_idx = part_container.num_int_comps - 1 for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): soa = pti.soa() - data_array.append(xp.array(soa.GetIntData(comp_idx), copy=False)) + data_array.append(xp.array(soa.get_int_data(comp_idx), copy=False)) else: container_wrapper = ParticleContainerWrapper(species_name) comp_idx = container_wrapper.particle_container.get_comp_index(comp_name) for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): soa = pti.soa() - data_array.append(xp.array(soa.GetRealData(comp_idx), copy=False)) + data_array.append(xp.array(soa.get_real_data(comp_idx), copy=False)) return data_array diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 8a9e35c6579..1079735279a 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "defb663d74ef9f50183b31c5dc9731cf6adb447c" +set(WarpX_pyamrex_branch "5aa700de18a61f933cb435adbe2299d74d794d6b" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") From 97e90bb48f89d9d975571ee14252b5b4f535e4ce Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 8 Feb 2024 10:18:08 -0800 Subject: [PATCH 146/176] AMReX: Weekly Update (#4678) --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- run_test.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 0e5b32c2de0..f77d06e87e5 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 296ed40e16ae1877640f5b78e9162dbd4ba1c279 && cd - + cd ../amrex && git checkout --detach 928a485af2949b2d41b20adc4585690908e41970 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 59123f0004b..9919912ece4 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 296ed40e16ae1877640f5b78e9162dbd4ba1c279 +branch = 928a485af2949b2d41b20adc4585690908e41970 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 42232d2a341..45f4218c162 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 296ed40e16ae1877640f5b78e9162dbd4ba1c279 +branch = 928a485af2949b2d41b20adc4585690908e41970 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 6c7c9466dfd..8928e628982 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "296ed40e16ae1877640f5b78e9162dbd4ba1c279" +set(WarpX_amrex_branch "928a485af2949b2d41b20adc4585690908e41970" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/run_test.sh b/run_test.sh index 6d8a1ddb014..a8f9f284283 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 296ed40e16ae1877640f5b78e9162dbd4ba1c279 && cd - +cd amrex && git checkout --detach 928a485af2949b2d41b20adc4585690908e41970 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 37a03387467564231f35f56a34623068f95a27fe Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 8 Feb 2024 14:15:49 -0800 Subject: [PATCH 147/176] Update openPMD-viewer documentation (#4682) --- Docs/source/dataanalysis/openpmdviewer.rst | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Docs/source/dataanalysis/openpmdviewer.rst b/Docs/source/dataanalysis/openpmdviewer.rst index efb5acb4e50..2e6f5c083d7 100644 --- a/Docs/source/dataanalysis/openpmdviewer.rst +++ b/Docs/source/dataanalysis/openpmdviewer.rst @@ -27,13 +27,7 @@ Usage ----- openPMD-viewer can be used either in simple Python scripts or in `Jupyter `__. -For interactive plots in Jupyter notebook, add this `"cell magic" `__ to the first line of your notebook: - -.. code-block:: python - - %matplotlib notebook - -and for Jupyter Lab use this instead: +For interactive plots in Jupyter notebook or Jupyter Lab, add this `"cell magic" `__ to the first line of your notebook: .. code-block:: python From c9bfc7eba9d2c34dcd2649b5f00d0eac8e07fc81 Mon Sep 17 00:00:00 2001 From: Olga Shapoval <30510597+oshapoval@users.noreply.github.com> Date: Thu, 8 Feb 2024 14:54:59 -0800 Subject: [PATCH 148/176] Apply filtering for the charge density in full diagnostics RZ PSATD (#4624) * Removed k-filtering when used RZ PSATD solver * Updated CI benchmarks for galilean_rz_psatd and multi_J_rz_psatd tests * Reversed k-space filtering for the charge density, but only in full diagnostics mode with RZ PSATD * Clean up * Updated CI benchmarks for galilean_rz_psatd and multi_J_rz_psatd tests * silence the unused variable warning * Clean up an include and move initialization to member initializer * Removed unnecessary includes. * Removed redundant class datamember. * Renamed a variable from apply_rho_filter to apply_rz_psatd_filter. --- .../benchmarks_json/galilean_rz_psatd.json | 34 +++++++++---------- .../benchmarks_json/multi_J_rz_psatd.json | 16 ++++----- Source/Diagnostics/BTDiagnostics.cpp | 2 +- .../ComputeDiagFunctors/RhoFunctor.H | 4 +++ .../ComputeDiagFunctors/RhoFunctor.cpp | 6 +++- Source/Diagnostics/FullDiagnostics.cpp | 11 +++--- .../Diagnostics/ReducedDiags/RhoMaximum.cpp | 2 +- 7 files changed, 41 insertions(+), 34 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/galilean_rz_psatd.json b/Regression/Checksum/benchmarks_json/galilean_rz_psatd.json index 0fb51eb303a..79af45a3f44 100644 --- a/Regression/Checksum/benchmarks_json/galilean_rz_psatd.json +++ b/Regression/Checksum/benchmarks_json/galilean_rz_psatd.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bt": 0.0001427751611710155, + "Er": 42705.92836687861, + "Et": 198517.8121599192, + "Ez": 4124.588455632591, + "divE": 50681.523267918594, + "jr": 7.07688551916584, + "jz": 469.93281767722846, + "rho": 4.488636594605648e-07 + }, "electrons": { - "particle_momentum_x": 7.188262495831869e-22, - "particle_momentum_y": 9.366670272998972e-24, - "particle_momentum_z": 8.903838605112867e-17, + "particle_momentum_x": 7.188262495831913e-22, + "particle_momentum_y": 9.366670273418532e-24, + "particle_momentum_z": 8.903838605112889e-17, "particle_position_x": 633733.1377069331, "particle_position_y": 7862152.008302979, "particle_theta": 51471.82060096203, "particle_weight": 1.0261080645329302e+20 }, "ions": { - "particle_momentum_x": 1.3139399113084718e-18, - "particle_momentum_y": 9.36530169181897e-24, + "particle_momentum_x": 1.313939911308472e-18, + "particle_momentum_y": 9.365301692233452e-24, "particle_momentum_z": 1.6348806629857118e-13, "particle_position_x": 633733.1365353089, "particle_position_y": 7862152.007285856, "particle_theta": 51471.6099169301, "particle_weight": 1.0261080645329302e+20 - }, - "lev=0": { - "Bt": 0.00014277516113528766, - "Er": 42705.928351478186, - "Et": 198517.8121599189, - "Ez": 4124.588451444761, - "divE": 50681.52326965637, - "jr": 7.0768855187909985, - "jz": 469.9328176825038, - "rho": 4.488636594800136e-07 - } -} + } + } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/multi_J_rz_psatd.json b/Regression/Checksum/benchmarks_json/multi_J_rz_psatd.json index 5ea6881b5ca..f30e773e7e0 100644 --- a/Regression/Checksum/benchmarks_json/multi_J_rz_psatd.json +++ b/Regression/Checksum/benchmarks_json/multi_J_rz_psatd.json @@ -1,13 +1,13 @@ { "lev=0": { - "Bt": 24080.41671546337, - "Er": 4536303784778.673, - "Ez": 4298815071343.0728, + "Bt": 24080.416715463354, + "Er": 4536303784778.672, + "Ez": 4298815071343.07, "jr": 361182004529074.8, - "jz": 1802215706551553.8, - "rho": 4884623.9573680265, + "jz": 1802215706551553.5, + "rho": 4884623.957368025, "rho_driver": 6288266.101815153, - "rho_plasma_e": 49568366.405371524, + "rho_plasma_e": 49568366.40537152, "rho_plasma_p": 50769182.21072973 }, "driver": { @@ -31,10 +31,10 @@ "plasma_e": { "particle_momentum_x": 6.655027717314839e-20, "particle_momentum_y": 6.730480164464723e-20, - "particle_momentum_z": 2.807381166958142e-20, + "particle_momentum_z": 2.8073811669581434e-20, "particle_position_x": 1.1423427658689635, "particle_position_y": 0.6140113094028803, "particle_theta": 20188.939948727297, "particle_weight": 1002457942911.3788 } -} +} \ No newline at end of file diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index f7965cd2688..16617159662 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -699,7 +699,7 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ - m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, -1, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, false, -1, false, ncomp); } } diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H index 31641fe23e9..bc0c8b9f270 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H @@ -27,6 +27,7 @@ public: */ RhoFunctor (int lev, amrex::IntVect crse_ratio, + bool apply_rz_psatd_filter = false, int species_index = -1, bool convertRZmodes2cartesian = true, int ncomp = 1); @@ -44,6 +45,9 @@ private: // Level on which source MultiFab mf_src is defined in RZ geometry int const m_lev; + // Whether to apply k-space filtering of charge density in the diagnostics output in RZ PSATD + bool m_apply_rz_psatd_filter; + // Species index to dump rho per species const int m_species_index; diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp index c36e31c0f93..340cc635fea 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp @@ -20,11 +20,13 @@ RhoFunctor::RhoFunctor (const int lev, const amrex::IntVect crse_ratio, + bool apply_rz_psatd_filter, const int species_index, bool convertRZmodes2cartesian, const int ncomp) : ComputeDiagFunctor(ncomp, crse_ratio), m_lev(lev), + m_apply_rz_psatd_filter(apply_rz_psatd_filter), m_species_index(species_index), m_convertRZmodes2cartesian(convertRZmodes2cartesian) {} @@ -62,7 +64,7 @@ RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp, const int /*i // Apply k-space filtering when using the PSATD solver if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { - if (WarpX::use_kspace_filter) { + if (WarpX::use_kspace_filter && m_apply_rz_psatd_filter) { auto & solver = warpx.get_spectral_solver_fp(m_lev); const SpectralFieldIndex& Idx = solver.m_spectral_index; solver.ForwardTransform(m_lev, *rho, Idx.rho_new); @@ -70,6 +72,8 @@ RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp, const int /*i solver.BackwardTransform(m_lev, *rho, Idx.rho_new); } } +#else + amrex::ignore_unused(m_apply_rz_psatd_filter); #endif InterpolateMFForDiag(mf_dst, *rho, dcomp, warpx.DistributionMap(m_lev), diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index fd329a38220..a011d1c1ecf 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -1,5 +1,4 @@ #include "FullDiagnostics.H" - #include "ComputeDiagFunctors/CellCenterFunctor.H" #include "ComputeDiagFunctors/DivBFunctor.H" #include "ComputeDiagFunctors/DivEFunctor.H" @@ -269,14 +268,14 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } } else if ( m_varnames_fields[comp] == "rho" ){ // Initialize rho functor to dump total rho - m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, -1, + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, -1, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("rho"), ncomp); } } else if ( m_varnames_fields[comp].rfind("rho_", 0) == 0 ){ // Initialize rho functor to dump rho per species - m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_rho_per_species_index[i], + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, m_rho_per_species_index[i], false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("rho_") + m_all_species_names[m_rho_per_species_index[i]], ncomp); @@ -444,7 +443,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) } // rho if (rho_requested) { - m_all_field_functors[lev][icomp] = std::make_unique(lev, m_crse_ratio, -1, false, ncomp_multimodefab); + m_all_field_functors[lev][icomp] = std::make_unique(lev, m_crse_ratio, true, -1, false, ncomp_multimodefab); icomp += 1; AddRZModesToOutputNames(std::string("rho"), ncomp_multimodefab); } @@ -641,10 +640,10 @@ FullDiagnostics::InitializeFieldFunctors (int lev) m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "rho" ){ // Initialize rho functor to dump total rho - m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true); } else if ( m_varnames[comp].rfind("rho_", 0) == 0 ){ // Initialize rho functor to dump rho per species - m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_rho_per_species_index[i]); + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, m_rho_per_species_index[i]); i++; } else if ( m_varnames[comp] == "F" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_F_fp(lev), lev, m_crse_ratio); diff --git a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp index 787a95ce412..4613a70b3b0 100644 --- a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp @@ -76,7 +76,7 @@ RhoMaximum::RhoMaximum (std::string rd_name) for (int lev = 0; lev < nLevel; ++lev) { // Initialize functors for the charge density of each charged species - m_rho_functors[lev].push_back(std::make_unique(lev, crse_ratio, i)); + m_rho_functors[lev].push_back(std::make_unique(lev, crse_ratio, false, i)); } } } From 4077f8e3e7193ba3b292ce133159d094fb5db96e Mon Sep 17 00:00:00 2001 From: Eya D <81635404+EyaDammak@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:23:16 -0800 Subject: [PATCH 149/176] Add automated test for spacecraft charging (#4413) * Adding of the bool `radially_weighted` as variable in the class Species * Remove error message * Fix bug with radially-weighted * Fix weighting of particles * ghost_cells problems fixed * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * modification of particle_containers to track the particle * Add PICMI interface for BoundaryScrapingDiagnostic * Add first iteration of the input script * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * second draft, < 2 min, good results * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * adding a draft of analysis code for spacecraft_charging * adding spacecraft_charging at the very end * adding spacecraft_charging analysis * Revert "Add PICMI interface for BoundaryScrapingDiagnostic" This reverts commit 521dc5dfe3ada534a65bf11a0abe8897e94d5337. * Revert changes in fields.py * Update PlasmaInjector.cpp Trying to fix the error during the test: /home/runner/work/WarpX/WarpX/Source/Initialization/PlasmaInjector.cpp:72:5: warning: 'zmax' should be initialized in a member initializer of the constructor [cppcoreguidelines-prefer-member-initializer] zmax = std::numeric_limits::max(); * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update PlasmaInjector.cpp Adding "amrex::Real" to define xmin,ymin,zmin,xmax,ymax,zmax * Update PlasmaInjector.cpp * modelisation code * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * new version gathering the electrons and the protons * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * deplaced in the Physics_applications file * not necessary * analysis file * changes in some syntaxes and adding checksum test * add png file * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * replaced by new syntax * change of syntax and keep just one version * Add checksum json file * Update PICMI_inputs_rz.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove changes related to radially-weighted * Update Regression/WarpX-tests.ini * Make script executable * Update format of automated test * Update test * adding description * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update PICMI_inputs_rz.py * Update analysis.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update PICMI_inputs_rz.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update PICMI_inputs_rz.py * Update PICMI_inputs_rz.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update PICMI_inputs_rz.py * analysis changed: I wish it works now * Update WarpX-tests.ini adding the outputFile * Update filename * Update checksum * Update Examples/Physics_applications/spacecraft_charging/analysis.py * Update PICMI_inputs_rz.py From a random to an identical test * Update analysis.py * Update spacecraft_charging.json * Update spacecraft_charging.json * Update analysis.py --------- Co-authored-by: Remi Lehe Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../spacecraft_charging/PICMI_inputs_rz.py | 299 ++++++++++++++++++ .../spacecraft_charging/analysis.py | 78 +++++ .../benchmarks_json/spacecraft_charging.json | 32 ++ Regression/WarpX-tests.ini | 22 ++ 4 files changed, 431 insertions(+) create mode 100644 Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py create mode 100755 Examples/Physics_applications/spacecraft_charging/analysis.py create mode 100644 Regression/Checksum/benchmarks_json/spacecraft_charging.json diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py new file mode 100644 index 00000000000..08ebe23f828 --- /dev/null +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +# +# --- Input file for spacecraft charging testing in RZ. +# --- This input defines a conducting sphere (spacecraft) immersed in a thermal +# --- plasma with the same given initial conditions as in the article: +# --- (*) J. Deca, G. Lapenta, R. Marchand, S. Markidis; +# --- Spacecraft charging analysis with the implicit particle-in-cell code iPic3D. +# --- Part III. A. pages 3-4 +# --- Phys. Plasmas 1 October 2013; 20 (10): 102902. https://doi.org/10.1063/1.4826951. +# --- The conducting sphere starts with an initial potential of 1V and will interact with +# --- the surrounding plasma, initially static. The charging of the spacecraft - by accumulation +# --- of electrons - leads to a decrease of the potential on the surface over the time +# --- until reaching an equilibrium floating potential of ~144.5 V (*). + +from mpi4py import MPI as mpi +import numpy as np +import scipy.constants as scc + +from pywarpx import picmi +from pywarpx.callbacks import installafterEsolve, installafterInitEsolve +from pywarpx.fields import ExWrapper, EzWrapper, PhiFPWrapper, RhoFPWrapper +from pywarpx.particle_containers import ParticleBoundaryBufferWrapper + + +# Utilities +class SpaceChargeFieldCorrector(object): + """ + Class used by the callback functions to calculate the + correct charge on the spacecraft at each initialisation. + """ + def __init__(self): + self.saved_first_iteration_fields = False + self.spacecraft_potential = 1. # Initial voltage: 1V + self.spacecraft_capacitance = None + + def correct_space_charge_fields(self, q=None): + """ + Function that will be called at each iteration, + after each electrostatic solve in WarpX + """ + assert self.saved_first_iteration_fields + + # Compute the charge that WarpX thinks there is on the spacecraft + # from phi and rho after the Poisson solver + q_v = compute_virtual_charge_on_spacecraft() + if q is None: + q = compute_actual_charge_on_spacecraft() + + # Correct fields so as to recover the actual charge + Er = ExWrapper(include_ghosts=True)[:,:] + Er[...] = Er[...]+(q - q_v)*self.normalized_Er[...] + Ez = EzWrapper(include_ghosts=True)[:,:] + Ez[...] += (q - q_v)*self.normalized_Ez[...] + phi = PhiFPWrapper(include_ghosts=True)[:,:] + phi[...] += (q - q_v)*self.normalized_phi[...] + self.spacecraft_potential += (q - q_v)*self.spacecraft_capacitance + sim.extension.warpx.set_potential_on_eb( "%f" %self.spacecraft_potential ) + print('Setting potential to %f' %self.spacecraft_potential) + + # Confirm that the charge on the spacecraft is now correct + compute_virtual_charge_on_spacecraft() + + def save_normalized_vacuum_Efields(self,): + # Compute the charge that WarpX thinks there is on the spacecraft + # from phi and rho after the Poisson solver + q_v = compute_virtual_charge_on_spacecraft() + self.spacecraft_capacitance = 1./q_v # the potential was set to 1V + + # Check that this iteration corresponded to a vacuum solve + rho = RhoFPWrapper(include_ghosts=False) + + # In principle, we should check that `rho` is exactly 0 + # However, due to machine precision errors when adding the charge + # of ions and electrons, this can be slightly different than 0 + assert np.all( abs(rho[...]) < 1.e-11 ) + + # Record fields + Er = ExWrapper(include_ghosts=True)[:,:] + self.normalized_Er = Er[...] /q_v + Ez = EzWrapper(include_ghosts=True)[:,:] + self.normalized_Ez = Ez[...] /q_v + phi = PhiFPWrapper(include_ghosts=True)[:,:] + self.normalized_phi = phi[...] /q_v + + self.saved_first_iteration_fields = True + self.correct_space_charge_fields(q=0) + + +def compute_virtual_charge_on_spacecraft(): + """ + Given that we asked WarpX to solve the Poisson + equation with phi=1 on the spacecraft and phi=0 + on the boundary of the domain, compute the charge + that WarpX thinks there should be on the spacecraft. + """ + # Get global array for the whole domain (across MPI ranks) + phi = PhiFPWrapper(include_ghosts=False)[:,:] + rho = RhoFPWrapper(include_ghosts=False)[:,:] + + # Check that this codes correspond to the global size of the box + assert phi.shape == (nr+1, nz+1) + assert rho.shape == (nr+1, nz+1) + + dr, dz = sim.extension.warpx.Geom(lev=0).data().CellSize() + + # Compute integral of grad phi over surfaces of the domain + r = np.linspace(rmin, rmax, len(phi), endpoint=False) + (rmax - rmin) / (2 * len(phi)) #shift of the r points because the derivaties are calculated in the middle + face_z0 = 2 * np.pi * 1./dz * ( (phi[:,0]-phi[:,1]) * r ).sum() * dr #here I am assuming that phi is a numpy array that can handle elementwise mult + face_zend = 2 * np.pi * 1./dz * ( (phi[:,-1]-phi[:,-2]) * r ).sum() * dr + face_rend = 2 * np.pi * 1./dr*((phi[-1,:]-phi[-2,:]) * rmax).sum() * dz + grad_phi_integral = face_z0 + face_zend + face_rend + + # Compute integral of rho over volume of the domain + # (i.e. total charge of the plasma particles) + rho_integral = 0.0 + for k in range(1, nz-1): + for i in range(1, nr-1): + rho_integral += rho[i,k] * r[i] * dr * dz + + # Due to an oddity in WarpX (which will probably be solved later) + # we need to multiply `rho` by `-epsilon_0` to get the correct charge + rho_integral *= 2 * np.pi * -scc.epsilon_0 #does this oddity still exist? + + # Compute charge of the spacecraft, based on Gauss theorem + q_spacecraft = - rho_integral - scc.epsilon_0 * grad_phi_integral + print('Virtual charge on the spacecraft: %e' %q_spacecraft) + return q_spacecraft + + +def compute_actual_charge_on_spacecraft(): + """ + Compute the actual charge on the spacecraft, + by counting how many electrons and protons + were collected by the WarpX embedded boundary (EB) + """ + charge = {'electrons': -scc.e, 'protons': scc.e} + q_spacecraft = 0 + particle_buffer = ParticleBoundaryBufferWrapper() + for species in charge.keys(): + weights = particle_buffer.get_particle_boundary_buffer(species, 'eb', 'w', 0) + sum_weights_over_tiles = sum([w.sum() for w in weights]) + + # Reduce across all MPI ranks + ntot = float(mpi.COMM_WORLD.allreduce(sum_weights_over_tiles, op=mpi.SUM)) + print('Total number of %s collected on spacecraft: %e'%(species, ntot)) + q_spacecraft += ntot * charge[species] + + print('Actual charge on the spacecraft: %e' %q_spacecraft) + return q_spacecraft + + +########################## +# numerics parameters +########################## + +dt=1.27e-8 + +# --- Nb time steps +max_steps = 1000 +diagnostic_interval = 10 + +# --- grid +nr = 40 +nz= 80 + +rmin = 0.0 +rmax = 3 +zmin = -3 +zmax = 3 + +number_per_cell =5 +number_per_cell_each_dim = [10,1, 1] + + +########################## +# physics components +########################## + +n = 7.0e9 #plasma density #particles/m^3 +Te = 85 #Electron temp in eV +Ti = 0.05 * Te #Ion temp in eV +qe = picmi.constants.q_e #elementary charge +m_e = picmi.constants.m_e #electron mass +m_i = 1836.0 * m_e #mass of ion +v_eth = (qe * Te / m_e) ** 0.5 +v_pth = (qe * Ti / m_i) ** 0.5 + +# nothing to change in the distribution function? +e_dist = picmi.UniformDistribution(density = n, rms_velocity=[v_eth, v_eth, v_eth] ) +e_dist2 = picmi.UniformFluxDistribution( + flux=n*v_eth/(2*np.pi)**.5, # Flux for Gaussian with vmean=0 + surface_flux_position=3, + flux_direction=-1, flux_normal_axis='r', + gaussian_flux_momentum_distribution=True, + rms_velocity=[v_eth, v_eth, v_eth] ) +electrons = picmi.Species(particle_type='electron', + name='electrons', + initial_distribution=[e_dist,e_dist2], + warpx_save_particles_at_eb=1) + +p_dist = picmi.UniformDistribution(density = n, rms_velocity=[v_pth, v_pth, v_pth] ) +p_dist2 = picmi.UniformFluxDistribution( + flux=n*v_pth/(2*np.pi)**.5, # Flux for Gaussian with vmean=0 + surface_flux_position=3, + flux_direction=-1, flux_normal_axis='r', + gaussian_flux_momentum_distribution=True, + rms_velocity=[v_pth, v_pth, v_pth] ) +protons = picmi.Species(particle_type='proton', + name='protons', + initial_distribution=[p_dist,p_dist2], + warpx_save_particles_at_eb=1) + + +########################## +# numerics components +########################## + +grid = picmi.CylindricalGrid( + number_of_cells = [nr, nz], + n_azimuthal_modes = 1, + lower_bound = [rmin, zmin], + upper_bound = [rmax, zmax], + lower_boundary_conditions = ['none', 'dirichlet'], + upper_boundary_conditions = ['dirichlet', 'dirichlet'], + lower_boundary_conditions_particles = ['absorbing', 'reflecting'], + upper_boundary_conditions_particles = ['absorbing', 'reflecting'] +) + +solver = picmi.ElectrostaticSolver( + grid=grid, method='Multigrid', + warpx_absolute_tolerance=1e-7 +) + +embedded_boundary = picmi.EmbeddedBoundary( + implicit_function="-(x**2+y**2+z**2-radius**2)", + potential=1., # arbitrary value ; this will be corrected by a callback function + radius = 0.3277 +) + + +########################## +# diagnostics +########################## + +field_diag = picmi.FieldDiagnostic( + name = 'diag1', + grid = grid, + period = diagnostic_interval, + data_list = ['Er', 'Ez', 'phi', 'rho', + 'rho_electrons', 'rho_protons'], + warpx_format = 'openpmd', + write_dir = '.', + warpx_file_prefix = 'spacecraft_charging_plt' +) + +part_diag = picmi.ParticleDiagnostic(name = 'diag1', + period = diagnostic_interval, + species = [electrons, protons], + warpx_format = 'openpmd', + write_dir = '.', + warpx_file_prefix = 'spacecraft_charging_plt' +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver = solver, + time_step_size = dt, + max_steps = max_steps, + warpx_embedded_boundary=embedded_boundary, + warpx_amrex_the_arena_is_managed=1, + warpx_random_seed=1 +) + +layout1=picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, + grid=grid) +layout2=picmi.PseudoRandomLayout(n_macroparticles_per_cell=number_per_cell, + grid=grid) +sim.add_species(electrons, + layout = [layout1,layout2]) + +sim.add_species(protons, + layout = [layout1,layout2]) + +sim.add_diagnostic(field_diag) +sim.add_diagnostic(part_diag) + +########################## +# simulation run +########################## + +spc = SpaceChargeFieldCorrector() + +installafterInitEsolve( spc.save_normalized_vacuum_Efields ) +installafterEsolve( spc.correct_space_charge_fields ) + +sim.step(max_steps) diff --git a/Examples/Physics_applications/spacecraft_charging/analysis.py b/Examples/Physics_applications/spacecraft_charging/analysis.py new file mode 100755 index 00000000000..d8e8ac86af8 --- /dev/null +++ b/Examples/Physics_applications/spacecraft_charging/analysis.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +""" +This script tests the potential-time profile on the surface of +a conducting sphere (spacecraft) immersed in an initially static +thermal plasma. The potential on the spacecraft decreases over +the time to reach an equilibrium floating potential. + +An input Python file PICMI_inputs_rz.py is used. + +The test will check the curve fitting parameters v0 and tau defined +by the following exponential function: phi(t)=v0(1-exp(-t/tau)) +""" + +import os +import sys + +import matplotlib.pyplot as plt +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.optimize import curve_fit +import yt + +yt.funcs.mylog.setLevel(0) +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# Open plotfile specified in command line +filename = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename, output_format='openpmd') + +ts = OpenPMDTimeSeries('./spacecraft_charging_plt') +dt = 1.27e-8 +t=[] +phi=[] +it = ts.iterations + +for i in it: + phi_i = ts.get_field('phi',iteration=i,plot=False) + # Find the minimum value among all grids for this iteration + phi_min = np.min(phi_i[0]) + phi.append(phi_min) + t.append(dt*i) + + +def func(x, v0, tau): + + return v0 * (1-np.exp(-np.array(x) / tau)) + + + +popt, pcov = curve_fit(func, t, phi) + +plt.plot(t,phi, label='modelisation') +plt.plot(t, func(t, *popt), 'r-',label='fit: v0=%5.3f, tau=%5.9f' % (popt[0], popt[1])) +plt.legend() +plt.savefig('min_phi_analysis.png') + +print('fit parameters between the min(phi) curve over the time and the function v0(1-exp(-t/tau)):') +print('v0=%5.3f, tau=%5.9f' % (popt[0], popt[1])) + + +tolerance_v0=0.01 +tolerance_tau=0.01 +print("tolerance for v0 = "+ str(tolerance_v0 *100) + '%') +print("tolerance for tau = "+ str(tolerance_tau*100) + '%') + +mean_v0=-151.347 +mean_tau=0.000004351 + +diff_v0=np.abs((popt[0]-mean_v0)/mean_v0) +diff_tau=np.abs((popt[1]-mean_tau)/mean_tau) + +print("percentage error for v0 = "+ str(diff_v0 *100) + '%') +print("percentage error for tau = "+ str(diff_tau*100) + '%') + +assert (diff_v0 < tolerance_v0) and (diff_tau < tolerance_tau), 'Test spacecraft_charging did not pass' diff --git a/Regression/Checksum/benchmarks_json/spacecraft_charging.json b/Regression/Checksum/benchmarks_json/spacecraft_charging.json new file mode 100644 index 00000000000..7534e0b8d35 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/spacecraft_charging.json @@ -0,0 +1,32 @@ +{ + "lev=0": { + "Er": 75713.05000099652, + "Ez": 75260.78239853957, + "phi": 55650.30604185804, + "rho": 1.4793075271598396e-06, + "rho_electrons": 6.506538129003745e-06, + "rho_protons": 6.98347902659172e-06 + }, + "electrons": { + "particle_theta": 46360.98014353502, + "particle_position_x": 59656.07603558921, + "particle_position_y": 0.0, + "particle_position_z": 45010.371467374425, + "particle_momentum_x": 8.27307207197173e-20, + "particle_momentum_y": 8.264475255806164e-20, + "particle_momentum_z": 8.271327169054914e-20, + "particle_weight": 1140673608016.2212 + }, + "protons": { + "particle_theta": 673143.6230804407, + "particle_position_x": 1180416.5550460955, + "particle_position_y": 0.0, + "particle_position_z": 644420.0485148785, + "particle_momentum_x": 1.468116154656724e-17, + "particle_momentum_y": 1.4650318746367807e-17, + "particle_momentum_z": 1.1638654342620123e-17, + "particle_weight": 1175692137613.312 + } +} + + diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 45f4218c162..7dd0b56c0ab 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4498,6 +4498,28 @@ doVis = 0 outputFile = BeamBeamCollision_plt analysisRoutine = Examples/analysis_default_openpmd_regression.py +[spacecraft_charging] +buildDir = . +inputFile = Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +runtime_params = +customRunCmd = python3 PICMI_inputs_rz.py +dim = 2 +addToCompileString = USE_PYTHON_MAIN=TRUE USE_RZ=TRUE +cmakeSetupOpts = -DWarpX_DIMS="RZ" -DWarpX_EB=ON -DWarpX_PYTHON=ON +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons protons +outputFile = spacecraft_charging_plt +analysisRoutine = Examples/Physics_applications/spacecraft_charging/analysis.py +analysisOutputImage = min_phi_analysis.png + [ImplicitPicard_1d] buildDir = . inputFile = Examples/Tests/Implicit/inputs_1d From 9684a17804054bb3a54a72bfca9b7724ccab7bd1 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 9 Feb 2024 19:45:45 -0800 Subject: [PATCH 150/176] Fix: openPMD RZ Positions (#4686) * Fix: openPMD RZ Positions Fix the particle position write in openPMD with SoA particle layout. For the openPMD I/O backend, we transform the r,z,theta particle components back to x,y,z. * Updates for CI --- .../benchmarks_json/spacecraft_charging.json | 10 +- Source/Diagnostics/WarpXOpenPMD.cpp | 104 +++++++++++++----- 2 files changed, 78 insertions(+), 36 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/spacecraft_charging.json b/Regression/Checksum/benchmarks_json/spacecraft_charging.json index 7534e0b8d35..d9f753e1df2 100644 --- a/Regression/Checksum/benchmarks_json/spacecraft_charging.json +++ b/Regression/Checksum/benchmarks_json/spacecraft_charging.json @@ -8,9 +8,8 @@ "rho_protons": 6.98347902659172e-06 }, "electrons": { - "particle_theta": 46360.98014353502, - "particle_position_x": 59656.07603558921, - "particle_position_y": 0.0, + "particle_position_x": 38158.7364935527, + "particle_position_y": 37779.25499255196, "particle_position_z": 45010.371467374425, "particle_momentum_x": 8.27307207197173e-20, "particle_momentum_y": 8.264475255806164e-20, @@ -18,9 +17,8 @@ "particle_weight": 1140673608016.2212 }, "protons": { - "particle_theta": 673143.6230804407, - "particle_position_x": 1180416.5550460955, - "particle_position_y": 0.0, + "particle_position_x": 751407.372588289, + "particle_position_y": 751687.788498272, "particle_position_z": 644420.0485148785, "particle_momentum_x": 1.468116154656724e-17, "particle_momentum_y": 1.4650318746367807e-17, diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 39717ef6ec5..e284e254d1e 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -10,6 +10,7 @@ #include "Diagnostics/ParticleDiag/ParticleDiag.H" #include "FieldIO.H" #include "Particles/Filter/FilterFunctors.H" +#include "Particles/NamedComponentParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/RelativeCellPosition.H" @@ -548,10 +549,11 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { // see openPMD ED-PIC extension for namings // note: an underscore separates the record name from its component // for non-scalar records + // note: in RZ, we reconstruct x,y,z positions from r,z,theta in WarpX #if !defined (WARPX_DIM_1D_Z) real_names.push_back("position_x"); #endif -#if defined (WARPX_DIM_3D) +#if defined (WARPX_DIM_3D) || defined(WARPX_DIM_RZ) real_names.push_back("position_y"); #endif real_names.push_back("position_z"); @@ -559,9 +561,7 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); -#ifdef WARPX_DIM_RZ - real_names.push_back("theta"); -#endif + // get the names of the real comps real_names.resize(tmp.NumRealComps()); auto runtime_rnames = tmp.getParticleRuntimeComps(); @@ -881,43 +881,87 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, amrex::Vector const& int_comp_names) const { - auto const numParticleOnTile = pti.numParticles(); - auto const numParticleOnTile64 = static_cast(numParticleOnTile); - auto const& soa = pti.GetStructOfArrays(); + auto const numParticleOnTile = pti.numParticles(); + auto const numParticleOnTile64 = static_cast(numParticleOnTile); + auto const& soa = pti.GetStructOfArrays(); - auto const getComponentRecord = [&currSpecies](std::string const comp_name) { - // handle scalar and non-scalar records by name - const auto [record_name, component_name] = detail::name2openPMD(comp_name); - return currSpecies[record_name][component_name]; - }; + auto const getComponentRecord = [&currSpecies](std::string const comp_name) { + // handle scalar and non-scalar records by name + const auto [record_name, component_name] = detail::name2openPMD(comp_name); + return currSpecies[record_name][component_name]; + }; // here we the save the SoA properties (idcpu) { // todo: add support to not write the particle index getComponentRecord("id").storeChunkRaw( - soa.GetIdCPUData().data(), {offset}, {numParticleOnTile64}); + soa.GetIdCPUData().data(), {offset}, {numParticleOnTile64}); } - // here we the save the SoA properties (real) - { - auto const real_counter = std::min(write_real_comp.size(), real_comp_names.size()); - for (auto idx=0; idx x,y,z +#if defined(WARPX_DIM_RZ) + auto const * const r = soa.GetRealData(PIdx::x).data(); + auto const * const theta = soa.GetRealData(PIdx::theta).data(); + + if (write_real_comp[0]) { + std::shared_ptr const x( + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + for (int i = 0; i < numParticleOnTile; ++i) { + x.get()[i] = r[i] * std::cos(theta[i]); + } + getComponentRecord(real_comp_names[0]).storeChunk( + x, {offset}, {numParticleOnTile64}); + } + if (write_real_comp[1]) { + std::shared_ptr const y( + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + for (int i = 0; i < numParticleOnTile; ++i) { + y.get()[i] = r[i] * std::sin(theta[i]); + } + getComponentRecord(real_comp_names[1]).storeChunk( + y, {offset}, {numParticleOnTile64}); + } +#endif + + for (auto idx=0; idx Date: Mon, 12 Feb 2024 13:59:49 -0800 Subject: [PATCH 151/176] AMReX: Weekly Update (#4690) --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- run_test.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index f77d06e87e5..e7ecf36a034 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 928a485af2949b2d41b20adc4585690908e41970 && cd - + cd ../amrex && git checkout --detach 68244ec91d118b5d4cc21f93376eaae8b56c51eb && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 9919912ece4..2760beed53b 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 928a485af2949b2d41b20adc4585690908e41970 +branch = 68244ec91d118b5d4cc21f93376eaae8b56c51eb [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 7dd0b56c0ab..627ba11bb5e 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 928a485af2949b2d41b20adc4585690908e41970 +branch = 68244ec91d118b5d4cc21f93376eaae8b56c51eb [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 8928e628982..9db2b1f1aad 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "928a485af2949b2d41b20adc4585690908e41970" +set(WarpX_amrex_branch "68244ec91d118b5d4cc21f93376eaae8b56c51eb" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/run_test.sh b/run_test.sh index a8f9f284283..a0ba9e056a5 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 928a485af2949b2d41b20adc4585690908e41970 && cd - +cd amrex && git checkout --detach 68244ec91d118b5d4cc21f93376eaae8b56c51eb && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 7392bc9a765ddfb8a3959d544488da76fcc78271 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 12 Feb 2024 15:47:43 -0800 Subject: [PATCH 152/176] Lassen (LLNL) TOSS3: Update (#4692) Update small typos and simplify source directory. --- Docs/source/install/hpc/lassen.rst | 2 +- .../lassen-llnl/lassen_v100_warpx_toss3.profile.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index 7285bc217f2..d8565f06983 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -178,7 +178,7 @@ Finally, since lassen does not yet provide software modules for some of our depe .. code-block:: bash bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh - source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen-toss3/bin/activate + source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate .. dropdown:: Script Details :color: light diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example index 19c74348a99..abad909943c 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example @@ -10,7 +10,7 @@ module load cuda/12.0.0 module load boost/1.70.0 # optional: for openPMD support -SRC_DIR="/usr/workspace/${USER}/lassen-toss3/src" +SRC_DIR="/usr/workspace/${USER}/lassen/src" SW_DIR="/usr/workspace/${USER}/lassen-toss3/gpu" export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH From c5aea94ae14d690e1d563b026b0c8b1ad39b6f2e Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 12 Feb 2024 16:11:42 -0800 Subject: [PATCH 153/176] Bugfix: sum guard cell values in current density diagnostic (#4596) --- Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index b21bbded011..9938cf095bd 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -52,6 +52,12 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ auto& mypc = warpx.GetPartContainer(); mypc.DepositCurrent(current_fp_temp, warpx.getdt(m_lev), 0.0); + + // sum values in guard cells - note that this does not filter the + // current density. + for (int idim = 0; idim < 3; ++idim) { + current_fp_temp[0][idim]->FillBoundary(warpx.Geom(m_lev).periodicity()); + } } InterpolateMFForDiag(mf_dst, *m_mf_src, dcomp, warpx.DistributionMap(m_lev), From 043fc63ab1d55a5c1422d1aef17f1dd097555295 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 12 Feb 2024 16:23:56 -0800 Subject: [PATCH 154/176] HPC3 (UCI): `-j 8` (#4694) Reduce the compile parallelism to avoid getting build processes killed by the watchdog daemon. --- Docs/source/install/hpc/hpc3.rst | 4 ++-- Tools/machines/hpc3-uci/install_gpu_dependencies.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Docs/source/install/hpc/hpc3.rst b/Docs/source/install/hpc/hpc3.rst index 33296e92b3d..e2f900de273 100644 --- a/Docs/source/install/hpc/hpc3.rst +++ b/Docs/source/install/hpc/hpc3.rst @@ -96,7 +96,7 @@ Use the following :ref:`cmake commands ` to compile the applicat rm -rf build cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build -j 12 + cmake --build build -j 8 The WarpX application executables are now in ``$HOME/src/warpx/build/bin/``. Additionally, the following commands will install WarpX as a Python module: @@ -106,7 +106,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_py cmake -S . -B build_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_py -j 12 --target pip_install + cmake --build build_py -j 8 --target pip_install Now, you can :ref:`submit HPC3 compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit HPC3 jobs (:ref:`example inputs `). diff --git a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh index 9334d0a2287..a2b9c7b3855 100755 --- a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh +++ b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh @@ -56,7 +56,7 @@ else fi rm -rf $HOME/src/c-blosc-pm-gpu-build cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-pm-gpu-build --target install --parallel 12 +cmake --build $HOME/src/c-blosc-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/c-blosc-pm-gpu-build # ADIOS2 @@ -71,7 +71,7 @@ else fi rm -rf $HOME/src/adios2-pm-gpu-build cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-pm-gpu-build --target install --parallel 12 +cmake --build $HOME/src/adios2-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/adios2-pm-gpu-build # BLAS++ (for PSATD+RZ) @@ -87,7 +87,7 @@ else fi rm -rf $HOME/src/blaspp-pm-gpu-build cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 12 +cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/blaspp-pm-gpu-build # LAPACK++ (for PSATD+RZ) @@ -103,7 +103,7 @@ else fi rm -rf $HOME/src/lapackpp-pm-gpu-build CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 12 +cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/lapackpp-pm-gpu-build From 5248fc3b75da508e3e499eb82c82cf5c04c82bbf Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 12 Feb 2024 16:24:48 -0800 Subject: [PATCH 155/176] Add PICMI documentation for ReducedDiagnostics (#4693) --- Docs/source/usage/python.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 448800f0d4d..b18606c9a53 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -90,6 +90,8 @@ Diagnostics .. autoclass:: pywarpx.picmi.Checkpoint +.. autoclass:: pywarpx.picmi.ReducedDiagnostic + Lab-frame diagnostics diagnostics are used when running boosted-frame simulations. .. autoclass:: pywarpx.picmi.LabFrameFieldDiagnostic From 7740464359e6472c5ead8ba7c62dc13537bca6fd Mon Sep 17 00:00:00 2001 From: Eya D <81635404+EyaDammak@users.noreply.github.com> Date: Tue, 13 Feb 2024 11:25:32 -0800 Subject: [PATCH 156/176] Obtain exact point of contact of Particles with EB (#4560) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * modifying the function ParticleSraper using bisection method * adding a variable const amrex::dt for the ScraperParticle function * adding the variable dt to ScraperParticle function * adding the variable dt in the ScraperParticle function * not real change * Fix compilation * Remove mutable * new folder for the test of calculation of the point of contact in the case of one particle and EB * make it quicker * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * make it quicker * case in 3D * deleating diags files * fix some issues * EB as a sphere * first draft of analysis * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add of getPosition but doesn t work * some changes * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * version Rémi non modifiée * reset the version of ParticleScraper.H with the branch development * adding the dt variable to make the file consistent with the modifications from the branch bisection * still 2 errors * still one error * some changes but still errors * WORKSgit add Source/Particles/ParticleBoundaryBuffer.cpp, but only in CPU * Update ParticleBoundaryBuffer.cpp Make it work for GPU * works for GPU * works for GPU * works * works but not for RZ * adding a test for a tolerance of 0.1% * comma missed * addind checkout + change of some details * add point_of_contact_EB_3d * updates with checksum and changes in calling diags folders * ready to be tested * ready to be tested * midification of one comment * adding test for 3d * the code works for RZ * json file for the point_of_contact test in rz * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Delete Examples/Tests/point_of_contact_EB/.ipynb_checkpoints/inputs_rz-checkpoint * Update analysis.py * Update ParticleBoundaryBuffer.cpp * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestions from code review * Update Source/EmbeddedBoundary/ParticleScraper.H * Update Source/EmbeddedBoundary/ParticleScraper.H * Update Source/EmbeddedBoundary/ParticleScraper.H * Update Source/Particles/ParticleBoundaryBuffer.cpp * Add type for phi_value * Fix compilation * Update ParticleScraper.H lines updated for OpenMP parallelized * Update Source/Particles/ParticleBoundaryBuffer.cpp * Avoid GPU compilation errors * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Source/Particles/ParticleBoundaryBuffer.cpp * Update Source/Particles/ParticleBoundaryBuffer.cpp * Update Source/Particles/ParticleBoundaryBuffer.cpp * Update WarpX-test.ini * Remove particles that are initialized in the EB * Fix error whereby the momentum was not particle-dependent * Add back the BeamBeam test * Apply suggestions from code review * Make analysis script executable * Correct format of analysis file * Update ParticleBoundaryBuffer.cpp syntax changed for saving the exact position * Update ParticleBoundaryBuffer.cpp try an other syntax for the positions * Update ParticleBoundaryBuffer.cpp fix a error * Update ParticleBoundaryBuffer.cpp change syntax due to new commits in WarpX * Update ParticleBoundaryBuffer.cpp little error in taping fixed * Update ParticleBoundaryBuffer.cpp fix issue in RZ. Now 'theta' --> 'y' in RZ * back to the previous version theta='theta' in RZ * udate syntax to be consistent with the merge * Update ParticleScraper.H * Update analysis.py * Update ParticleBoundaryBuffer.cpp coherent with the updates of development * modification due to the merge * Update analysis.py * Update analysis.py * Update Point_of_contact_EB_rz.json * changes of coordinates analysis in RZ * Update Point_of_contact_EB_rz.json * Update WarpXOpenPMD.cpp * Update Point_of_contact_EB_rz.json * Update analysis_rz.py * delete in tests in RZ * deleting RZ tests in Regression file * Update WarpXOpenPMD.cpp * Update analysis.py * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * Update WarpX-tests.ini * Update Examples/Tests/point_of_contact_EB/analysis.py Co-authored-by: Remi Lehe * Update analysis.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update analysis.py * Update analysis.py * Update Examples/Tests/point_of_contact_EB/inputs_3d Co-authored-by: Remi Lehe * Update Examples/Tests/point_of_contact_EB/inputs_3d Co-authored-by: Remi Lehe * Update Source/Particles/ParticleBoundaryBuffer.cpp Co-authored-by: Remi Lehe * Update Source/Particles/ParticleBoundaryBuffer.cpp Co-authored-by: Remi Lehe * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp --------- Co-authored-by: Remi Lehe Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../Tests/point_of_contact_EB/analysis.py | 51 ++++++++++ Examples/Tests/point_of_contact_EB/inputs_3d | 48 ++++++++++ .../Point_of_contact_EB_3d.json | 14 +++ Regression/WarpX-tests.ini | 19 ++++ Source/EmbeddedBoundary/ParticleScraper.H | 22 ++--- Source/Particles/ParticleBoundaryBuffer.cpp | 92 ++++++++++++++++++- Source/Particles/Pusher/UpdatePosition.H | 1 + 7 files changed, 235 insertions(+), 12 deletions(-) create mode 100755 Examples/Tests/point_of_contact_EB/analysis.py create mode 100644 Examples/Tests/point_of_contact_EB/inputs_3d create mode 100644 Regression/Checksum/benchmarks_json/Point_of_contact_EB_3d.json diff --git a/Examples/Tests/point_of_contact_EB/analysis.py b/Examples/Tests/point_of_contact_EB/analysis.py new file mode 100755 index 00000000000..fd4782241b2 --- /dev/null +++ b/Examples/Tests/point_of_contact_EB/analysis.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +""" +This script tests the coordinates of the point of contact of an electron hitting a sphere in 3D. +It compares the numerical results with the analytical solutions. +The sphere is centered on O and has a radius of 0.2 (EB) +The electron is initially at: (-0.25,0,0) and moves with a normalized momentum: (1,0.5,0) +An input file PICMI_inputs_3d.py is used. +""" +import os +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +import yt + +yt.funcs.mylog.setLevel(0) +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# Open plotfile specified in command line +filename = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename, output_format='openpmd') + +ts_scraping = OpenPMDTimeSeries('./diags/diag2/particles_at_eb/') + +it=ts_scraping.iterations +x,y,z=ts_scraping.get_particle( ['x','y','z'], species='electron', iteration=it ) + +# Analytical results calculated +x_analytic=-0.1983 +y_analytic=0.02584 +z_analytic=0.0000 + +print('NUMERICAL coordinates of the point of contact:') +print('x=%5.4f, y=%5.4f, z=%5.4f' % (x[0], y[0], z[0])) +print('\n') +print('ANALYTICAL coordinates of the point of contact:') +print('x=%5.4f, y=%5.4f, z=%5.4f' % (x_analytic, y_analytic, z_analytic)) + +tolerance=0.001 +print("tolerance = "+ str(tolerance *100) + '%') + +diff_x=np.abs((x[0]-x_analytic)/x_analytic) +diff_y=np.abs((y[0]-y_analytic)/y_analytic) + +print("percentage error for x = %5.4f %%" %(diff_x *100)) +print("percentage error for y = %5.4f %%" %(diff_y *100)) + +assert (diff_x < tolerance) and (diff_y < tolerance) and (np.abs(z[0]) < 1e-8) , 'Test point_of_contact did not pass' diff --git a/Examples/Tests/point_of_contact_EB/inputs_3d b/Examples/Tests/point_of_contact_EB/inputs_3d new file mode 100644 index 00000000000..64f8b520161 --- /dev/null +++ b/Examples/Tests/point_of_contact_EB/inputs_3d @@ -0,0 +1,48 @@ +amr.max_level = 0 + +max_step = 3 + +amr.n_cell = 64 64 64 +amr.blocking_factor = 8 +amr.max_grid_size = 128 + +geometry.dims = 3 +geometry.prob_lo = -0.26 -0.26 -0.26 +geometry.prob_hi = 0.26 0.26 0.26 + +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +warpx.const_dt = 1.216119097e-10 +warpx.eb_implicit_function = "-(x**2+y**2+z**2-0.2**2)" + +# Do not evolve the E and B fields +algo.maxwell_solver = none +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = -0.25 0 0 +electron.single_particle_u = 1 0.5 0.0 +electron.single_particle_weight = 1 + +electron.save_particles_at_eb = 1 + +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex +diag1.format = openpmd + +diag2.diag_type = BoundaryScraping +diag2.format = openpmd diff --git a/Regression/Checksum/benchmarks_json/Point_of_contact_EB_3d.json b/Regression/Checksum/benchmarks_json/Point_of_contact_EB_3d.json new file mode 100644 index 00000000000..357e0823e39 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Point_of_contact_EB_3d.json @@ -0,0 +1,14 @@ +{ + "electron": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_weight": 0.0 + }, + "lev=0": { + "Ex": 0.0 + } +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 627ba11bb5e..066faadc0a8 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4520,6 +4520,25 @@ outputFile = spacecraft_charging_plt analysisRoutine = Examples/Physics_applications/spacecraft_charging/analysis.py analysisOutputImage = min_phi_analysis.png +[Point_of_contact_EB_3d] +buildDir = . +inputFile = Examples/Tests/point_of_contact_EB/inputs_3d +runtime_params = +dim = 3 +addToCompileString = USE_EB=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +particleTypes = electrons +outputFile = Point_of_contact_EB_3d_plt +analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py + [ImplicitPicard_1d] buildDir = . inputFile = Examples/Tests/Implicit/inputs_1d diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index a175fe23133..7531a03487b 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -188,26 +188,26 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t if (phi_value < 0.0) { + amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, phi, dxi); + DistanceToEB::normalize(normal); - // the closest point on the surface to pos is pos - grad phi(pos) * phi(pos) amrex::RealVect pos; #if (defined WARPX_DIM_3D) - pos[0] = xp - normal[0]*phi_value; - pos[1] = yp - normal[1]*phi_value; - pos[2] = zp - normal[2]*phi_value; + pos[0] = xp; + pos[1] = yp; + pos[2] = zp; #elif (defined WARPX_DIM_XZ) - pos[0] = xp - normal[0]*phi_value; - pos[1] = zp - normal[1]*phi_value; + pos[0] = xp; + pos[1] = zp; #elif (defined WARPX_DIM_RZ) - pos[0] = std::sqrt(xp*xp + yp*yp) - normal[0]*phi_value; - pos[1] = zp - normal[1]*phi_value; + pos[0] = std::sqrt(xp*xp + yp*yp); + pos[1] = zp; #elif (defined WARPX_DIM_1D_Z) - pos[0] = zp - normal[0]*phi_value; + pos[0] = zp; #endif - DistanceToEB::normalize(normal); - f(ptd, ip, pos, normal, engine); + } }); } diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 88304bd8a9c..eaeb255b54b 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -11,6 +11,8 @@ #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" +#include "Particles/Pusher/GetAndSetPosition.H" +#include "Particles/Pusher/UpdatePosition.H" #include @@ -19,6 +21,7 @@ #include #include #include +#include struct IsOutsideDomainBoundary { amrex::GpuArray m_plo; @@ -41,6 +44,90 @@ struct IsOutsideDomainBoundary { } }; +struct FindEmbeddedBoundaryIntersection { + const int m_index; + const int m_step; + const amrex::Real m_dt; + amrex::Array4 m_phiarr; + amrex::GpuArray m_dxi; + amrex::GpuArray m_plo; + + template + AMREX_GPU_HOST_DEVICE + void operator() (const DstData& dst, const SrcData& src, + int src_i, int dst_i) const noexcept + { + // Copy all particle attributes, from the source to the destination + dst.m_idcpu[dst_i] = src.m_idcpu[src_i]; + for (int j = 0; j < SrcData::NAR; ++j) { + dst.m_rdata[j][dst_i] = src.m_rdata[j][src_i]; + } + for (int j = 0; j < src.m_num_runtime_real; ++j) { + dst.m_runtime_rdata[j][dst_i] = src.m_runtime_rdata[j][src_i]; + } + for (int j = 0; j < src.m_num_runtime_int; ++j) { + dst.m_runtime_idata[j][dst_i] = src.m_runtime_idata[j][src_i]; + } + + // Also record the integer timestep on the destination + dst.m_runtime_idata[m_index][dst_i] = m_step; + + // Modify the position of the destination particle: + // Move it to the point of intersection with the embedded boundary + // (which is found by using a bisection algorithm) + + const auto& p = dst.getSuperParticle(dst_i); + amrex::ParticleReal xp, yp, zp; + get_particle_position( p, xp, yp, zp ); + amrex::ParticleReal const ux = dst.m_rdata[PIdx::ux][dst_i]; + amrex::ParticleReal const uy = dst.m_rdata[PIdx::uy][dst_i]; + amrex::ParticleReal const uz = dst.m_rdata[PIdx::uz][dst_i]; + + // Temporary variables to avoid implicit capture + amrex::Real dt = m_dt; + amrex::Array4 phiarr = m_phiarr; + amrex::GpuArray dxi = m_dxi; + amrex::GpuArray plo = m_plo; + + // Bisection algorithm to find the point where phi(x,y,z)=0 (i.e. on the embedded boundary) + + amrex::Real dt_fraction = amrex::bisect( 0.0, 1.0, + [=] (amrex::Real dt_frac) { + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + amrex::ParticleReal x_temp=xp, y_temp=yp, z_temp=zp; + UpdatePosition(x_temp, y_temp, z_temp, ux, uy, uz, -dt_frac*dt); + ablastr::particles::compute_weights_nodal(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); + amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phiarr); + return phi_value; + } ); + + // Now that dt_fraction has be obtained (with bisect) + // Save the corresponding position of the particle at the boundary + amrex::ParticleReal x_temp=xp, y_temp=yp, z_temp=zp; + UpdatePosition(x_temp, y_temp, z_temp, ux, uy, uz, -dt_fraction*m_dt); + +#if (defined WARPX_DIM_3D) + dst.m_rdata[PIdx::x][dst_i] = x_temp; + dst.m_rdata[PIdx::y][dst_i] = y_temp; + dst.m_rdata[PIdx::z][dst_i] = z_temp; +#elif (defined WARPX_DIM_XZ) + dst.m_rdata[PIdx::x][dst_i] = x_temp; + dst.m_rdata[PIdx::z][dst_i] = z_temp; + amrex::ignore_unused(y_temp); +#elif (defined WARPX_DIM_RZ) + dst.m_rdata[PIdx::x][dst_i] = std::sqrt(x_temp*x_temp + y_temp*y_temp); + dst.m_rdata[PIdx::z][dst_i] = z_temp; + dst.m_rdata[PIdx::theta][dst_i] = std::atan2(y_temp, x_temp); +#elif (defined WARPX_DIM_1D_Z) + dst.m_rdata[PIdx::z][dst_i] = z_temp; + amrex::ignore_unused(x_temp, y_temp); +#else + amrex::ignore_unused(x_temp, y_temp, z_temp); +#endif + } +}; + struct CopyAndTimestamp { int m_index; int m_step; @@ -360,10 +447,13 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, const int timestamp_index = ptile_buffer.NumRuntimeIntComps()-1; const int timestep = warpx_instance.getistep(0); + auto& warpx = WarpX::GetInstance(); + const auto dt = warpx.getdt(pti.GetLevel()); + { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterTransformEB"); amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, - CopyAndTimestamp{timestamp_index, timestep}, 0, dst_index); + FindEmbeddedBoundaryIntersection{timestamp_index, timestep, dt, phiarr, dxi, plo}, 0, dst_index); } } } diff --git a/Source/Particles/Pusher/UpdatePosition.H b/Source/Particles/Pusher/UpdatePosition.H index 4743c79c00f..6e2e82632ee 100644 --- a/Source/Particles/Pusher/UpdatePosition.H +++ b/Source/Particles/Pusher/UpdatePosition.H @@ -15,6 +15,7 @@ #include + /** \brief Push the particle's positions over one timestep, * given the value of its momenta `ux`, `uy`, `uz`. * This uses the standard leapfrog algorithm From 9f52a452df86a1bff9f1ec43faf8196d5453a877 Mon Sep 17 00:00:00 2001 From: "S. Eric Clark" <25495882+clarkse@users.noreply.github.com> Date: Wed, 14 Feb 2024 10:32:21 -0800 Subject: [PATCH 157/176] =?UTF-8?q?Removed=20const=20modifier=20from=20LPI?= =?UTF-8?q?nfo=20object=20on=20stack=20that=20was=20causing=20a=E2=80=A6?= =?UTF-8?q?=20(#4696)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Removed const modifier from LPInfo object on stack that was causing a warning from not being initialized properly. * Changing instantiation of LPInfo to make consistent with usage elsewhere in AMReX and avoiding compile time warning. * LPInfo in PoissonSolver can be adjusted for semi-coarsening, so reverting back to non-const. --- Source/ablastr/fields/VectorPoissonSolver.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index a8833b070b8..8f9021456eb 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -130,7 +130,7 @@ computeVectorPotential ( amrex::Vector > co ); } - const amrex::LPInfo info; + const amrex::LPInfo& info = amrex::LPInfo(); // Loop over dimensions of A to solve each component individually for (int lev=0; lev<=finest_level; lev++) { From e038ef935a87dd2fc1db227f452cce684edb0ad8 Mon Sep 17 00:00:00 2001 From: Eya D <81635404+EyaDammak@users.noreply.github.com> Date: Thu, 15 Feb 2024 16:38:09 -0800 Subject: [PATCH 158/176] Adding a RZ version for the test "Point_of_contact_EB" (#4699) * adding RZ example * Update particle_containers.py * Update Point_of_contact_EB_rz.json --- Examples/Tests/point_of_contact_EB/inputs_3d | 2 +- Examples/Tests/point_of_contact_EB/inputs_rz | 48 +++++++++++++++++++ .../Point_of_contact_EB_rz.json | 15 ++++++ Regression/WarpX-tests.ini | 19 ++++++++ 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 Examples/Tests/point_of_contact_EB/inputs_rz create mode 100644 Regression/Checksum/benchmarks_json/Point_of_contact_EB_rz.json diff --git a/Examples/Tests/point_of_contact_EB/inputs_3d b/Examples/Tests/point_of_contact_EB/inputs_3d index 64f8b520161..005733487e5 100644 --- a/Examples/Tests/point_of_contact_EB/inputs_3d +++ b/Examples/Tests/point_of_contact_EB/inputs_3d @@ -19,7 +19,7 @@ boundary.potential_hi_y = 0 boundary.potential_lo_z = 0 boundary.potential_hi_z = 0 -warpx.const_dt = 1.216119097e-10 +warpx.const_dt = 1e-10 warpx.eb_implicit_function = "-(x**2+y**2+z**2-0.2**2)" # Do not evolve the E and B fields diff --git a/Examples/Tests/point_of_contact_EB/inputs_rz b/Examples/Tests/point_of_contact_EB/inputs_rz new file mode 100644 index 00000000000..a492c1f705a --- /dev/null +++ b/Examples/Tests/point_of_contact_EB/inputs_rz @@ -0,0 +1,48 @@ +amr.max_level = 0 + +max_step = 3 + +amr.n_cell = 64 64 +amr.blocking_factor = 8 +amr.max_grid_size = 128 + +geometry.dims = RZ +geometry.prob_lo = 0.0 -0.26 +geometry.prob_hi = 0.26 0.26 + +boundary.field_lo = none periodic +boundary.field_hi = pec periodic +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +warpx.const_dt = 1e-10 +warpx.eb_implicit_function = "-(x**2 -0.2**2)" + +# Do not evolve the E and B fields +algo.maxwell_solver = none +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = -0.25 0 0 +electron.single_particle_u = 1 0.5 0.0 +electron.single_particle_weight = 1 + +electron.save_particles_at_eb = 1 + +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Er +diag1.format = openpmd + +diag2.diag_type = BoundaryScraping +diag2.format = openpmd diff --git a/Regression/Checksum/benchmarks_json/Point_of_contact_EB_rz.json b/Regression/Checksum/benchmarks_json/Point_of_contact_EB_rz.json new file mode 100644 index 00000000000..422f31548d2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Point_of_contact_EB_rz.json @@ -0,0 +1,15 @@ +{ + "lev=0": { + "Er": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} + diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 066faadc0a8..9fb763b683e 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4539,6 +4539,25 @@ particleTypes = electrons outputFile = Point_of_contact_EB_3d_plt analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py +[Point_of_contact_EB_rz] +buildDir = . +inputFile = Examples/Tests/point_of_contact_EB/inputs_rz +runtime_params = +dim = 2 +addToCompileString = USE_RZ=TRUE USE_EB=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +particleTypes = electrons +outputFile = Point_of_contact_EB_rz_plt +analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py + [ImplicitPicard_1d] buildDir = . inputFile = Examples/Tests/Implicit/inputs_1d From a5c8f9fc4340985f5400c6a53de8246367ed4156 Mon Sep 17 00:00:00 2001 From: Eya D <81635404+EyaDammak@users.noreply.github.com> Date: Fri, 16 Feb 2024 16:27:27 -0800 Subject: [PATCH 159/176] Obtain exact time (real) when particles hit boundaries + test (#4695) * adding the time * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update particle_containers.py * Update ParticleBoundaryBuffer.cpp * Update PICMI_inputs_rz.py * Update PICMI_inputs_rz.py * Update particle_containers.py * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * fix issue * fix issue * issue fixed * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * Update analysis_rz.py * Update analysis_rz_filter.py * Update analysis.py * Update PICMI_inputs_scrape.py * Update PICMI_inputs_reflection.py * Update particle_containers.py * Update PICMI_inputs_reflection.py * Update PICMI_inputs_scrape.py * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * Update Source/Particles/ParticleBoundaryBuffer.cpp Co-authored-by: Remi Lehe * Update Source/Particles/ParticleBoundaryBuffer.cpp Co-authored-by: Remi Lehe * Update ParticleBoundaryBuffer.cpp --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Remi Lehe --- Docs/source/usage/parameters.rst | 6 ++- .../Tests/point_of_contact_EB/analysis.py | 17 +++++-- Examples/Tests/scraping/analysis_rz.py | 4 +- Examples/Tests/scraping/analysis_rz_filter.py | 4 +- Python/pywarpx/particle_containers.py | 24 +++++----- Source/Diagnostics/BTDiagnostics.cpp | 2 +- Source/Particles/ParticleBoundaryBuffer.cpp | 46 ++++++++++++------- .../PinnedMemoryParticleContainer.cpp | 13 ++++++ 8 files changed, 78 insertions(+), 38 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index b02e6198213..ea670173a1d 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2752,8 +2752,10 @@ The data collected at each boundary is written out to a subdirectory of the diag By default, all of the collected particle data is written out at the end of the simulation. Optionally, the ``.intervals`` parameter can be given to specify writing out the data more often. This can be important if a large number of particles are lost, avoiding filling up memory with the accumulated lost particle data. -In addition to their usual attributes, the saved particles have an integer attribute ``timestamp``, which -indicates the PIC iteration at which each particle was absorbed at the boundary. +In addition to their usual attributes, the saved particles have an integer attribute ``step_scraped``, which +indicates the PIC iteration at which each particle was absorbed at the boundary, +and a real attribute ``time_scraped``, which indicates the exact time calculated when each +particle hits the EB. ``BoundaryScrapingDiagnostics`` can be used with ``..random_fraction``, ``..uniform_stride``, and ``..plot_filter_function``, which have the same behavior as for ``FullDiagnostics``. For ``BoundaryScrapingDiagnostics``, these filters are applied at the time the data is written to file. An implication of this is that more particles may initially be accumulated in memory than are ultimately written. ``t`` in ``plot_filter_function`` refers to the time the diagnostic is written rather than the time the particle crossed the boundary. diff --git a/Examples/Tests/point_of_contact_EB/analysis.py b/Examples/Tests/point_of_contact_EB/analysis.py index fd4782241b2..743cbc2fb39 100755 --- a/Examples/Tests/point_of_contact_EB/analysis.py +++ b/Examples/Tests/point_of_contact_EB/analysis.py @@ -26,26 +26,35 @@ ts_scraping = OpenPMDTimeSeries('./diags/diag2/particles_at_eb/') it=ts_scraping.iterations -x,y,z=ts_scraping.get_particle( ['x','y','z'], species='electron', iteration=it ) +step_scraped, time_scraped, x, y, z=ts_scraping.get_particle( ['stepScraped','timeScraped','x','y','z'], species='electron', iteration=it ) +time_scraped_reduced=time_scraped[0]*1e10 # Analytical results calculated x_analytic=-0.1983 y_analytic=0.02584 z_analytic=0.0000 +#result obtained by analysis of simulations +step=3 +time_reduced=3.58 + print('NUMERICAL coordinates of the point of contact:') -print('x=%5.4f, y=%5.4f, z=%5.4f' % (x[0], y[0], z[0])) +print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f' % (step_scraped[0],time_reduced,x[0], y[0], z[0])) print('\n') print('ANALYTICAL coordinates of the point of contact:') -print('x=%5.4f, y=%5.4f, z=%5.4f' % (x_analytic, y_analytic, z_analytic)) +print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f' % (step, time_reduced, x_analytic, y_analytic, z_analytic)) tolerance=0.001 +tolerance_t=0.003 print("tolerance = "+ str(tolerance *100) + '%') +print("tolerance for the time = "+ str(tolerance_t *100) + '%') +diff_step=np.abs((step_scraped[0]-step)/step) +diff_time=np.abs((time_scraped_reduced-time_reduced)/time_reduced) diff_x=np.abs((x[0]-x_analytic)/x_analytic) diff_y=np.abs((y[0]-y_analytic)/y_analytic) print("percentage error for x = %5.4f %%" %(diff_x *100)) print("percentage error for y = %5.4f %%" %(diff_y *100)) -assert (diff_x < tolerance) and (diff_y < tolerance) and (np.abs(z[0]) < 1e-8) , 'Test point_of_contact did not pass' +assert (diff_x < tolerance) and (diff_y < tolerance) and (np.abs(z[0]) < 1e-8) and (diff_step < 1e-8) and (diff_time < tolerance_t) , 'Test point_of_contact did not pass' diff --git a/Examples/Tests/scraping/analysis_rz.py b/Examples/Tests/scraping/analysis_rz.py index 0390eed580e..7d40bd5edf4 100755 --- a/Examples/Tests/scraping/analysis_rz.py +++ b/Examples/Tests/scraping/analysis_rz.py @@ -53,8 +53,8 @@ def n_remaining_particles( iteration ): w, = ts_full.get_particle(['w'], iteration=iteration) return len(w) def n_scraped_particles( iteration ): - timestamp = ts_scraping.get_particle( ['timestamp'], iteration=ts_scraping.iterations[0] ) - return (timestamp <= iteration).sum() + step_scraped = ts_scraping.get_particle( ['stepScraped'], iteration=ts_scraping.iterations[0] ) + return (step_scraped <= iteration).sum() n_remaining = np.array([ n_remaining_particles(iteration) for iteration in ts_full.iterations ]) n_scraped = np.array([ n_scraped_particles(iteration) for iteration in ts_full.iterations ]) n_total = n_remaining[0] diff --git a/Examples/Tests/scraping/analysis_rz_filter.py b/Examples/Tests/scraping/analysis_rz_filter.py index b97b6e0eb5a..dd13b993dc5 100755 --- a/Examples/Tests/scraping/analysis_rz_filter.py +++ b/Examples/Tests/scraping/analysis_rz_filter.py @@ -50,8 +50,8 @@ def n_remaining_particles( iteration ): w, = ts_full.get_particle(['w'], iteration=iteration) return len(w) def n_scraped_particles( iteration ): - timestamp = ts_scraping.get_particle( ['timestamp'], iteration=ts_scraping.iterations[0] ) - return (timestamp <= iteration).sum() + step_scraped = ts_scraping.get_particle( ['stepScraped'], iteration=ts_scraping.iterations[0] ) + return (step_scraped <= iteration).sum() def n_scraped_z_leq_zero( iteration ): z_pos, = ts_scraping.get_particle( ['z'], iteration=ts_scraping.iterations[0] ) return (z_pos <= 0).sum() diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index 393663bc8cf..43eefa0337d 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -772,9 +772,9 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) form x/y/z_hi/lo or eb. comp_name : str - The component of the array data that will be returned. If - "step_scraped" the special attribute holding the timestep at - which a particle was scraped will be returned. + The component of the array data that will be returned. + "x", "y", "z", "ux", "uy", "uz", "w" + "step_scraped","time_scraped", "nx", "ny", "nz" level : int Which AMR level to retrieve scraped particle data from. @@ -785,18 +785,20 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) species_name, self._get_boundary_number(boundary) ) data_array = [] - if comp_name == 'step_scraped': - # the step scraped is always the final integer component - comp_idx = part_container.num_int_comps - 1 + #loop over the real attributes + if comp_name in part_container.real_comp_names: + comp_idx = part_container.real_comp_names[comp_name] for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): + soa = pti.soa() + data_array.append(xp.array(soa.get_real_data(comp_idx), copy=False)) + #loop over the integer attributes + elif comp_name in part_container.int_comp_names: + comp_idx = part_container.int_comp_names[comp_name] + for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): soa = pti.soa() data_array.append(xp.array(soa.get_int_data(comp_idx), copy=False)) else: - container_wrapper = ParticleContainerWrapper(species_name) - comp_idx = container_wrapper.particle_container.get_comp_index(comp_name) - for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): - soa = pti.soa() - data_array.append(xp.array(soa.get_real_data(comp_idx), copy=False)) + raise RuntimeError('Name %s not found' %comp_name) return data_array diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 16617159662..41c710805e9 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -1299,7 +1299,7 @@ BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_pat BTDPlotfileHeaderImpl buffer_HeaderImpl(buffer_Header_path); buffer_HeaderImpl.ReadHeaderData(); - // Update timestamp of snapshot with recently flushed buffer + // Update step_scraped of snapshot with recently flushed buffer snapshot_HeaderImpl.set_time( buffer_HeaderImpl.time() ); snapshot_HeaderImpl.set_timestep( buffer_HeaderImpl.timestep() ); diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index eaeb255b54b..2b90adac172 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -45,7 +45,8 @@ struct IsOutsideDomainBoundary { }; struct FindEmbeddedBoundaryIntersection { - const int m_index; + const int m_step_index; + const int m_time_index; const int m_step; const amrex::Real m_dt; amrex::Array4 m_phiarr; @@ -69,9 +70,6 @@ struct FindEmbeddedBoundaryIntersection { dst.m_runtime_idata[j][dst_i] = src.m_runtime_idata[j][src_i]; } - // Also record the integer timestep on the destination - dst.m_runtime_idata[m_index][dst_i] = m_step; - // Modify the position of the destination particle: // Move it to the point of intersection with the embedded boundary // (which is found by using a bisection algorithm) @@ -90,7 +88,6 @@ struct FindEmbeddedBoundaryIntersection { amrex::GpuArray plo = m_plo; // Bisection algorithm to find the point where phi(x,y,z)=0 (i.e. on the embedded boundary) - amrex::Real dt_fraction = amrex::bisect( 0.0, 1.0, [=] (amrex::Real dt_frac) { int i, j, k; @@ -102,6 +99,10 @@ struct FindEmbeddedBoundaryIntersection { return phi_value; } ); + // Also record the real time on the destination + dst.m_runtime_idata[m_step_index][dst_i] = m_step; + dst.m_runtime_rdata[m_time_index][dst_i] = m_step*m_dt + (1- dt_fraction)*m_dt; + // Now that dt_fraction has be obtained (with bisect) // Save the corresponding position of the particle at the boundary amrex::ParticleReal x_temp=xp, y_temp=yp, z_temp=zp; @@ -129,8 +130,10 @@ struct FindEmbeddedBoundaryIntersection { }; struct CopyAndTimestamp { - int m_index; + int m_step_index; + int m_time_index; int m_step; + const amrex::Real m_dt; template AMREX_GPU_HOST_DEVICE @@ -147,10 +150,12 @@ struct CopyAndTimestamp { for (int j = 0; j < src.m_num_runtime_int; ++j) { dst.m_runtime_idata[j][dst_i] = src.m_runtime_idata[j][src_i]; } - dst.m_runtime_idata[m_index][dst_i] = m_step; + dst.m_runtime_idata[m_step_index][dst_i] = m_step; + dst.m_runtime_rdata[m_time_index][dst_i] = m_step*m_dt; } }; + ParticleBoundaryBuffer::ParticleBoundaryBuffer () { m_particle_containers.resize(numBoundaries()); @@ -328,7 +333,8 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, if (!buffer[i].isDefined()) { buffer[i] = pc.make_alike(); - buffer[i].AddIntComp("timestamp", false); + buffer[i].AddIntComp("step_scraped", false); + buffer[i].AddRealComp("time_scraped", false); } auto& species_buffer = buffer[i]; for (int lev = 0; lev < pc.numLevels(); ++lev) @@ -368,12 +374,17 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, } { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterAndTransform"); - const int timestamp_index = ptile_buffer.NumRuntimeIntComps()-1; - const int timestep = warpx_instance.getistep(0); + auto& warpx = WarpX::GetInstance(); + const auto dt = warpx.getdt(pti.GetLevel()); + auto string_to_index_intcomp = buffer[i].getParticleRuntimeiComps(); + const int step_scraped_index = string_to_index_intcomp.at("step_scraped"); + auto string_to_index_realcomp = buffer[i].getParticleRuntimeComps(); + const int time_scraped_index = string_to_index_realcomp.at("time_scraped"); + const int step = warpx_instance.getistep(0); amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, - CopyAndTimestamp{timestamp_index, timestep}, + CopyAndTimestamp{step_scraped_index,time_scraped_index, step,dt}, 0, dst_index); } } @@ -393,7 +404,8 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, if (!buffer[i].isDefined()) { buffer[i] = pc.make_alike(); - buffer[i].AddIntComp("timestamp", false); + buffer[i].AddIntComp("step_scraped", false); + buffer[i].AddRealComp("time_scraped", false); } auto& species_buffer = buffer[i]; for (int lev = 0; lev < pc.numLevels(); ++lev) @@ -444,16 +456,18 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::resize_eb"); ptile_buffer.resize(dst_index + amrex::get<0>(reduce_data.value())); } - - const int timestamp_index = ptile_buffer.NumRuntimeIntComps()-1; - const int timestep = warpx_instance.getistep(0); auto& warpx = WarpX::GetInstance(); const auto dt = warpx.getdt(pti.GetLevel()); + auto string_to_index_intcomp = buffer[i].getParticleRuntimeiComps(); + const int step_scraped_index = string_to_index_intcomp.at("step_scraped"); + auto string_to_index_realcomp = buffer[i].getParticleRuntimeComps(); + const int time_scraped_index = string_to_index_realcomp.at("time_scraped"); + const int step = warpx_instance.getistep(0); { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterTransformEB"); amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, - FindEmbeddedBoundaryIntersection{timestamp_index, timestep, dt, phiarr, dxi, plo}, 0, dst_index); + FindEmbeddedBoundaryIntersection{step_scraped_index,time_scraped_index, step, dt, phiarr, dxi, plo}, 0, dst_index); } } } diff --git a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp index d4f6a422dbe..21dd6a9d364 100644 --- a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp +++ b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp @@ -15,4 +15,17 @@ void init_PinnedMemoryParticleContainer (py::module& m) PinnedMemoryParticleContainer, amrex::ParticleContainerPureSoA > pmpc (m, "PinnedMemoryParticleContainer"); + pmpc + .def_property_readonly("real_comp_names", + [](PinnedMemoryParticleContainer& pc) + { + return pc.getParticleComps(); + } + ) + .def_property_readonly("int_comp_names", + [](PinnedMemoryParticleContainer& pc) + { + return pc.getParticleiComps(); + } + ); } From 32c5b9252d09509dbe86d713f290aed1c7b92df1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Feb 2024 02:33:25 +0000 Subject: [PATCH 160/176] [pre-commit.ci] pre-commit autoupdate (#4707) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/Lucas-C/pre-commit-hooks: v1.5.4 → v1.5.5](https://github.com/Lucas-C/pre-commit-hooks/compare/v1.5.4...v1.5.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 35c38ac1ad6..f3d14c123c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,7 +49,7 @@ repos: # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.4 + rev: v1.5.5 hooks: - id: remove-tabs exclude: 'Make.WarpX|Make.package|Makefile|GNUmake' From 5a7c32577fd921a8bdda6edae7c6aa1abfff1c86 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 20 Feb 2024 08:47:52 +0100 Subject: [PATCH 161/176] Docs: update instructions for Fugaku supercomputer (Riken, Japan) (#4656) * update instructions for the Fugaku supercomputer * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add missing instruction * update Fugaku instructions * fix bug * fix bug * fix bug * fix bug * fix bug * fix bug * fix issue with Fujitsu compiler * update install_dependencies script to save some space * delete BinaryCollision move constructor * update compilation instructions * update submission script --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/install/hpc/fugaku.rst | 67 +++++++++---------- .../BinaryCollision/BinaryCollision.H | 6 +- .../fugaku-riken/fugaku_warpx.profile.example | 31 +++++++++ .../fugaku-riken/install_dependencies.sh | 53 +++++++++++++++ Tools/machines/fugaku-riken/submit.sh | 6 ++ 5 files changed, 126 insertions(+), 37 deletions(-) create mode 100644 Tools/machines/fugaku-riken/fugaku_warpx.profile.example create mode 100755 Tools/machines/fugaku-riken/install_dependencies.sh diff --git a/Docs/source/install/hpc/fugaku.rst b/Docs/source/install/hpc/fugaku.rst index e70aeefa504..5e5fc57c3b0 100644 --- a/Docs/source/install/hpc/fugaku.rst +++ b/Docs/source/install/hpc/fugaku.rst @@ -14,8 +14,10 @@ If you are new to this system, **please see the following resources**: * `Fugaku user guide `__ -Installation ------------- +.. _building-fugaku-preparation: + +Preparation +----------- Use the following commands to download the WarpX source code and switch to the correct branch: @@ -31,45 +33,42 @@ Compiling WarpX on Fugaku is more practical on a compute node. Use the following pjsub --interact -L "elapse=02:00:00" -L "node=1" --sparam "wait-time=300" --mpi "max-proc-per-node=48" --all-mount-gfscache -Then, load ``cmake`` and ``ninja`` using ``spack``: +We use system software modules, add environment hints and further dependencies via the file ``$HOME/fugaku_warpx.profile``. +Create it now, modify it if needed, and source it (it will take few minutes): .. code-block:: bash - . /vol0004/apps/oss/spack/share/spack/setup-env.sh - spack load cmake@3.21.4%fj@4.8.1 arch=linux-rhel8-a64fx + cp $HOME/src/warpx/Tools/machines/fugaku-riken/fugaku_warpx.profile.example $HOME/fugaku_warpx.profile + source $HOME/fugaku_warpx.profile - # optional: faster builds - spack load ninja@1.11.1%fj@4.8.1 +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - # avoid harmless warning messages "[WARN] xos LPG [...]" - export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH + .. literalinclude:: ../../../../Tools/machines/fugaku-riken/fugaku_warpx.profile.example + :language: bash - -At this point we need to download and compile the libraries required for OpenPMD support: +Finally, since Fugaku does not yet provide software modules for some of our dependencies, install them once: .. code-block:: bash - export CC=$(which mpifcc) - export CXX=$(which mpiFCC) - export CFLAGS="-O3 -Nclang -Nlibomp -Klib -g -DNDEBUG" - export CXXFLAGS="${CFLAGS}" + bash $HOME/src/warpx/Tools/machines/fugaku-riken/install_dependencies.sh + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - export CMAKE_PREFIX_PATH=${HOME}/sw/a64fx-fj490/c-blosc-1.21.1-install:$CMAKE_PREFIX_PATH - export CMAKE_PREFIX_PATH=${HOME}/sw/a64fx-fj490/adios2-2.8.3-install:$CMAKE_PREFIX_PATH + .. literalinclude:: ../../../../Tools/machines/fugaku-riken/install_dependencies.sh + :language: bash - # c-blosc (I/O compression) - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc - rm -rf src/c-blosc-fg-build - cmake -S src/c-blosc -B src/c-blosc-fg-build -DBUILD_SHARED_LIBS=OFF -DBUILD_SHARED=OFF -DBUILD_STATIC=ON -DBUILD_TESTS=OFF -DBUILD_FUZZERS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/a64fx-fj490/c-blosc-1.21.1-install - cmake --build src/c-blosc-fg-build --target install --parallel 24 +.. _building-fugaku-compilation: - # ADIOS2 - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git src/adios2 - rm -rf src/adios2-fg-build - cmake -S src/adios2 -B src/adios2-fg-build -DBUILD_SHARED_LIBS=OFF -DADIOS2_USE_Blosc=ON -DBUILD_TESTING=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/a64fx-fj490/adios2-2.8.3-install - cmake --build src/adios2-fg-build --target install -j 24 +Compilation +----------- -Finally, ``cd`` into the directory ``$HOME/src/warpx`` and use the following commands to compile: +Use the following :ref:`cmake commands ` to compile the application executable: .. code-block:: bash @@ -82,16 +81,14 @@ Finally, ``cd`` into the directory ``$HOME/src/warpx`` and use the following com export CXXFLAGS="-Nclang" cmake -S . -B build -DWarpX_COMPUTE=OMP \ - -DWarpX_DIMS="1;2;3" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_FLAGS_RELEASE="-Ofast -mllvm -polly -mllvm -polly-parallel" \ - -DAMReX_DIFFERENT_COMPILER=ON \ - -DWarpX_MPI_THREAD_MULTIPLE=OFF + -DWarpX_DIMS="1;2;3" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_FLAGS_RELEASE="-Ofast" \ + -DAMReX_DIFFERENT_COMPILER=ON \ + -DWarpX_MPI_THREAD_MULTIPLE=OFF cmake --build build -j 48 -The general :ref:`cmake compile-time options ` apply as usual. - **That's it!** A 3D WarpX executable is now in ``build/bin/`` and :ref:`can be run ` with a :ref:`3D example inputs file `. diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index c69f07acdb2..78381f22258 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -110,8 +110,10 @@ public: BinaryCollision ( BinaryCollision const &) = default; BinaryCollision& operator= ( BinaryCollision const & ) = default; - BinaryCollision ( BinaryCollision&& ) noexcept = default; - BinaryCollision& operator= ( BinaryCollision&& ) noexcept = default; + + + BinaryCollision ( BinaryCollision&& ) = delete; + BinaryCollision& operator= ( BinaryCollision&& ) = delete; /** Perform the collisions * diff --git a/Tools/machines/fugaku-riken/fugaku_warpx.profile.example b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example new file mode 100644 index 00000000000..a430579a7d5 --- /dev/null +++ b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example @@ -0,0 +1,31 @@ +. /vol0004/apps/oss/spack/share/spack/setup-env.sh + +# required dependencies +spack load cmake@3.24.3%fj@4.10.0 arch=linux-rhel8-a64fx + +# avoid harmless warning messages "[WARN] xos LPG [...]" +export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH + +# optional: faster builds +spack load ninja@1.11.1%fj@4.10.0 + +# optional: for PSATD +spack load fujitsu-fftw + +# optional: for QED lookup table generation support +spack load boost@1.80.0%fj@4.8.1/zc5pwgc + +# optional: for openPMD support +spack load hdf5@1.12.2%fj@4.8.1/im6lxev +export CMAKE_PREFIX_PATH=${HOME}/sw/fugaku/a64fx/c-blosc-1.21.1-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/fugaku/a64fx/adios2-2.8.3-install:$CMAKE_PREFIX_PATH + +# compiler environment hints +export CC=$(which mpifcc) +export CXX=$(which mpiFCC) +export FC=$(which mpifrt) +export CFLAGS="-O3 -Nclang -Nlibomp -Klib -g -DNDEBUG" +export CXXFLAGS="-O3 -Nclang -Nlibomp -Klib -g -DNDEBUG" + +# avoid harmless warning messages "[WARN] xos LPG [...]" +export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH diff --git a/Tools/machines/fugaku-riken/install_dependencies.sh b/Tools/machines/fugaku-riken/install_dependencies.sh new file mode 100755 index 00000000000..3ceb45e4558 --- /dev/null +++ b/Tools/machines/fugaku-riken/install_dependencies.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl, Luca Fedeli +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + +# Remove old dependencies ##################################################### +# +SRC_DIR="${HOME}/src/" +SW_DIR="${HOME}/sw/fugaku/a64fx/" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} +mkdir -p ${SRC_DIR} + +# General extra dependencies ################################################## +# + +# c-blosc (I/O compression) +if [ -d ${SRC_DIR}/c-blosc ] +then + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.1 + cd - +else + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc +fi + rm -rf ${SRC_DIR}/c-blosc-fugaku-build + cmake -S ${SRC_DIR}/c-blosc -B ${SRC_DIR}/c-blosc-fugaku-build -DBUILD_SHARED_LIBS=OFF -DBUILD_SHARED=OFF -DBUILD_STATIC=ON -DBUILD_TESTS=OFF -DBUILD_FUZZERS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1-install + cmake --build ${SRC_DIR}/c-blosc-fugaku-build --target install --parallel 48 + rm -rf ${SRC_DIR}/c-blosc-fugaku-build + +# ADIOS2 (I/O) +if [ -d ${SRC_DIR}/c-blosc ] +then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.8.3 + cd - +else + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +rm -rf ${SRC_DIR}/adios2-fugaku-build +cmake -S ${SRC_DIR}/adios2 -B ${SRC_DIR}/adios2-fugaku-build -DBUILD_SHARED_LIBS=OFF -DADIOS2_USE_Blosc=ON -DBUILD_TESTING=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3-install +cmake --build ${SRC_DIR}/adios2-fugaku-build --target install -j 48 +rm -rf ${SRC_DIR}/adios2-fugaku-build diff --git a/Tools/machines/fugaku-riken/submit.sh b/Tools/machines/fugaku-riken/submit.sh index caf254ba236..3768ed62f46 100644 --- a/Tools/machines/fugaku-riken/submit.sh +++ b/Tools/machines/fugaku-riken/submit.sh @@ -18,6 +18,12 @@ export INPUT="i.3d" export XOS_MMM_L_PAGING_POLICY=demand:demand:demand +# Add HDF5 library path to LD_LIBRARY_PATH +# This is done manually to avoid calling spack during the run, +# since this would take a significant amount of time. +export LD_LIBRARY_PATH=/vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/fj-4.8.1/hdf5-1.12.2-im6lxevf76cu6cbzspi4itgz3l4gncjj/lib:$LD_LIBRARY_PATH + +# Broadcast WarpX executable to all the nodes llio_transfer ${EXE} mpiexec -stdout-proc ./output.%j/%/1000r/stdout -stderr-proc ./output.%j/%/1000r/stderr -n ${MPI_RANKS} ${EXE} ${INPUT} From 8bb8ed971573e4226ebb5702a5b829c7f1d82409 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 21 Feb 2024 00:39:45 -0800 Subject: [PATCH 162/176] Fix Particle Id Overflows (#4708) * AMReX: Weekly Update * pyAMReX: Weekly Update * Azure: Bump up Timeout to 4hrs * Fix `ParticleContainer`: `amrex::Long pid` overflows ``` error: narrowing conversion from 'amrex::Long' (aka 'long') to signed type 'int' is implementation-defined [bugprone-narrowing-conversions,-warnings-as-errors] ``` * Cleanup: no AoS --- .azure-pipelines.yml | 2 +- .github/workflows/cuda.yml | 2 +- Docs/source/developers/particles.rst | 10 +++---- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- .../Particles/ParticleCreation/SmartUtils.H | 7 ++--- .../Particles/PhysicalParticleContainer.cpp | 27 +++++++++---------- Source/Particles/Sorting/Partition.cpp | 1 - cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 11 files changed, 27 insertions(+), 32 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 1806bdb00d1..7e4edc2dc36 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -41,7 +41,7 @@ jobs: WARPX_CI_EB: 'TRUE' # default: 60; maximum: 360 - timeoutInMinutes: 180 + timeoutInMinutes: 240 steps: # set up caches: diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index e7ecf36a034..f16b405973e 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 68244ec91d118b5d4cc21f93376eaae8b56c51eb && cd - + cd ../amrex && git checkout --detach 2230caa24c7d4bd07edb08b54e9368f9c73eae6e && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Docs/source/developers/particles.rst b/Docs/source/developers/particles.rst index a0182f3845f..4b203e483c4 100644 --- a/Docs/source/developers/particles.rst +++ b/Docs/source/developers/particles.rst @@ -36,17 +36,14 @@ A typical loop over particles reads: } } -The innermost step ``[MY INNER LOOP]`` typically calls ``amrex::ParallelFor`` to perform operations on all particles in a portable way. For this reasons, the particle data needs to be converted in plain-old-data structures. The innermost loop in the code snippet above could look like: +The innermost step ``[MY INNER LOOP]`` typically calls ``amrex::ParallelFor`` to perform operations on all particles in a portable way. The innermost loop in the code snippet above could look like: .. code-block:: cpp - // Get Array-Of-Struct particle data, also called data - // (x, y, z, id, cpu) - const auto& particles = pti.GetArrayOfStructs(); // Get Struct-Of-Array particle data, also called attribs - // (ux, uy, uz, w, Exp, Ey, Ez, Bx, By, Bz) + // (x, y, z, ux, uy, uz, w) auto& attribs = pti.GetAttribs(); - auto& Exp = attribs[PIdx::Ex]; + auto& x = attribs[PIdx::x]; // [...] // Number of particles in this box const long np = pti.numParticles(); @@ -66,7 +63,6 @@ On a loop over boxes in a ``MultiFab`` (``MFIter``), it can be useful to access const int tile_id = mfi.LocalTileIndex(); // Get GPU-friendly arrays of particle data auto& ptile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; - ParticleType* pp = particle_tile.GetArrayOfStructs()().data(); // Only need attribs (i.e., SoA data) auto& soa = ptile.GetStructOfArrays(); // As an example, let's get the ux momentum diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 2760beed53b..13cbd2c22d5 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 68244ec91d118b5d4cc21f93376eaae8b56c51eb +branch = 2230caa24c7d4bd07edb08b54e9368f9c73eae6e [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 9fb763b683e..1e5e7c79a65 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 68244ec91d118b5d4cc21f93376eaae8b56c51eb +branch = 2230caa24c7d4bd07edb08b54e9368f9c73eae6e [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/Source/Particles/ParticleCreation/SmartUtils.H b/Source/Particles/ParticleCreation/SmartUtils.H index f84734308fb..dbac563ca28 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.H +++ b/Source/Particles/ParticleCreation/SmartUtils.H @@ -49,7 +49,7 @@ SmartCopyTag getSmartCopyTag (const NameMap& src, const NameMap& dst) noexcept; * \param num_added the number of particles to set the ids for. */ template -void setNewParticleIDs (PTile& ptile, int old_size, int num_added) +void setNewParticleIDs (PTile& ptile, amrex::Long old_size, amrex::Long num_added) { amrex::Long pid; #ifdef AMREX_USE_OMP @@ -64,8 +64,9 @@ void setNewParticleIDs (PTile& ptile, int old_size, int num_added) auto ptd = ptile.getParticleTileData(); amrex::ParallelFor(num_added, [=] AMREX_GPU_DEVICE (int ip) noexcept { - auto const new_id = ip + old_size; - ptd.m_idcpu[new_id] = amrex::SetParticleIDandCPU(pid+ip, cpuid); + auto const lip = static_cast(ip); + auto const new_id = lip + old_size; + ptd.m_idcpu[new_id] = amrex::SetParticleIDandCPU(pid+lip, cpuid); }); } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 08c784709fa..da1655b9dab 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -51,7 +51,6 @@ #include #include #include -#include #include #include #include @@ -1022,8 +1021,8 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int overlap_realbox.lo(2))}; // count the number of particles that each cell in overlap_box could add - Gpu::DeviceVector counts(overlap_box.numPts(), 0); - Gpu::DeviceVector offset(overlap_box.numPts()); + Gpu::DeviceVector counts(overlap_box.numPts(), 0); + Gpu::DeviceVector offset(overlap_box.numPts()); auto *pcounts = counts.data(); const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). @@ -1042,7 +1041,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int if (inj_pos->overlapsWith(lo, hi)) { auto index = overlap_box.index(iv); - const int r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv))? + const amrex::Long r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv))? (AMREX_D_TERM(lrrfac[0],*lrrfac[1],*lrrfac[2])) : (1); pcounts[index] = num_ppc*r; // update pcount by checking if cell-corners or cell-center @@ -1080,10 +1079,10 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // Max number of new particles. All of them are created, // and invalid ones are then discarded - const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); + const amrex::Long max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - int pid; + amrex::Long pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1092,7 +1091,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid) + static_cast(max_new_particles) < LongParticleIds::LastParticleID, + pid + max_new_particles < LongParticleIds::LastParticleID, "ERROR: overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); @@ -1103,8 +1102,8 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int DefineAndReturnParticleTile(lev, grid_id, tile_id); } - auto old_size = particle_tile.size(); - auto new_size = old_size + max_new_particles; + auto const old_size = static_cast(particle_tile.size()); + auto const new_size = old_size + max_new_particles; particle_tile.resize(new_size); auto& soa = particle_tile.GetStructOfArrays(); @@ -1639,10 +1638,10 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // Max number of new particles. All of them are created, // and invalid ones are then discarded - const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); + const amrex::Long max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - int pid; + amrex::Long pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1651,15 +1650,15 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid) + static_cast(max_new_particles) < LongParticleIds::LastParticleID, + pid + max_new_particles < LongParticleIds::LastParticleID, "overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); auto& particle_tile = tmp_pc.DefineAndReturnParticleTile(0, grid_id, tile_id); - auto old_size = particle_tile.size(); - auto new_size = old_size + max_new_particles; + auto const old_size = static_cast(particle_tile.size()); + auto const new_size = old_size + max_new_particles; particle_tile.resize(new_size); auto& soa = particle_tile.GetStructOfArrays(); diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp index 58e3450f47d..9f92739a5d5 100644 --- a/Source/Particles/Sorting/Partition.cpp +++ b/Source/Particles/Sorting/Partition.cpp @@ -10,7 +10,6 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" -#include #include #include #include diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 9db2b1f1aad..6c4570a4078 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "68244ec91d118b5d4cc21f93376eaae8b56c51eb" +set(WarpX_amrex_branch "2230caa24c7d4bd07edb08b54e9368f9c73eae6e" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 1079735279a..71831357e66 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "5aa700de18a61f933cb435adbe2299d74d794d6b" +set(WarpX_pyamrex_branch "0cbf4b08c9045e1845595c836b99f94bb3c1ac9f" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index a0ba9e056a5..2b53571163a 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 68244ec91d118b5d4cc21f93376eaae8b56c51eb && cd - +cd amrex && git checkout --detach 2230caa24c7d4bd07edb08b54e9368f9c73eae6e && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 994feddef52c23d82e185ea3b770fa8e77535d36 Mon Sep 17 00:00:00 2001 From: Kale Weichman <46941919+kale-j@users.noreply.github.com> Date: Wed, 21 Feb 2024 18:38:35 -0500 Subject: [PATCH 163/176] Implementation of 4th order particle shape beyond implicit solver cases (#4706) * Merged with latest Modified shape factors written for Villasenor algorithm to use simpler (mathematically equivalent) expressions * Added test Added test for particle_shape=4 Choice of 2d Langmuir wave case is based on cost (resolution would need to be increased relative to existing 3d tests) Fixed bugs related to rebase * Removed tabs Removed tabs that did not properly convert to whitespace * Undid custom settings in regression testing * Update Source/Particles/WarpXParticleContainer.cpp --- Docs/source/usage/parameters.rst | 4 +- Examples/Tests/langmuir/analysis_2d.py | 9 +++- ...psatd_Vay_deposition_particle_shape_4.json | 29 ++++++++++ Regression/WarpX-tests.ini | 19 +++++++ Source/Particles/Gather/FieldGather.H | 21 ++++++++ Source/Particles/ShapeFactors.H | 54 ++++++++++--------- Source/Particles/WarpXParticleContainer.cpp | 52 ++++++++++++++++++ Source/WarpX.cpp | 30 +++++------ 8 files changed, 176 insertions(+), 42 deletions(-) create mode 100644 Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4.json diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index ea670173a1d..11dffae446b 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2032,8 +2032,8 @@ Particle push, charge and current deposition, field gathering If ``algo.particle_pusher`` is not specified, ``boris`` is the default. -* ``algo.particle_shape`` (`integer`; `1`, `2`, or `3`) - The order of the shape factors (splines) for the macro-particles along all spatial directions: `1` for linear, `2` for quadratic, `3` for cubic. +* ``algo.particle_shape`` (`integer`; `1`, `2`, `3`, or `4`) + The order of the shape factors (splines) for the macro-particles along all spatial directions: `1` for linear, `2` for quadratic, `3` for cubic, `4` for quartic. Low-order shape factors result in faster simulations, but may lead to more noisy results. High-order shape factors are computationally more expensive, but may increase the overall accuracy of the results. For production runs it is generally safer to use high-order shape factors, such as cubic order. diff --git a/Examples/Tests/langmuir/analysis_2d.py b/Examples/Tests/langmuir/analysis_2d.py index e137d50229d..b3327703b82 100755 --- a/Examples/Tests/langmuir/analysis_2d.py +++ b/Examples/Tests/langmuir/analysis_2d.py @@ -38,6 +38,9 @@ # Parse test name and check if Vay current deposition (algo.current_deposition=vay) is used vay_deposition = True if re.search( 'Vay_deposition', fn ) else False +# Parse test name and check if particle_shape = 4 is used +particle_shape_4 = True if re.search('particle_shape_4', fn) else False + # Parameters (these parameters must match the parameters in `inputs.multi.rt`) epsilon = 0.01 n = 4.e24 @@ -114,7 +117,11 @@ def get_theoretical_field( field, t ): fig.tight_layout() fig.savefig('Langmuir_multi_2d_analysis.png', dpi = 200) -tolerance_rel = 0.05 +if particle_shape_4: +# lower fidelity, due to smoothing + tolerance_rel = 0.07 +else: + tolerance_rel = 0.05 print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4.json new file mode 100644 index 00000000000..10f16df934f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4.json @@ -0,0 +1,29 @@ +{ + "electrons": { + "particle_momentum_x": 5.696397887862144e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.696397887862156e-20, + "particle_position_x": 0.6553600000001614, + "particle_position_y": 0.6553600000001614, + "particle_weight": 3200000000000000.5 + }, + "lev=0": { + "Ex": 3616987165668.129, + "Ey": 0.0, + "Ez": 3616987165667.9756, + "divE": 2.269072850514105e+18, + "jx": 1.0121499183289864e+16, + "jy": 0.0, + "jz": 1.0121499183289892e+16, + "part_per_cell": 131072.0, + "rho": 20090797.21028113 + }, + "positrons": { + "particle_momentum_x": 5.696397887862144e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.696397887862156e-20, + "particle_position_x": 0.6553600000001614, + "particle_position_y": 0.6553600000001614, + "particle_weight": 3200000000000000.5 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 1e5e7c79a65..08c3a4bb805 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -1434,6 +1434,25 @@ particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png +[Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4] +buildDir = . +inputFile = Examples/Tests/langmuir/inputs_2d +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 +dim = 2 +addToCompileString = USE_PSATD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +restartTest = 0 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons positrons +analysisRoutine = Examples/Tests/langmuir/analysis_2d.py +analysisOutputImage = langmuir_multi_2d_analysis.png + [Langmuir_multi_2d_psatd_Vay_deposition_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index 670d95014a0..d43c6c0741d 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -1689,6 +1689,11 @@ void doGatherShapeN (const amrex::ParticleReal xp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 4) { + doGatherShapeN<4,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); } } else { if (nox == 1) { @@ -1706,6 +1711,11 @@ void doGatherShapeN (const amrex::ParticleReal xp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 4) { + doGatherShapeN<4,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); } } } @@ -1782,6 +1792,12 @@ void doGatherShapeNImplicit ( ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 4) { + doGatherShapeNEsirkepovStencilImplicit<4>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); } } else if (depos_type==3) { // CurrentDepositionAlgo::Villasenor @@ -1827,6 +1843,11 @@ void doGatherShapeNImplicit ( ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 4) { + doGatherShapeN<4,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); } } } diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index 73e8f7243bb..7c56da457ed 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -16,7 +16,7 @@ /** * Compute shape factor and return index of leftmost cell where * particle writes. - * Specializations are defined for orders 0 to 3 (using "if constexpr"). + * Specializations are defined for orders 0 to 4 (using "if constexpr"). * Shape factor functors may be evaluated with double arguments * in current deposition to ensure that current deposited by * particles that move only a small distance is still resolved. @@ -67,13 +67,11 @@ struct Compute_shape_factor else if constexpr (depos_order == 4){ const auto j = static_cast(xmid + T(0.5)); const T xint = xmid - T(j); - const T xint_p1 = xint + T(1.0); - const T xint_m1 = xint - T(1.0); - sx[0] = T(1.0)/T(384.0)*(T(1.0) - T(2.0)*xint)*(T(1.0) - T(2.0)*xint)*(T(1.0) - T(2.0)*xint)*(T(1.0) - T(2.0)*xint); - sx[1] = T(1.0)/T(96.0)*(T(55.0) + T(4.0)*xint_p1*(T(5.0) - T(2.0)*xint_p1*(T(15.0) + T(2.0)*xint_p1*(xint_p1 - T(5.0))))); - sx[2] = T(115.0)/T(192.0) + xint*xint*(xint*xint/T(4.0) - T(5.0)/T(8.0)); - sx[3] = T(1.0)/T(96.0)*(T(55.0) - T(4.0)*xint_m1*(T(5.0) + T(2.0)*xint_m1*(T(15.0) - T(2.0)*xint_m1*(-xint_m1 - T(5.0))))); - sx[4] = T(1.0)/T(384.0)*(T(1.0) + T(2.0)*xint)*(T(1.0) + T(2.0)*xint)*(T(1.0) + T(2.0)*xint)*(T(1.0) + T(2.0)*xint); + sx[0] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); + sx[1] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); + sx[2] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); + sx[3] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); + sx[4] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); // index of the leftmost cell where particle deposits return j-2; } @@ -90,7 +88,7 @@ struct Compute_shape_factor /** * Compute shifted shape factor and return index of leftmost cell where * particle writes, for Esirkepov algorithm. - * Specializations are defined below for orders 1, 2 and 3 (using "if constexpr"). + * Specializations are defined below for orders 1, 2, 3, and 4 (using "if constexpr"). */ template struct Compute_shifted_shape_factor @@ -129,7 +127,7 @@ struct Compute_shifted_shape_factor else if constexpr (depos_order == 3){ const auto i = static_cast(x_old); const int i_shift = i - (i_new + 1); - const T xint = x_old - i; + const T xint = x_old - T(i); sx[1+i_shift] = (T(1.0))/(T(6.0))*(T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - xint); sx[2+i_shift] = (T(2.0))/(T(3.0)) - xint*xint*(T(1.0) - xint/(T(2.0))); sx[3+i_shift] = (T(2.0))/(T(3.0)) - (T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - T(0.5)*(T(1.0) - xint)); @@ -137,6 +135,18 @@ struct Compute_shifted_shape_factor // index of the leftmost cell where particle deposits return i - 1; } + else if constexpr (depos_order == 4){ + const auto i = static_cast(x_old + T(0.5)); + const int i_shift = i - (i_new + 2); + const T xint = x_old - T(i); + sx[1+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); + sx[2+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); + sx[3+i_shift] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); + sx[4+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); + sx[5+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); + // index of the leftmost cell where particle deposits + return i - 2; + } else{ WARPX_ABORT_WITH_MESSAGE("Unknown particle shape selected in Compute_shifted_shape_factor"); amrex::ignore_unused(sx, x_old, i_new); @@ -209,22 +219,18 @@ struct Compute_shape_factor_pair else if constexpr (depos_order == 4){ const auto j = static_cast(xmid + T(0.5)); const T xint_old = xold - T(j); - T xint_p1 = xint_old + T(1.0); - T xint_m1 = xint_old - T(1.0); - sx_old[0] = T(1.0)/T(384.0)*(T(1.0) - T(2.0)*xint_old)*(T(1.0) - T(2.0)*xint_old)*(T(1.0) - T(2.0)*xint_old)*(T(1.0) - T(2.0)*xint_old); - sx_old[1] = T(1.0)/T(96.0)*(T(55.0) + T(4.0)*xint_p1*(T(5.0) - T(2.0)*xint_p1*(T(15.0) + T(2.0)*xint_p1*(xint_p1 - T(5.0))))); - sx_old[2] = T(115.0)/T(192.0) + xint_old*xint_old*(xint_old*xint_old/T(4.0) - T(5.0)/T(8.0)); - sx_old[3] = T(1.0)/T(96.0)*(T(55.0) - T(4.0)*xint_m1*(T(5.0) + T(2.0)*xint_m1*(T(15.0) - T(2.0)*xint_m1*(-xint_m1 - T(5.0))))); - sx_old[4] = T(1.0)/T(384.0)*(T(1.0) + T(2.0)*xint_old)*(T(1.0) + T(2.0)*xint_old)*(T(1.0) + T(2.0)*xint_old)*(T(1.0) + T(2.0)*xint_old); + sx_old[0] = (T(1.0))/(T(24.0))*(T(0.5) - xint_old)*(T(0.5) - xint_old)*(T(0.5) - xint_old)*(T(0.5) - xint_old); + sx_old[1] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint_old + T(4.0)*xint_old*xint_old*(T(1.5) + xint_old - xint_old*xint_old)); + sx_old[2] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint_old*xint_old*(xint_old*xint_old - T(2.5))); + sx_old[3] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint_old + T(4.0)*xint_old*xint_old*(T(1.5) - xint_old - xint_old*xint_old)); + sx_old[4] = (T(1.0))/(T(24.0))*(T(0.5) + xint_old)*(T(0.5) + xint_old)*(T(0.5) + xint_old)*(T(0.5)+xint_old); // const T xint_new = xnew - T(j); - xint_p1 = xint_new + T(1.0); - xint_m1 = xint_new - T(1.0); - sx_new[0] = T(1.0)/T(384.0)*(T(1.0) - T(2.0)*xint_new)*(T(1.0) - T(2.0)*xint_new)*(T(1.0) - T(2.0)*xint_new)*(T(1.0) - T(2.0)*xint_new); - sx_new[1] = T(1.0)/T(96.0)*(T(55.0) + T(4.0)*xint_p1*(T(5.0) - T(2.0)*xint_p1*(T(15.0) + T(2.0)*xint_p1*(xint_p1 - T(5.0))))); - sx_new[2] = T(115.0)/T(192.0) + xint_new*xint_new*(xint_new*xint_new/T(4.0) - T(5.0)/T(8.0)); - sx_new[3] = T(1.0)/T(96.0)*(T(55.0) - T(4.0)*xint_m1*(T(5.0) + T(2.0)*xint_m1*(T(15.0) - T(2.0)*xint_m1*(-xint_m1 - T(5.0))))); - sx_new[4] = T(1.0)/T(384.0)*(T(1.0) + T(2.0)*xint_new)*(T(1.0) + T(2.0)*xint_new)*(T(1.0) + T(2.0)*xint_new)*(T(1.0) + T(2.0)*xint_new); + sx_new[0] = (T(1.0))/(T(24.0))*(T(0.5) - xint_new)*(T(0.5) - xint_new)*(T(0.5) - xint_new)*(T(0.5) - xint_new); + sx_new[1] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint_new + T(4.0)*xint_new*xint_new*(T(1.5) + xint_new - xint_new*xint_new)); + sx_new[2] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint_new*xint_new*(xint_new*xint_new - T(2.5))); + sx_new[3] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint_new + T(4.0)*xint_new*xint_new*(T(1.5) - xint_new - xint_new*xint_new)); + sx_new[4] = (T(1.0))/(T(24.0))*(T(0.5) + xint_new)*(T(0.5) + xint_new)*(T(0.5) + xint_new)*(T(0.5)+xint_new); // // index of the leftmost cell where particle deposits return j-2; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 0d565c039e6..6e6c0b27c85 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -547,6 +547,13 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + } else if (WarpX::nox == 4){ + doDepositionSharedShapeN<4>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } WARPX_PROFILE_VAR_STOP(direct_current_dep_kernel); } @@ -576,6 +583,13 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doEsirkepovDepositionShapeN<4>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } } else if (push_type == PushType::Implicit) { #if (AMREX_SPACEDIM >= 2) @@ -622,6 +636,15 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doChargeConservingDepositionShapeNImplicit<4>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } } } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { @@ -709,6 +732,13 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doVayDepositionShapeN<4>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } } else { // Direct deposition if (push_type == PushType::Explicit) { @@ -733,6 +763,13 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doDepositionShapeN<4>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } } else if (push_type == PushType::Implicit) { auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); @@ -765,6 +802,15 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doDepositionShapeNImplicit<4>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } } } @@ -1064,6 +1110,12 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); + } else if (WarpX::nox == 4){ + doChargeDepositionSharedShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, + rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::shared_tilesize); } #ifndef AMREX_USE_GPU // CPU, tiling: atomicAdd local_rho into rho diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 5031381561f..b13d7aaf63b 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -440,6 +440,11 @@ WarpX::WarpX () costs_heuristic_cells_wt = 0.250_rt; costs_heuristic_particles_wt = 0.750_rt; break; + case 4: + // this is only a guess + costs_heuristic_cells_wt = 0.200_rt; + costs_heuristic_particles_wt = 0.800_rt; + break; } } else { // FDTD switch (WarpX::nox) @@ -456,6 +461,11 @@ WarpX::WarpX () costs_heuristic_cells_wt = 0.145_rt; costs_heuristic_particles_wt = 0.855_rt; break; + case 4: + // this is only a guess + costs_heuristic_cells_wt = 0.100_rt; + costs_heuristic_particles_wt = 0.900_rt; + break; } } #else // CPU @@ -1327,19 +1337,10 @@ WarpX::ReadParameters () int particle_shape; if (!species_names.empty() || !lasers_names.empty()) { if (utils::parser::queryWithParser(pp_algo, "particle_shape", particle_shape)){ - - if(current_deposition_algo == CurrentDepositionAlgo::Villasenor) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (particle_shape >= 1) && (particle_shape <=4), - "algo.particle_shape can be only 1, 2, 3, or 4 with villasenor deposition" - ); - } - else { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (particle_shape >= 1) && (particle_shape <=3), - "algo.particle_shape can be only 1, 2, or 3" - ); - } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (particle_shape >= 1) && (particle_shape <=4), + "algo.particle_shape can be only 1, 2, 3, or 4" + ); nox = particle_shape; noy = particle_shape; @@ -1348,8 +1349,7 @@ WarpX::ReadParameters () else{ WARPX_ABORT_WITH_MESSAGE( "algo.particle_shape must be set in the input file:" - " please set algo.particle_shape to 1, 2, or 3." - " if using the villasenor deposition, can use 4 also."); + " please set algo.particle_shape to 1, 2, 3, or 4"); } if ((maxLevel() > 0) && (particle_shape > 1) && (do_pml_j_damping == 1)) From f40084e7b59e770a791d18b6a5a50f7a41b4c98c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 21 Feb 2024 16:29:35 -0800 Subject: [PATCH 164/176] Cleanup some docstrings (#4714) --- Source/Particles/MultiParticleContainer.H | 31 +++++++++++------------ 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 285b0a9777c..9946a680fcd 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -95,11 +95,11 @@ public: void InitMultiPhysicsModules (); - /// - /// This evolves all the particles by one PIC time step, including current deposition, the - /// field solve, and pushing the particles, for all the species in the MultiParticleContainer. - /// This is the electromagnetic version. - /// + /** + * \brief This evolves all the particles by one PIC time step, including current deposition, the + * field solve, and pushing the particles, for all the species in the MultiParticleContainer. + * This is the electromagnetic version. + */ void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, @@ -111,19 +111,18 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, PushType push_type=PushType::Explicit); - /// - /// This pushes the particle positions by one half time step for all the species in the - /// MultiParticleContainer. It is used to desynchronize the particles after initialization - /// or when restarting from a checkpoint. - /// + /** + * \brief This pushes the particle positions by one time step for all the species in the + * MultiParticleContainer. + */ void PushX (amrex::Real dt); - /// - /// This pushes the particle momenta by dt for all the species in the - /// MultiParticleContainer. It is used to desynchronize the particles after initialization - /// or when restarting from a checkpoint. It is also used to synchronize particles at the - /// the end of the run. This is the electromagnetic version. - /// + /** + * This pushes the particle momenta by dt for all the species in the + * MultiParticleContainer. It is used to desynchronize the particles after initialization + * or when restarting from a checkpoint. It is also used to synchronize particles at the + * the end of the run. This is the electromagnetic version. + */ void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz); From 07738d37aa5390cdd7f9b8c7c9ce83b3520458ee Mon Sep 17 00:00:00 2001 From: Eya D <81635404+EyaDammak@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:40:43 -0800 Subject: [PATCH 165/176] Save the normal components in the particle buffer when there is EB (#4702) * modifications for adding the normal components * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update ParticleBoundaryBuffer.cpp * Update analysis.py * Update ParticleBoundaryBuffer.cpp * Update NodalFieldGather.H * Update NodalFieldGather.H * Update NodalFieldGather.H * Update Docs/source/usage/parameters.rst Co-authored-by: Remi Lehe * Update Source/EmbeddedBoundary/ParticleScraper.H Co-authored-by: Remi Lehe * Update Source/EmbeddedBoundary/ParticleScraper.H Co-authored-by: Remi Lehe * Update Source/EmbeddedBoundary/ParticleScraper.H Co-authored-by: Remi Lehe * Update Source/ablastr/particles/NodalFieldGather.H Co-authored-by: Remi Lehe * Update Source/ablastr/particles/NodalFieldGather.H Co-authored-by: Remi Lehe * Update Source/ablastr/particles/NodalFieldGather.H Co-authored-by: Remi Lehe * Update Source/ablastr/particles/NodalFieldGather.H Co-authored-by: Remi Lehe * Update Source/ablastr/particles/NodalFieldGather.H Co-authored-by: Remi Lehe * Update DistanceToEB.H * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update DistanceToEB.H * Update Source/Particles/ParticleBoundaryBuffer.cpp Co-authored-by: Remi Lehe * Update Source/Particles/ParticleBoundaryBuffer.cpp Co-authored-by: Remi Lehe * makes it concise * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update ParticleScraper.H * updates * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update DistanceToEB.H * Update DistanceToEB.H --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Remi Lehe --- Docs/source/usage/parameters.rst | 8 +- .../Tests/point_of_contact_EB/analysis.py | 18 ++- Python/pywarpx/particle_containers.py | 3 +- Source/EmbeddedBoundary/DistanceToEB.H | 111 ++++++++++++------ Source/EmbeddedBoundary/ParticleScraper.H | 12 +- Source/Particles/ParticleBoundaryBuffer.cpp | 41 ++++++- Source/ablastr/particles/NodalFieldGather.H | 36 +++--- 7 files changed, 161 insertions(+), 68 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 11dffae446b..a4bea338b6a 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2752,10 +2752,10 @@ The data collected at each boundary is written out to a subdirectory of the diag By default, all of the collected particle data is written out at the end of the simulation. Optionally, the ``.intervals`` parameter can be given to specify writing out the data more often. This can be important if a large number of particles are lost, avoiding filling up memory with the accumulated lost particle data. -In addition to their usual attributes, the saved particles have an integer attribute ``step_scraped``, which -indicates the PIC iteration at which each particle was absorbed at the boundary, -and a real attribute ``time_scraped``, which indicates the exact time calculated when each -particle hits the EB. +In addition to their usual attributes, the saved particles have + an integer attribute ``step_scraped``, which indicates the PIC iteration at which each particle was absorbed at the boundary, + a real attribute ``time_scraped``, which indicates the exact time calculated when each particle hit the boundary, + 3 real attributes ``nx``, ``ny``, ``nz``, which represents the three components of the normal to the boundary on the point of contact of the particles (not saved if they reach non-EB boundaries) ``BoundaryScrapingDiagnostics`` can be used with ``..random_fraction``, ``..uniform_stride``, and ``..plot_filter_function``, which have the same behavior as for ``FullDiagnostics``. For ``BoundaryScrapingDiagnostics``, these filters are applied at the time the data is written to file. An implication of this is that more particles may initially be accumulated in memory than are ultimately written. ``t`` in ``plot_filter_function`` refers to the time the diagnostic is written rather than the time the particle crossed the boundary. diff --git a/Examples/Tests/point_of_contact_EB/analysis.py b/Examples/Tests/point_of_contact_EB/analysis.py index 743cbc2fb39..9bf7ce1aae5 100755 --- a/Examples/Tests/point_of_contact_EB/analysis.py +++ b/Examples/Tests/point_of_contact_EB/analysis.py @@ -26,35 +26,45 @@ ts_scraping = OpenPMDTimeSeries('./diags/diag2/particles_at_eb/') it=ts_scraping.iterations -step_scraped, time_scraped, x, y, z=ts_scraping.get_particle( ['stepScraped','timeScraped','x','y','z'], species='electron', iteration=it ) +step_scraped, time_scraped, x, y, z, nx, ny, nz=ts_scraping.get_particle( ['stepScraped','timeScraped','x','y','z', 'nx', 'ny', 'nz'], species='electron', iteration=it ) time_scraped_reduced=time_scraped[0]*1e10 # Analytical results calculated x_analytic=-0.1983 y_analytic=0.02584 z_analytic=0.0000 +nx_analytic=-0.99 +ny_analytic=0.13 +nz_analytic=0.0 #result obtained by analysis of simulations step=3 time_reduced=3.58 print('NUMERICAL coordinates of the point of contact:') -print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f' % (step_scraped[0],time_reduced,x[0], y[0], z[0])) +print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f' % (step_scraped[0],time_reduced,x[0], y[0], z[0], nx[0], ny[0], nz[0])) print('\n') print('ANALYTICAL coordinates of the point of contact:') -print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f' % (step, time_reduced, x_analytic, y_analytic, z_analytic)) +print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f' % (step, time_reduced, x_analytic, y_analytic, z_analytic, nx_analytic, ny_analytic, nz_analytic)) tolerance=0.001 tolerance_t=0.003 +tolerance_n=0.01 print("tolerance = "+ str(tolerance *100) + '%') print("tolerance for the time = "+ str(tolerance_t *100) + '%') +print("tolerance for the normal components = "+ str(tolerance_n *100) + '%') diff_step=np.abs((step_scraped[0]-step)/step) diff_time=np.abs((time_scraped_reduced-time_reduced)/time_reduced) diff_x=np.abs((x[0]-x_analytic)/x_analytic) diff_y=np.abs((y[0]-y_analytic)/y_analytic) +diff_nx=np.abs((nx[0]-nx_analytic)/nx_analytic) +diff_ny=np.abs((ny[0]-ny_analytic)/ny_analytic) print("percentage error for x = %5.4f %%" %(diff_x *100)) print("percentage error for y = %5.4f %%" %(diff_y *100)) +print("percentage error for nx = %5.2f %%" %(diff_nx *100)) +print("percentage error for ny = %5.2f %%" %(diff_ny *100)) +print("nz = %5.2f " %(nz[0])) -assert (diff_x < tolerance) and (diff_y < tolerance) and (np.abs(z[0]) < 1e-8) and (diff_step < 1e-8) and (diff_time < tolerance_t) , 'Test point_of_contact did not pass' +assert (diff_x < tolerance) and (diff_y < tolerance) and (np.abs(z[0]) < 1e-8) and (diff_step < 1e-8) and (diff_time < tolerance_t) and (diff_nx < tolerance_n) and (diff_ny < tolerance_n) and (np.abs(nz) < 1e-8) , 'Test point_of_contact did not pass' diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index 43eefa0337d..9526e9322a2 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -774,7 +774,8 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) comp_name : str The component of the array data that will be returned. "x", "y", "z", "ux", "uy", "uz", "w" - "step_scraped","time_scraped", "nx", "ny", "nz" + "step_scraped","time_scraped", + if boundary='eb': "nx", "ny", "nz" level : int Which AMR level to retrieve scraped particle data from. diff --git a/Source/EmbeddedBoundary/DistanceToEB.H b/Source/EmbeddedBoundary/DistanceToEB.H index 8f00984eefa..b46def938db 100644 --- a/Source/EmbeddedBoundary/DistanceToEB.H +++ b/Source/EmbeddedBoundary/DistanceToEB.H @@ -34,56 +34,95 @@ void normalize (amrex::RealVect& a) noexcept a[2] *= inv_norm); } + + +// This function calculates the normal vector using the nodal and cell-centered data. +// i,j,k are the index of the nearest node to the left of the point at which we interpolate. +// W are the interpolation weight for the left and right nodes (for the 0th component and 1st component respectively) +// ic,jc,kc are the index of the nearest cell-center to the left of the point at which we interpolate. AMREX_GPU_HOST_DEVICE AMREX_INLINE amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SPACEDIM][2], + int ic, int jc, int kc, const amrex::Real Wc[AMREX_SPACEDIM][2], amrex::Array4 const& phi, amrex::GpuArray const& dxi) noexcept { + #if (defined WARPX_DIM_3D) amrex::RealVect normal{0.0, 0.0, 0.0}; + for (int iic = 0; iic < 2; ++iic) { + for (int kk = 0; kk < 2; ++kk) { + for (int jj=0; jj< 2; ++jj) { + for (int ii = 0; ii < 2; ++ii) { + int icstart = ic + iic; + amrex::Real sign = (ii%2)*2. - 1.; + int wccomp = static_cast(iic%2); + int w1comp = static_cast(jj%2); + int w2comp = static_cast(kk%2); + normal[0] += sign * phi(icstart + ii, j + jj, k + kk) * dxi[0] * Wc[0][wccomp] * W[1][w1comp] * W[2][w2comp]; + } + } + } + } + for (int iic = 0; iic < 2; ++iic) { + for (int kk = 0; kk < 2; ++kk) { + for (int ii=0; ii< 2; ++ii) { + for (int jj = 0; jj < 2; ++jj) { + int jcstart = jc + iic; + amrex::Real sign = (jj%2)*2. - 1.; + int wccomp = static_cast(iic%2); + int w1comp = static_cast(ii%2); + int w2comp = static_cast(kk%2); + normal[1] += sign * phi(i + ii, jcstart + jj, k + kk) * dxi[1] * W[0][w1comp] * Wc[1][wccomp] * W[2][w2comp]; + } + } + } + } + for (int iic = 0; iic < 2; ++iic) { + for (int jj = 0; jj < 2; ++jj) { + for (int ii=0; ii< 2; ++ii) { + for (int kk = 0; kk < 2; ++kk) { + int kcstart = kc + iic; + amrex::Real sign = (kk%2)*2. - 1.; + int wccomp = static_cast(iic%2); + int w1comp = static_cast(ii%2); + int w2comp = static_cast(jj%2); + normal[2] += sign * phi(i + ii, j + jj, kcstart + kk) * dxi[2] * W[0][w1comp] * W[1][w2comp] * Wc[2][wccomp]; + } + } + } + } - normal[0] -= phi(i, j , k ) * dxi[0] * W[1][0] * W[2][0]; - normal[0] += phi(i+1, j , k ) * dxi[0] * W[1][0] * W[2][0]; - normal[0] -= phi(i, j+1, k ) * dxi[0] * W[1][1] * W[2][0]; - normal[0] += phi(i+1, j+1, k ) * dxi[0] * W[1][1] * W[2][0]; - normal[0] -= phi(i, j , k+1) * dxi[0] * W[1][0] * W[2][1]; - normal[0] += phi(i+1, j , k+1) * dxi[0] * W[1][0] * W[2][1]; - normal[0] -= phi(i , j+1, k+1) * dxi[0] * W[1][1] * W[2][1]; - normal[0] += phi(i+1, j+1, k+1) * dxi[0] * W[1][1] * W[2][1]; - - normal[1] -= phi(i, j , k ) * dxi[1] * W[0][0] * W[2][0]; - normal[1] += phi(i , j+1, k ) * dxi[1] * W[0][0] * W[2][0]; - normal[1] -= phi(i+1, j , k ) * dxi[1] * W[0][1] * W[2][0]; - normal[1] += phi(i+1, j+1, k ) * dxi[1] * W[0][1] * W[2][0]; - normal[1] -= phi(i, j , k+1) * dxi[1] * W[0][0] * W[2][1]; - normal[1] += phi(i , j+1, k+1) * dxi[1] * W[0][0] * W[2][1]; - normal[1] -= phi(i+1, j , k+1) * dxi[1] * W[0][1] * W[2][1]; - normal[1] += phi(i+1, j+1, k+1) * dxi[1] * W[0][1] * W[2][1]; - - normal[2] -= phi(i , j , k ) * dxi[2] * W[0][0] * W[1][0]; - normal[2] += phi(i , j , k+1) * dxi[2] * W[0][0] * W[1][0]; - normal[2] -= phi(i+1, j , k ) * dxi[2] * W[0][1] * W[1][0]; - normal[2] += phi(i+1, j , k+1) * dxi[2] * W[0][1] * W[1][0]; - normal[2] -= phi(i, j+1, k ) * dxi[2] * W[0][0] * W[1][1]; - normal[2] += phi(i , j+1, k+1) * dxi[2] * W[0][0] * W[1][1]; - normal[2] -= phi(i+1, j+1, k ) * dxi[2] * W[0][1] * W[1][1]; - normal[2] += phi(i+1, j+1, k+1) * dxi[2] * W[0][1] * W[1][1]; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::RealVect normal{0.0, 0.0}; + for (int iic = 0; iic < 2; ++iic) { + for (int jj=0; jj< 2; ++jj) { + for (int ii = 0; ii < 2; ++ii) { + int icstart = ic + iic; + amrex::Real sign = (ii%2)*2. - 1.; + int wccomp = static_cast(iic%2); + int w1comp = static_cast(jj%2); + normal[0] += sign * phi(icstart + ii, j + jj, k) * dxi[0] * Wc[0][wccomp] * W[1][w1comp]; + } + } + } + for (int iic = 0; iic < 2; ++iic) { + for (int ii=0; ii< 2; ++ii) { + for (int jj = 0; jj < 2; ++jj) { + int jcstart = jc + iic; + amrex::Real sign = (jj%2)*2. - 1.; + int wccomp = static_cast(iic%2); + int w1comp = static_cast(ii%2); + normal[1] += sign * phi(i + ii, jcstart + jj, k) * dxi[1] * W[0][w1comp] * Wc[1][wccomp]; + } + } + } + amrex::ignore_unused(kc); - normal[0] -= phi(i, j , k) * dxi[0] * W[1][0]; - normal[0] += phi(i+1, j , k) * dxi[0] * W[1][0]; - normal[0] -= phi(i, j+1, k) * dxi[0] * W[1][1]; - normal[0] += phi(i+1, j+1, k) * dxi[0] * W[1][1]; - - normal[1] -= phi(i, j , k) * dxi[1] * W[0][0]; - normal[1] += phi(i , j+1, k) * dxi[1] * W[0][0]; - normal[1] -= phi(i+1, j , k) * dxi[1] * W[0][1]; - normal[1] += phi(i+1, j+1, k) * dxi[1] * W[0][1]; #else + amrex::ignore_unused(i, j, k, ic, jc, kc, W, Wc, phi, dxi); amrex::RealVect normal{0.0, 0.0}; - amrex::ignore_unused(i, j, k, W, phi, dxi); WARPX_ABORT_WITH_MESSAGE("Error: interp_distance not yet implemented in 1D"); + #endif return normal; } diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index 7531a03487b..7d6934cbb9c 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -10,6 +10,7 @@ #include "EmbeddedBoundary/DistanceToEB.H" #include "Particles/Pusher/GetAndSetPosition.H" + #include #include @@ -182,16 +183,17 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights_nodal(xp, yp, zp, plo, dxi, i, j, k, W); - + ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi); if (phi_value < 0.0) { - - amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, phi, dxi); + int ic, jc, kc; // Cell-centered indices + int nodal; + amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weights + ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, ic, jc, kc, Wc, nodal=0); + amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phi, dxi); DistanceToEB::normalize(normal); - amrex::RealVect pos; #if (defined WARPX_DIM_3D) pos[0] = xp; diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 2b90adac172..f91f7a348e4 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -44,9 +44,11 @@ struct IsOutsideDomainBoundary { } }; +#ifdef AMREX_USE_EB struct FindEmbeddedBoundaryIntersection { const int m_step_index; const int m_time_index; + const int m_normal_index; const int m_step; const amrex::Real m_dt; amrex::Array4 m_phiarr; @@ -94,7 +96,7 @@ struct FindEmbeddedBoundaryIntersection { amrex::Real W[AMREX_SPACEDIM][2]; amrex::ParticleReal x_temp=xp, y_temp=yp, z_temp=zp; UpdatePosition(x_temp, y_temp, z_temp, ux, uy, uz, -dt_frac*dt); - ablastr::particles::compute_weights_nodal(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phiarr); return phi_value; } ); @@ -108,26 +110,55 @@ struct FindEmbeddedBoundaryIntersection { amrex::ParticleReal x_temp=xp, y_temp=yp, z_temp=zp; UpdatePosition(x_temp, y_temp, z_temp, ux, uy, uz, -dt_fraction*m_dt); + // record the components of the normal on the destination + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); + int ic, jc, kc; // Cell-centered indices + int nodal; + amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weight + ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, ic, jc, kc, Wc, nodal=0); // nodal=0 to calculate the weights with respect to the cell-centered nodes + amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phiarr, dxi); + DistanceToEB::normalize(normal); + #if (defined WARPX_DIM_3D) dst.m_rdata[PIdx::x][dst_i] = x_temp; dst.m_rdata[PIdx::y][dst_i] = y_temp; dst.m_rdata[PIdx::z][dst_i] = z_temp; + //save normal components + dst.m_runtime_rdata[m_normal_index][dst_i] = normal[0]; + dst.m_runtime_rdata[m_normal_index+1][dst_i] = normal[1]; + dst.m_runtime_rdata[m_normal_index+2][dst_i] = normal[2]; #elif (defined WARPX_DIM_XZ) dst.m_rdata[PIdx::x][dst_i] = x_temp; dst.m_rdata[PIdx::z][dst_i] = z_temp; amrex::ignore_unused(y_temp); + //save normal components + dst.m_runtime_rdata[m_normal_index][dst_i] = normal[0]; + dst.m_runtime_rdata[m_normal_index+1][dst_i] = 0.0; + dst.m_runtime_rdata[m_normal_index+2][dst_i] = normal[1]; #elif (defined WARPX_DIM_RZ) dst.m_rdata[PIdx::x][dst_i] = std::sqrt(x_temp*x_temp + y_temp*y_temp); dst.m_rdata[PIdx::z][dst_i] = z_temp; dst.m_rdata[PIdx::theta][dst_i] = std::atan2(y_temp, x_temp); + //save normal components + amrex::Real theta=std::atan2(y_temp, x_temp); + dst.m_runtime_rdata[m_normal_index][dst_i] = normal[0]*std::cos(theta); + dst.m_runtime_rdata[m_normal_index+1][dst_i] = normal[0]*std::sin(theta); + dst.m_runtime_rdata[m_normal_index+2][dst_i] = normal[1]; #elif (defined WARPX_DIM_1D_Z) dst.m_rdata[PIdx::z][dst_i] = z_temp; amrex::ignore_unused(x_temp, y_temp); + //normal not defined + dst.m_runtime_rdata[m_normal_index][dst_i] = 0.0; + dst.m_runtime_rdata[m_normal_index+1][dst_i] = 0.0; + dst.m_runtime_rdata[m_normal_index+2][dst_i] = 0.0; #else - amrex::ignore_unused(x_temp, y_temp, z_temp); + amrex::ignore_unused(x_temp, y_temp, z_temp,normal); #endif } }; +#endif struct CopyAndTimestamp { int m_step_index; @@ -406,6 +437,9 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, buffer[i] = pc.make_alike(); buffer[i].AddIntComp("step_scraped", false); buffer[i].AddRealComp("time_scraped", false); + buffer[i].AddRealComp("nx", false); + buffer[i].AddRealComp("ny", false); + buffer[i].AddRealComp("nz", false); } auto& species_buffer = buffer[i]; for (int lev = 0; lev < pc.numLevels(); ++lev) @@ -462,12 +496,13 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, const int step_scraped_index = string_to_index_intcomp.at("step_scraped"); auto string_to_index_realcomp = buffer[i].getParticleRuntimeComps(); const int time_scraped_index = string_to_index_realcomp.at("time_scraped"); + const int normal_index = string_to_index_realcomp.at("nx"); const int step = warpx_instance.getistep(0); { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterTransformEB"); amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, - FindEmbeddedBoundaryIntersection{step_scraped_index,time_scraped_index, step, dt, phiarr, dxi, plo}, 0, dst_index); + FindEmbeddedBoundaryIntersection{step_scraped_index,time_scraped_index, normal_index, step, dt, phiarr, dxi, plo}, 0, dst_index); } } } diff --git a/Source/ablastr/particles/NodalFieldGather.H b/Source/ablastr/particles/NodalFieldGather.H index b0151627198..5a10c73ae20 100644 --- a/Source/ablastr/particles/NodalFieldGather.H +++ b/Source/ablastr/particles/NodalFieldGather.H @@ -18,30 +18,36 @@ namespace ablastr::particles { /** - * \brief Compute weight of each surrounding node in interpolating a nodal field - * to the given coordinates. + * \brief Compute weight of each surrounding node (or cell-centered nodes) in interpolating a nodal ((or a cell-centered node) field + * to the given coordinates. If nodal=1, then the calculations will be done with respect to the nodes (default). If nodal=0, then the calculations will be done with respect to the cell-centered nodal) * * This currently only does linear order. * * \param xp,yp,zp Particle position coordinates * \param plo Index lower bounds of domain. * \param dxi inverse 3D cell spacing - * \param i,j,k Variables to store indices of position on grid - * \param W 2D array of weights to store each neighbouring node + * \param i,j,k Variables to store indices of position on grid (nodal or cell-centered, depending of the value of `nodal`) + * \param W 2D array of weights to store each neighbouring node (or cell-centered node) + * \param nodal Int that tells if the weights are calculated in respect to the nodes (nodal=1) of the cell-centered nodes (nodal=0) */ AMREX_GPU_HOST_DEVICE AMREX_INLINE -void compute_weights_nodal (const amrex::ParticleReal xp, +void compute_weights (const amrex::ParticleReal xp, const amrex::ParticleReal yp, const amrex::ParticleReal zp, amrex::GpuArray const& plo, amrex::GpuArray const& dxi, - int& i, int& j, int& k, amrex::Real W[AMREX_SPACEDIM][2]) noexcept + int& i, int& j, int& k, amrex::Real W[AMREX_SPACEDIM][2], int nodal=1) noexcept { using namespace amrex::literals; + +#if !((nodal==0)||(nodal==1)) + ABLASTR_ABORT_WITH_MESSAGE("Error: 'nodal' has to be equal to 0 or 1"); +#endif + #if (defined WARPX_DIM_3D) - const amrex::Real x = (xp - plo[0]) * dxi[0]; - const amrex::Real y = (yp - plo[1]) * dxi[1]; - const amrex::Real z = (zp - plo[2]) * dxi[2]; + const amrex::Real x = (xp - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; + const amrex::Real y = (yp - plo[1]) * dxi[1] + static_cast(nodal-1)*0.5_rt; + const amrex::Real z = (zp - plo[2]) * dxi[2] + static_cast(nodal-1)*0.5_rt; i = static_cast(amrex::Math::floor(x)); j = static_cast(amrex::Math::floor(y)); @@ -58,17 +64,17 @@ void compute_weights_nodal (const amrex::ParticleReal xp, #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) # if (defined WARPX_DIM_XZ) - const amrex::Real x = (xp - plo[0]) * dxi[0]; + const amrex::Real x = (xp - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; amrex::ignore_unused(yp); i = static_cast(amrex::Math::floor(x)); W[0][1] = x - i; # elif (defined WARPX_DIM_RZ) - const amrex::Real r = (std::sqrt(xp*xp+yp*yp) - plo[0]) * dxi[0]; + const amrex::Real r = (std::sqrt(xp*xp+yp*yp) - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; i = static_cast(amrex::Math::floor(r)); W[0][1] = r - i; # endif - const amrex::Real z = (zp - plo[1]) * dxi[1]; + const amrex::Real z = (zp - plo[1]) * dxi[1] + static_cast(nodal-1)*0.5_rt; j = static_cast(amrex::Math::floor(z)); W[1][1] = z - j; @@ -77,7 +83,7 @@ void compute_weights_nodal (const amrex::ParticleReal xp, k = 0; #else - amrex::ignore_unused(xp, yp, zp, plo, dxi, i, j, k, W); + amrex::ignore_unused(xp, yp, zp, plo, dxi, i, j, k, W, nodal); ABLASTR_ABORT_WITH_MESSAGE("Error: compute_weights not yet implemented in 1D"); #endif } @@ -138,7 +144,7 @@ amrex::Real doGatherScalarFieldNodal (const amrex::ParticleReal xp, // first find the weight of surrounding nodes to use during interpolation int ii, jj, kk; amrex::Real W[AMREX_SPACEDIM][2]; - compute_weights_nodal(xp, yp, zp, lo, dxi, ii, jj, kk, W); + compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); return interp_field_nodal(ii, jj, kk, W, scalar_field); } @@ -166,7 +172,7 @@ doGatherVectorFieldNodal (const amrex::ParticleReal xp, // first find the weight of surrounding nodes to use during interpolation int ii, jj, kk; amrex::Real W[AMREX_SPACEDIM][2]; - compute_weights_nodal(xp, yp, zp, lo, dxi, ii, jj, kk, W); + compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); amrex::GpuArray const field_interp = { interp_field_nodal(ii, jj, kk, W, vector_field_x), From d45d1734eae28eec9c1c27c881287704831cadb0 Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Thu, 22 Feb 2024 17:44:26 -0800 Subject: [PATCH 166/176] Add focusing position to Gaussian beam initialization (#4639) * add focusing position * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * removed prints * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix stashed * generalized to focal distance * first test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * ci test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix formatting in benchmark checksums * updated test and start docs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * updated test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * focal distance is optional * updated test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixed unused variables vbulk * updated docs * updated test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * reset benchmark * vx vy mistake * fix conversion error clang tidy * fix clang tidy error * fix typo * Update Docs/source/usage/parameters.rst Co-authored-by: Remi Lehe * Update Examples/Tests/gaussian_beam/analysis_focusing_beam.py Co-authored-by: Remi Lehe * Update Examples/Tests/gaussian_beam/analysis_focusing_beam.py Co-authored-by: Remi Lehe * Update Source/Particles/PhysicalParticleContainer.cpp Co-authored-by: Remi Lehe * Update Examples/Tests/gaussian_beam/analysis_focusing_beam.py Co-authored-by: Remi Lehe * Update Source/Particles/PhysicalParticleContainer.cpp Co-authored-by: Remi Lehe * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Source/Particles/PhysicalParticleContainer.cpp Co-authored-by: Remi Lehe * Update Source/Particles/PhysicalParticleContainer.cpp Co-authored-by: Remi Lehe * integrate comments by remi * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove unused variable * reset benchmark * fix v_dot_n * Update Source/Particles/PhysicalParticleContainer.cpp Co-authored-by: Remi Lehe * Update Source/Particles/PhysicalParticleContainer.cpp Co-authored-by: Remi Lehe * Update Source/Particles/PhysicalParticleContainer.cpp Co-authored-by: Remi Lehe * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Tools Co-authored-by: Remi Lehe --- Docs/source/usage/parameters.rst | 6 +- .../gaussian_beam/analysis_focusing_beam.py | 68 ++++++++++ .../Tests/gaussian_beam/inputs_focusing_beam | 120 ++++++++++++++++++ .../focusing_gaussian_beam.json | 14 ++ Regression/WarpX-tests.ini | 18 +++ Source/Initialization/PlasmaInjector.H | 2 + Source/Initialization/PlasmaInjector.cpp | 6 + Source/Particles/PhysicalParticleContainer.H | 3 +- .../Particles/PhysicalParticleContainer.cpp | 55 ++++++-- 9 files changed, 282 insertions(+), 10 deletions(-) create mode 100755 Examples/Tests/gaussian_beam/analysis_focusing_beam.py create mode 100644 Examples/Tests/gaussian_beam/inputs_focusing_beam create mode 100644 Regression/Checksum/benchmarks_json/focusing_gaussian_beam.json diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index a4bea338b6a..f7462351800 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -779,7 +779,7 @@ Particle initialization * ``.q_tot`` (beam charge), - * ``.npart`` (number of particles in the beam), + * ``.npart`` (number of macroparticles in the beam), * ``.x/y/z_m`` (average position in `x/y/z`), @@ -797,6 +797,10 @@ Particle initialization If set to 4, symmetrization is in the x and y direction, (x,y) (-x,y) (x,-y) (-x,-y). If set to 8, symmetrization is also done with x and y exchanged, (y,x), (-y,x), (y,-x), (-y,-x)). + * ``.focal_distance`` (optional, distance between the beam centroid and the position of the focal plane of the beam, along the direction of the beam mean velocity; space charge is ignored in the initialization of the particles) + + If ``.focal_distance`` is specified, ``x_rms``, ``y_rms`` and ``z_rms`` are the size of the beam in the focal plane. Since the beam is not necessarily initialized close to its focal plane, the initial size of the beam will differ from ``x_rms``, ``y_rms``, ``z_rms``. + * ``external_file``: Inject macroparticles with properties (mass, charge, position, and momentum - :math:`\gamma \beta m c`) read from an external openPMD file. With it users can specify the additional arguments: diff --git a/Examples/Tests/gaussian_beam/analysis_focusing_beam.py b/Examples/Tests/gaussian_beam/analysis_focusing_beam.py new file mode 100755 index 00000000000..4a5fa3b927b --- /dev/null +++ b/Examples/Tests/gaussian_beam/analysis_focusing_beam.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Arianna Formenti +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +import os +import sys + +import numpy as np +from scipy.constants import c, eV, m_e, micro, nano + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') + +import checksumAPI +from openpmd_viewer import OpenPMDTimeSeries + +GeV=1e9*eV +energy = 125.*GeV +gamma = energy/(m_e*c**2) +sigmax = 516.0*nano +sigmay = 7.7*nano +sigmaz = 300.*micro +nz = 256 +Lz = 20*sigmaz +gridz = np.linspace(-0.5*Lz, 0.5*Lz, nz) +tol = gridz[1] - gridz[0] +emitx = 50*micro +emity = 20*nano +focal_distance = 4*sigmaz + +def s(z, sigma0, emit): + '''The theoretical size of a focusing beam (in the absence of space charge), + at position z, given its emittance and size at focus.''' + return np.sqrt(sigma0**2 + emit**2 * (z - focal_distance)**2 / sigma0**2) + +filename = sys.argv[1] + +ts = OpenPMDTimeSeries('./diags/openpmd/') + +x, y, z, w, = ts.get_particle( ['x', 'y', 'z', 'w'], species='beam1', iteration=0, plot=False) + +imin = np.argmin(np.sqrt((gridz+0.8*focal_distance)**2)) +imax = np.argmin(np.sqrt((gridz-0.8*focal_distance)**2)) + +sx, sy = [], [] +# Compute the size of the beam in each z slice +subgrid = gridz[imin:imax] +for d in subgrid: + i = np.sqrt((z - d)**2) < tol + if (np.sum(i)!=0): + mux = np.average(x[i], weights=w[i]) + muy = np.average(y[i], weights=w[i]) + sx.append(np.sqrt(np.average((x[i]-mux)**2, weights=w[i]))) + sy.append(np.sqrt(np.average((y[i]-muy)**2, weights=w[i]))) + +# Theoretical prediction for the size of the beam in each z slice +sx_theory = s(subgrid, sigmax, emitx/gamma) +sy_theory = s(subgrid, sigmay, emity/gamma) + +assert(np.allclose(sx, sx_theory, rtol=0.051, atol=0)) +assert(np.allclose(sy, sy_theory, rtol=0.038, atol=0)) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/gaussian_beam/inputs_focusing_beam b/Examples/Tests/gaussian_beam/inputs_focusing_beam new file mode 100644 index 00000000000..08e978671fc --- /dev/null +++ b/Examples/Tests/gaussian_beam/inputs_focusing_beam @@ -0,0 +1,120 @@ +################################# +########## MY CONSTANTS ######### +################################# +my_constants.mc2 = m_e*clight*clight +my_constants.nano = 1.0e-9 +my_constants.micro = 1.e-6 +my_constants.GeV = q_e*1.e9 + +# BEAMS +my_constants.energy = 125.*GeV +my_constants.gamma = energy/mc2 +my_constants.npart = 2.e10 +my_constants.nmacropart = 1000000 +my_constants.charge = q_e * npart + +my_constants.sigmax = 516.0*nano +my_constants.sigmay = 7.7*nano +my_constants.sigmaz = 300.*micro + +my_constants.mux = 0.0 +my_constants.muy = 0.0 +my_constants.muz = 0.0 + +my_constants.ux = 0.0 +my_constants.uy = 0.0 +my_constants.uz = gamma + +my_constants.emitx = 50*micro +my_constants.emity = 20*nano +my_constants.emitz = 0. + +my_constants.dux = emitx / sigmax +my_constants.duy = emity / sigmay +my_constants.duz = emitz / sigmaz + +my_constants.focal_distance = 4*sigmaz + +# BOX +my_constants.Lx = 20*sigmax +my_constants.Ly = 20*sigmay +my_constants.Lz = 20*sigmaz +my_constants.nx = 256 +my_constants.ny = 256 +my_constants.nz = 256 + + + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 0 +amr.n_cell = nx ny nz +amr.max_grid_size = 256 +amr.blocking_factor = 2 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.5*Lx -0.5*Ly -0.5*Lz +geometry.prob_hi = 0.5*Lx 0.5*Ly 0.5*Lz + +################################# +######## BOUNDARY CONDITION ##### +################################# +boundary.field_lo = PEC PEC PEC +boundary.field_hi = PEC PEC PEC +boundary.particle_lo = Absorbing Absorbing Absorbing +boundary.particle_hi = Absorbing Absorbing Absorbing + +################################# +############ NUMERICS ########### +################################# +algo.particle_shape = 3 + +################################# +########### PARTICLES ########### +################################# +particles.species_names = beam1 + +beam1.species_type = electron +beam1.injection_style = gaussian_beam +beam1.x_rms = sigmax +beam1.y_rms = sigmay +beam1.z_rms = sigmaz +beam1.x_m = muy +beam1.y_m = mux +beam1.z_m = muz +beam1.focal_distance = focal_distance +beam1.npart = nmacropart +beam1.q_tot = -charge + +beam1.momentum_distribution_type = gaussian +beam1.ux_m = ux +beam1.uy_m = uy +beam1.uz_m = uz +beam1.ux_th = dux +beam1.uy_th = duy +beam1.uz_th = duz + +################################# +######### DIAGNOSTICS ########### +################################# +# FULL +diagnostics.diags_names = diag1 openpmd + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.write_species = 1 +diag1.species = beam1 +diag1.fields_to_plot = rho_beam1 +diag1.format = plotfile +diag1.dump_last_timestep = 1 + + +openpmd.intervals = 1 +openpmd.diag_type = Full +openpmd.write_species = 1 +openpmd.species = beam1 +openpmd.beam1.variables = w x y z +openpmd.fields_to_plot = none +openpmd.format = openpmd +openpmd.dump_last_timestep = 1 diff --git a/Regression/Checksum/benchmarks_json/focusing_gaussian_beam.json b/Regression/Checksum/benchmarks_json/focusing_gaussian_beam.json new file mode 100644 index 00000000000..e7c4ec424dc --- /dev/null +++ b/Regression/Checksum/benchmarks_json/focusing_gaussian_beam.json @@ -0,0 +1,14 @@ +{ + "lev=0": { + "rho_beam1": 5637659324885.731 + }, + "beam1": { + "particle_momentum_x": 2.113335069047891e-14, + "particle_momentum_y": 5.653782702472731e-16, + "particle_momentum_z": 6.68023056405556e-11, + "particle_position_x": 0.5640698094900541, + "particle_position_y": 0.011947117763454793, + "particle_position_z": 239.4325333696459, + "particle_weight": 19999620000.0 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 08c3a4bb805..c9feea370af 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4627,3 +4627,21 @@ compileTest = 0 doVis = 0 compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py + +[focusing_gaussian_beam] +buildDir = . +inputFile = Examples/Tests/gaussian_beam/inputs_focusing_beam +runtime_params = +dim = 3 +addToCompileString = USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +doComparison = 0 +analysisRoutine = Examples/Tests/gaussian_beam/analysis_focusing_beam.py diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index 616b1a74b17..2651aad455f 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -112,6 +112,8 @@ public: long npart; int do_symmetrize = 0; int symmetrization_order = 4; + bool do_focusing = false; + amrex::Real focal_distance; bool external_file = false; //! initialize from an openPMD file amrex::Real z_shift = 0.0; //! additional z offset for particle positions diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index 008709abe48..7ae5a9db3d3 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -232,6 +232,11 @@ void PlasmaInjector::setupGaussianBeam (amrex::ParmParse const& pp_species) utils::parser::getWithParser(pp_species, source_name, "npart", npart); utils::parser::queryWithParser(pp_species, source_name, "do_symmetrize", do_symmetrize); utils::parser::queryWithParser(pp_species, source_name, "symmetrization_order", symmetrization_order); + const bool focusing_is_specified = pp_species.contains("focal_distance"); + if(focusing_is_specified){ + do_focusing = true; + utils::parser::queryWithParser(pp_species, source_name, "focal_distance", focal_distance); + } const std::set valid_symmetries = {4,8}; WARPX_ALWAYS_ASSERT_WITH_MESSAGE( valid_symmetries.count(symmetrization_order), "Error: Symmetrization only supported to orders 4 or 8 "); @@ -240,6 +245,7 @@ void PlasmaInjector::setupGaussianBeam (amrex::ParmParse const& pp_species) ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel); + #if defined(WARPX_DIM_XZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( y_rms > 0._rt, "Error: Gaussian beam y_rms must be strictly greater than 0 in 2D " diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index edf91a84526..b77c1147a15 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -231,7 +231,8 @@ public: amrex::Real x_m, amrex::Real y_m, amrex::Real z_m, amrex::Real x_rms, amrex::Real y_rms, amrex::Real z_rms, amrex::Real x_cut, amrex::Real y_cut, amrex::Real z_cut, - amrex::Real q_tot, long npart, int do_symmetrize, int symmetrization_order); + amrex::Real q_tot, long npart, int do_symmetrize, int symmetrization_order, + amrex::Real focal_distance); /** Load a particle beam from an external file * @param[in] the PlasmaInjector instance holding the input parameters diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index da1655b9dab..3d5cbe0007d 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -528,7 +528,7 @@ PhysicalParticleContainer::AddGaussianBeam ( const Real x_cut, const Real y_cut, const Real z_cut, const Real q_tot, long npart, const int do_symmetrize, - const int symmetrization_order) { + const int symmetrization_order, const Real focal_distance) { // Declare temporary vectors on the CPU Gpu::HostVector particle_x; @@ -549,28 +549,65 @@ PhysicalParticleContainer::AddGaussianBeam ( for (long i = 0; i < npart; ++i) { #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) const Real weight = q_tot/(npart*charge); - const Real x = amrex::RandomNormal(x_m, x_rms); - const Real y = amrex::RandomNormal(y_m, y_rms); - const Real z = amrex::RandomNormal(z_m, z_rms); + Real x = amrex::RandomNormal(x_m, x_rms); + Real y = amrex::RandomNormal(y_m, y_rms); + Real z = amrex::RandomNormal(z_m, z_rms); #elif defined(WARPX_DIM_XZ) const Real weight = q_tot/(npart*charge*y_rms); - const Real x = amrex::RandomNormal(x_m, x_rms); + Real x = amrex::RandomNormal(x_m, x_rms); constexpr Real y = 0._prt; - const Real z = amrex::RandomNormal(z_m, z_rms); + Real z = amrex::RandomNormal(z_m, z_rms); #elif defined(WARPX_DIM_1D_Z) const Real weight = q_tot/(npart*charge*x_rms*y_rms); constexpr Real x = 0._prt; constexpr Real y = 0._prt; - const Real z = amrex::RandomNormal(z_m, z_rms); + Real z = amrex::RandomNormal(z_m, z_rms); #endif if (plasma_injector.insideBounds(x, y, z) && std::abs( x - x_m ) <= x_cut * x_rms && std::abs( y - y_m ) <= y_cut * y_rms && std::abs( z - z_m ) <= z_cut * z_rms ) { XDim3 u = plasma_injector.getMomentum(x, y, z); + + if (plasma_injector.do_focusing){ + XDim3 u_bulk = plasma_injector.getInjectorMomentumHost()->getBulkMomentum(x,y,z); + Real u_bulk_norm = std::sqrt( u_bulk.x*u_bulk.x+u_bulk.y*u_bulk.y+u_bulk.z*u_bulk.z ); + + // Compute the position of the focal plane + // (it is located at a distance `focal_distance` from the beam centroid, in the direction of the bulk velocity) + Real n_x = u_bulk.x/u_bulk_norm; + Real n_y = u_bulk.y/u_bulk_norm; + Real n_z = u_bulk.z/u_bulk_norm; + Real x_f = x_m + focal_distance * n_x; + Real y_f = y_m + focal_distance * n_y; + Real z_f = z_m + focal_distance * n_z; + Real gamma = std::sqrt( 1._rt + (u.x*u.x+u.y*u.y+u.z*u.z) ); + + Real v_x = u.x / gamma * PhysConst::c; + Real v_y = u.y / gamma * PhysConst::c; + Real v_z = u.z / gamma * PhysConst::c; + + // Compute the time at which the particle will cross the focal plane + Real v_dot_n = v_x * n_x + v_y * n_y + v_z * n_z; + Real t = ((x_f-x)*n_x + (y_f-y)*n_y + (z_f-z)*n_z) / v_dot_n; + + // Displace particles in the direction orthogonal to the beam bulk momentum + // i.e. orthogonal to (n_x, n_y, n_z) +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + x = x - (v_x - v_dot_n*n_x) * t; + y = y - (v_y - v_dot_n*n_y) * t; + z = z - (v_z - v_dot_n*n_z) * t; +#elif defined(WARPX_DIM_XZ) + x = x - (v_x - v_dot_n*n_x) * t; + z = z - (v_z - v_dot_n*n_z) * t; +#elif defined(WARPX_DIM_1D_Z) + z = z - (v_z - v_dot_n*n_z) * t; +#endif + } u.x *= PhysConst::c; u.y *= PhysConst::c; u.z *= PhysConst::c; + if (do_symmetrize && symmetrization_order == 8){ // Add eight particles to the beam: CheckAndAddParticle(x, y, z, u.x, u.y, u.z, weight/8._rt, @@ -634,6 +671,7 @@ PhysicalParticleContainer::AddGaussianBeam ( } // Add the temporary CPU vectors to the particle structure auto const np = static_cast(particle_z.size()); + amrex::Vector xp(particle_x.data(), particle_x.data() + np); amrex::Vector yp(particle_y.data(), particle_y.data() + np); amrex::Vector zp(particle_z.data(), particle_z.data() + np); @@ -893,7 +931,8 @@ PhysicalParticleContainer::AddParticles (int lev) plasma_injector->q_tot, plasma_injector->npart, plasma_injector->do_symmetrize, - plasma_injector->symmetrization_order); + plasma_injector->symmetrization_order, + plasma_injector->focal_distance); } if (plasma_injector->external_file) { From daa94bb1f149383b35004dca0c5df4dca7865a25 Mon Sep 17 00:00:00 2001 From: Eya D <81635404+EyaDammak@users.noreply.github.com> Date: Thu, 22 Feb 2024 21:00:52 -0800 Subject: [PATCH 167/176] Update the documentation particles.rst in adding the new attributes (#4719) * Update particles.rst * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update particles.rst * Update particles.rst --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/developers/particles.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Docs/source/developers/particles.rst b/Docs/source/developers/particles.rst index 4b203e483c4..53a0090b5c9 100644 --- a/Docs/source/developers/particles.rst +++ b/Docs/source/developers/particles.rst @@ -118,6 +118,21 @@ Attribute name ``int``/``real`` Description Wher where the particle was created. idcpu ``cpu`` ``int`` CPU index where the particle SoA CT Last 24 bytes of idcpu was created. +``stepScraped`` ``int`` PIC iteration of the last step SoA RT Added when there is + before the particle hits the particle-boundary + boundary. interaction. + Saved in the boundary + buffers. +``deltaTimeScraped`` ``real`` Difference of time between the SoA RT Added when there is + ``stepScraped`` and the exact time particle-boundary + when the particle hits the interaction. + boundary. Saved in the boundary + buffers. +``n_x/y/z`` ``real`` Normal components to the boundary SoA RT Added when there is + on the position where the particle particle-boundary + hits the boundary. interaction. + Saved in the boundary + buffers. ``ionizationLevel`` ``int`` Ion ionization level SoA RT Added when ionization physics is used. ``opticalDepthQSR`` ``real`` QED: optical depth of the Quantum- SoA RT Added when PICSAR QED From 511fb371ce98e33a872246579a4b6d2d5436ac65 Mon Sep 17 00:00:00 2001 From: Marco Garten Date: Thu, 22 Feb 2024 23:30:57 -0800 Subject: [PATCH 168/176] Fix unresolved reference (#4722) The variable `species` was an unresolved reference and needed to accessed via `self.species` instead. Fixed also a word `specie` where `species` was meant. --- Python/pywarpx/picmi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index f11ecb379f2..02c9e38c8d5 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2469,7 +2469,7 @@ def diagnostic_initialize_inputs(self): if self.species is None: species_names = pywarpx.particles.species_names elif np.iterable(self.species): - species_names = [specie.name for specie in self.species] + species_names = [species.name for species in self.species] else: species_names = [self.species.name] @@ -2696,9 +2696,9 @@ def diagnostic_initialize_inputs(self): if self.species is None: species_names = pywarpx.particles.species_names elif np.iterable(self.species): - species_names = [specie.name for specie in self.species] + species_names = [species.name for species in self.species] else: - species_names = [species.name] + species_names = [self.species.name] for name in species_names: diag = pywarpx.Bucket.Bucket(self.name + '.' + name, From 4ef16e54c644d5018361ca836125237b052de954 Mon Sep 17 00:00:00 2001 From: Marco Garten Date: Thu, 22 Feb 2024 23:31:51 -0800 Subject: [PATCH 169/176] Improve Documentation for Laser-Ion Acceleration from Planar Target (#4569) * Update laser-ion acc example - add PICMI input - add vis script - update docs - add Python regression test - update phase space analysis script - add checksum regression test * Update docs with figures from hi-res runs * Add captions * Fix checksum regression tests * Change outputs to h5 for CI * Fix: Do Not Deselect Positions * Synchronize PICMI and Inputs More - Species selections - Filter details * Fix checksum regression test Update checksums in Python_LaserIonAcc2d Co-authored-by: Axel Huebl --- .../laser_ion/PICMI_inputs_2d.py | 313 ++++++++++++++++++ .../Physics_applications/laser_ion/README.rst | 71 +++- .../laser_ion/analysis_histogram_2D.py | 44 ++- .../Physics_applications/laser_ion/inputs_2d | 42 ++- .../Physics_applications/laser_ion/plot_2d.py | 229 +++++++++++++ .../benchmarks_json/LaserIonAcc2d.json | 51 +-- .../benchmarks_json/Python_LaserIonAcc2d.json | 34 ++ Regression/WarpX-tests.ini | 24 +- 8 files changed, 732 insertions(+), 76 deletions(-) create mode 100755 Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py create mode 100644 Examples/Physics_applications/laser_ion/plot_2d.py create mode 100644 Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json diff --git a/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py b/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py new file mode 100755 index 00000000000..844501992c3 --- /dev/null +++ b/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c +q_e = picmi.constants.q_e + +# We only run 100 steps for tests +# Disable `max_step` below to run until the physical `stop_time`. +max_step = 100 +# time-scale with highly kinetic dynamics +stop_time = 0.2e-12 + +# proper resolution for 30 n_c (dx<=3.33nm) incl. acc. length +# (>=6x V100) +# --> choose larger `max_grid_size` and `blocking_factor` for 1 to 8 grids per GPU accordingly +#nx = 7488 +#nz = 14720 + +# Number of cells +nx = 384 +nz = 512 + +# Domain decomposition (deactivate `warpx_numprocs` in `picmi.Simulation` for this to take effect) +max_grid_size = 64 +blocking_factor = 32 + +# Physical domain +xmin = -7.5e-06 +xmax = 7.5e-06 +zmin = -5.0e-06 +zmax = 25.0e-06 + +# Create grid +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, nz], + lower_bound=[xmin, zmin], + upper_bound=[xmax, zmax], + lower_boundary_conditions=['open', 'open'], + upper_boundary_conditions=['open', 'open'], + lower_boundary_conditions_particles=['absorbing', 'absorbing'], + upper_boundary_conditions_particles=['absorbing', 'absorbing'], + warpx_max_grid_size=max_grid_size, + warpx_blocking_factor=blocking_factor) + +# Particles: plasma parameters +# critical plasma density +nc = 1.742e27 # [m^-3] 1.11485e21 * 1.e6 / 0.8**2 +# number density: "fully ionized" electron density as reference +# [material 1] cryogenic H2 +n0 = 30.0 # [n_c] +# [material 2] liquid crystal +# n0 = 192 +# [material 3] PMMA +# n0 = 230 +# [material 4] Copper (ion density: 8.49e28/m^3; times ionization level) +# n0 = 1400 +plasma_density = n0 * nc +preplasma_L = 0.05e-6 # [m] scale length (>0) +preplasma_Lcut = 2.0e-6 # [m] hard cutoff from surface +plasma_r0 = 2.5e-6 # [m] radius or half-thickness +plasma_eps_z = 0.05e-6 # [m] small offset in z to make zmin, zmax interval larger than 2*(r0 + Lcut) +plasma_creation_limit_z = plasma_r0 + preplasma_Lcut + plasma_eps_z # [m] upper limit in z for particle creation + +plasma_xmin = None +plasma_ymin = None +plasma_zmin = -plasma_creation_limit_z +plasma_xmax = None +plasma_ymax = None +plasma_zmax = plasma_creation_limit_z + +density_expression_str = f'{plasma_density}*((abs(z)<={plasma_r0}) + (abs(z)<{plasma_r0}+{preplasma_Lcut}) * (abs(z)>{plasma_r0}) * exp(-(abs(z)-{plasma_r0})/{preplasma_L}))' + +slab_with_ramp_dist_hydrogen = picmi.AnalyticDistribution( + density_expression=density_expression_str, + lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], + upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax] +) + +# thermal velocity spread for electrons in gamma*beta +ux_th = .01 +uz_th = .01 + +slab_with_ramp_dist_electrons = picmi.AnalyticDistribution( + density_expression=density_expression_str, + lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], + upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax], + # if `momentum_expressions` and `momentum_spread_expressions` are unset, + # a Gaussian momentum distribution is assumed given that `rms_velocity` has any non-zero elements + rms_velocity=[c*ux_th, 0., c*uz_th] # thermal velocity spread in m/s +) + +# TODO: add additional attributes orig_x and orig_z +electrons = picmi.Species( + particle_type='electron', + name='electrons', + initial_distribution=slab_with_ramp_dist_electrons, +) + +# TODO: add additional attributes orig_x and orig_z +hydrogen = picmi.Species( + particle_type='proton', + name='hydrogen', + initial_distribution=slab_with_ramp_dist_hydrogen +) + +# Laser +# e_max = a0 * 3.211e12 / lambda_0[mu] +# a0 = 16, lambda_0 = 0.8mu -> e_max = 64.22 TV/m +e_max = 64.22e12 +position_z = -4.0e-06 +profile_t_peak = 50.e-15 +profile_focal_distance = 4.0e-06 +laser = picmi.GaussianLaser( + wavelength=0.8e-06, + waist=4.e-06, + duration=30.e-15, + focal_position=[0, 0, profile_focal_distance + position_z], + centroid_position=[0, 0, position_z - c * profile_t_peak], + propagation_direction=[0, 0, 1], + polarization_direction=[1, 0, 0], + E0=e_max, + fill_in=False) +laser_antenna = picmi.LaserAntenna( + position=[0., 0., position_z], + normal_vector=[0, 0, 1]) + +# Electromagnetic solver +solver = picmi.ElectromagneticSolver( + grid=grid, + method='Yee', + cfl=0.999, + divE_cleaning=0, + #warpx_pml_ncell=10 +) + +# Diagnostics +particle_diag = picmi.ParticleDiagnostic( + name='Python_LaserIonAcc2d_plt', + period=100, + write_dir='./diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5', + # demonstration of a spatial and momentum filter + warpx_plot_filter_function='(uz>=0) * (x<1.0e-6) * (x>-1.0e-6)' +) +# reduce resolution of output fields +coarsening_ratio = [4, 4] +ncell_field = [] +for (ncell_comp, cr) in zip([nx,nz], coarsening_ratio): + ncell_field.append(int(ncell_comp/cr)) +field_diag = picmi.FieldDiagnostic( + name='Python_LaserIonAcc2d_plt', + grid=grid, + period=100, + number_of_cells=ncell_field, + data_list=['B', 'E', 'J', 'rho', 'rho_electrons', 'rho_hydrogen'], + write_dir='./diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5' +) + +particle_fw_diag = picmi.ParticleDiagnostic( + name='openPMDfw', + period=100, + write_dir='./diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5', + warpx_plot_filter_function='(uz>=0) * (x<1.0e-6) * (x>-1.0e-6)' +) + +particle_bw_diag = picmi.ParticleDiagnostic( + name='openPMDbw', + period=100, + write_dir='./diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5', + warpx_plot_filter_function='(uz<0)' +) + +# histograms with 2.0 degree acceptance angle in fw direction +# 2 deg * pi / 180 : 0.03490658503 rad +# half-angle +/- : 0.017453292515 rad +histuH_rdiag = picmi.ReducedDiagnostic( + diag_type='ParticleHistogram', + name='histuH', + period=100, + species=hydrogen, + bin_number=1000, + bin_min=0.0, + bin_max=0.474, # 100 MeV protons + histogram_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)', + filter_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)') + +histue_rdiag = picmi.ReducedDiagnostic( + diag_type='ParticleHistogram', + name='histue', + period=100, + species=electrons, + bin_number=1000, + bin_min=0.0, + bin_max=197.0, # 100 MeV electrons + histogram_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)', + filter_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)') + +# just a test entry to make sure that the histogram filter is purely optional: +# this one just records uz of all hydrogen ions, independent of their pointing +histuzAll_rdiag = picmi.ReducedDiagnostic( + diag_type='ParticleHistogram', + name='histuzAll', + period=100, + species=hydrogen, + bin_number=1000, + bin_min=-0.474, + bin_max=0.474, + histogram_function='uz') + +field_probe_z_rdiag = picmi.ReducedDiagnostic( + diag_type='FieldProbe', + name='FieldProbe_Z', + period=100, + integrate=0, + probe_geometry='Line', + x_probe=0.0, + z_probe=-5.0e-6, + x1_probe=0.0, + z1_probe=25.0e-6, + resolution=3712) + +field_probe_scat_point_rdiag = picmi.ReducedDiagnostic( + diag_type='FieldProbe', + name='FieldProbe_ScatPoint', + period=1, + integrate=0, + probe_geometry='Point', + x_probe=0.0, + z_probe=15.0e-6) + +field_probe_scat_line_rdiag = picmi.ReducedDiagnostic( + diag_type='FieldProbe', + name='FieldProbe_ScatLine', + period=100, + integrate=1, + probe_geometry='Line', + x_probe=-2.5e-6, + z_probe=15.0e-6, + x1_probe=2.5e-6, + z1_probe=15e-6, + resolution=201) + +load_balance_costs_rdiag = picmi.ReducedDiagnostic( + diag_type='LoadBalanceCosts', + name='LBC', + period=100) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, + max_time=stop_time, # need to remove `max_step` to run this far + verbose=1, + particle_shape='cubic', + warpx_numprocs=[1, 2], # deactivate `numprocs` for dynamic load balancing + warpx_use_filter=1, + warpx_load_balance_intervals=100, + warpx_load_balance_costs_update='heuristic' +) + +# Add plasma electrons +sim.add_species( + electrons, + layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[2,2]) + # for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c + #layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[4,8]) +) + +# Add hydrogen ions +sim.add_species( + hydrogen, + layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[2,2]) + # for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c + #layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[4,8]) +) + +# Add laser +sim.add_laser( + laser, + injection_method=laser_antenna) + +# Add full diagnostics +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) +sim.add_diagnostic(particle_fw_diag) +sim.add_diagnostic(particle_bw_diag) +# Add reduced diagnostics +sim.add_diagnostic(histuH_rdiag) +sim.add_diagnostic(histue_rdiag) +sim.add_diagnostic(histuzAll_rdiag) +sim.add_diagnostic(field_probe_z_rdiag) +sim.add_diagnostic(field_probe_scat_point_rdiag) +sim.add_diagnostic(field_probe_scat_line_rdiag) +sim.add_diagnostic(load_balance_costs_rdiag) +# TODO: make ParticleHistogram2D available + +# Write input file that can be used to run with the compiled version +sim.write_input_file(file_name='inputs_2d_picmi') + +# Initialize inputs and WarpX instance +sim.initialize_inputs() +sim.initialize_warpx() + +# Advance simulation until last time step +sim.step(max_step) diff --git a/Examples/Physics_applications/laser_ion/README.rst b/Examples/Physics_applications/laser_ion/README.rst index f382b36bc85..18a427aa1c7 100644 --- a/Examples/Physics_applications/laser_ion/README.rst +++ b/Examples/Physics_applications/laser_ion/README.rst @@ -4,18 +4,15 @@ Laser-Ion Acceleration with a Planar Target =========================================== This example shows how to model laser-ion acceleration with planar targets of solid density :cite:p:`ex-Wilks2001,ex-Bulanov2008,ex-Macchi2013`. -The acceleration mechanism in this scenario depends on target parameters +The acceleration mechanism in this scenario depends on target parameters. Although laser-ion acceleration requires full 3D modeling for adequate description of the acceleration dynamics, especially the acceleration field lengths and decay times, this example models a 2D example. 2D modeling can often hint at a qualitative overview of the dynamics, but mostly saves computational costs since the plasma frequency (and Debye length) of the plasma determines the resolution need in laser-solid interaction modeling. -.. note:: - - TODO: The Python (PICMI) input file needs to be created. - .. note:: The resolution of this 2D case is extremely low by default. + This includes spatial and temporal resolution, but also the number of macro-particles per cell representing the target density for proper phase space sampling. You will need a computing cluster for adequate resolution of the target density, see comments in the input file. .. warning:: @@ -30,18 +27,19 @@ Run This example can be run **either** as: -* **Python** script: (*TODO*) or -* WarpX **executable** using an input file: ``warpx.2d inputs_2d`` +* **Python** script: ``mpiexec -n 2 python3 PICMI_inputs_2d.py`` or +* WarpX **executable** using an input file: ``mpiexec -n 2 warpx.2d inputs_2d`` -For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. +For `MPI-parallel `__ runs on computing clusters, change the prefix to ``mpiexec -n ...`` or ``srun -n ...``, depending on the system and number of MPI ranks you want to allocate. .. tab-set:: .. tab-item:: Python: Script - .. note:: + .. literalinclude:: PICMI_inputs_2d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py``. - TODO: This input file should be created following the ``inputs_2d`` file. .. tab-item:: Executable: Input File @@ -52,14 +50,61 @@ For `MPI-parallel `__ runs, prefix these lines with ` Analyze ------- -.. note:: +.. _fig-tnsa-ps-electrons-pinhole: + +.. figure:: https://user-images.githubusercontent.com/5416860/295003882-c755fd47-4bb3-4439-9319-c48214cbaafd.png + :alt: Longitudinal phase space of forward-flying electrons in a 2 degree opening angle. + :width: 100% + + Longitudinal phase space of forward-flying electrons in a 2 degree opening angle. + +.. _fig-tnsa-ps-protons-pinhole: + +.. figure:: https://user-images.githubusercontent.com/5416860/295003988-dea3dfb7-0d55-4616-b32d-061fb429f9ac.png + :alt: Longitudinal phase space of forward-flying protons in a 2 degree opening angle. + :width: 100% + + Longitudinal phase space of forward-flying protons in a 2 degree opening angle. + +Time-resolved phase electron space analysis as in :numref:`fig-tnsa-ps-electrons-pinhole` gives information about, e.g., how laser energy is locally converted into electron kinetic energy. +Later in time, ion phase spaces like :numref:`fig-tnsa-ps-protons-pinhole` can reveal where accelerated ion populations originate. - This section is TODO. +.. dropdown:: Script ``analysis_histogram_2D.py`` + .. literalinclude:: analysis_histogram_2D.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/analysis_histogram_2D.py``. Visualize --------- .. note:: - This section is TODO. + The following images for densities and electromagnetic fields were created with a run on 64 NVidia A100 GPUs featuring a total number of cells of ``nx = 8192`` and ``nz = 16384``, as well as 64 particles per cell per species. + +.. _fig-tnsa-densities: + +.. figure:: https://user-images.githubusercontent.com/5416860/296338802-8059c39c-0be8-4e4d-b41b-f976b626bd7f.png + :alt: Particle densities for electrons (top), protons (middle), and electrons again in logarithmic scale (bottom). + :width: 80% + + Particle densities for electrons (top), protons (middle), and electrons again in logarithmic scale (bottom). + +Particle density output illustrates the evolution of the target in time and space. +Logarithmic scales can help to identify where the target becomes transparent for the laser pulse (bottom panel in :numref:`fig-tnsa-densities` ). + +.. _fig-tnsa-fields: + +.. figure:: https://user-images.githubusercontent.com/5416860/296338609-a49eee7f-6793-4b55-92f1-0b887e6437ab.png + :alt: Electromagnetic field visualization for E_x (top), B_y (middle), and E_z (bottom). + :width: 80% + + Electromagnetic field visualization for :math:`E_x` (top), :math:`B_y` (middle), and :math:`E_z` (bottom). + +Electromagnetic field output shows where the laser field is strongest at a given point in time, and where accelerating fields build up :numref:`fig-tnsa-fields`. + +.. dropdown:: Script ``plot_2d.py`` + + .. literalinclude:: plot_2d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/plot_2d.py``. diff --git a/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py b/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py index 175d8eed568..a262a2373e5 100644 --- a/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py +++ b/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py @@ -6,6 +6,7 @@ import matplotlib.colors as colors import matplotlib.pyplot as plt +import numpy as np from openpmd_viewer import OpenPMDTimeSeries parser = argparse.ArgumentParser(description='Process a 2D histogram name and an integer.') @@ -18,30 +19,43 @@ ts = OpenPMDTimeSeries(path) it = ts.iterations -data, info = ts.get_field(field = "data", iteration = 0, plot = True) +data, info = ts.get_field(field="data", iteration=0, plot=True) print('The available iterations of the simulation are:', it) -print('The axis of the histogram are (0: ordinate ; 1: abscissa):', info.axes) +print('The axes of the histogram are (0: ordinate ; 1: abscissa):', info.axes) print('The data shape is:', data.shape) # Add the simulation time to the title once this information # is available in the "info" FieldMetaInformation object. if args.iter == 'All' : - for i in it : + for it_idx, i in enumerate(it): plt.figure() - data, info = ts.get_field(field = "data", iteration = i, plot = False) - plt.imshow(data, aspect="auto", norm=colors.LogNorm(), extent=info.imshow_extent) - plt.title(args.hist2D + " (iteration %d)" % i) - plt.xlabel(info.axes[1] + ' (m)') - plt.ylabel(info.axes[0] + ' (m.s-1)') + data, info = ts.get_field(field="data", iteration=i, plot=False) + abscissa_name = info.axes[1] # This might be 'z' or something else + abscissa_values = getattr(info, abscissa_name, None) + ordinate_name = info.axes[0] # This might be 'z' or something else + ordinate_values = getattr(info, ordinate_name, None) + + plt.pcolormesh(abscissa_values/1e-6, ordinate_values, data, norm=colors.LogNorm(), rasterized=True) + plt.title(args.hist2D + f" Time: {ts.t[it_idx]:.2e} s (Iteration: {i:d})") + plt.xlabel(info.axes[1]+r' ($\mu$m)') + plt.ylabel(info.axes[0]+r' ($m_\mathrm{species} c$)') plt.colorbar() - plt.savefig('Histogram_2D_' + args.hist2D + '_iteration_' + str(i) + '.pdf') + plt.tight_layout() + plt.savefig('Histogram_2D_' + args.hist2D + '_iteration_' + str(i) + '.png') else : i = int(args.iter) + it_idx = np.where(i == it)[0][0] plt.figure() - data, info = ts.get_field(field = "data", iteration = i, plot = False) - plt.imshow(data, aspect="auto", norm=colors.LogNorm(), extent=info.imshow_extent) - plt.title(args.hist2D + " (iteration %d)" % i) - plt.xlabel(info.axes[1] + ' (m)') - plt.ylabel(info.axes[0] + ' (m.s-1)') + data, info = ts.get_field(field="data", iteration=i, plot=False) + abscissa_name = info.axes[1] # This might be 'z' or something else + abscissa_values = getattr(info, abscissa_name, None) + ordinate_name = info.axes[0] # This might be 'z' or something else + ordinate_values = getattr(info, ordinate_name, None) + + plt.pcolormesh(abscissa_values/1e-6, ordinate_values, data, norm=colors.LogNorm(), rasterized=True) + plt.title(args.hist2D + f" Time: {ts.t[it_idx]:.2e} s (Iteration: {i:d})") + plt.xlabel(info.axes[1]+r' ($\mu$m)') + plt.ylabel(info.axes[0]+r' ($m_\mathrm{species} c$)') plt.colorbar() - plt.savefig('Histogram_2D_' + args.hist2D + '_iteration_' + str(i) + '.pdf') + plt.tight_layout() + plt.savefig('Histogram_2D_' + args.hist2D + '_iteration_' + str(i) + '.png') diff --git a/Examples/Physics_applications/laser_ion/inputs_2d b/Examples/Physics_applications/laser_ion/inputs_2d index 8fe5acf5f46..5ad8334e9ef 100644 --- a/Examples/Physics_applications/laser_ion/inputs_2d +++ b/Examples/Physics_applications/laser_ion/inputs_2d @@ -2,6 +2,9 @@ # Domain, Resolution & Numerics # +# We only run 100 steps for tests +# Disable `max_step` below to run until the physical `stop_time`. +max_step = 100 # time-scale with highly kinetic dynamics stop_time = 0.2e-12 # [s] # time-scale for converged ion energy @@ -9,12 +12,12 @@ stop_time = 0.2e-12 # [s] # - ions will start to leave the box #stop_time = 1.0e-12 # [s] -# quick tests at ultra-low res. (CI) -#amr.n_cell = 384 512 +# quick tests at ultra-low res. (for CI, and local computer) +amr.n_cell = 384 512 # proper resolution for 10 n_c excl. acc. length # (>=1x V100) -amr.n_cell = 2688 3712 +#amr.n_cell = 2688 3712 # proper resolution for 30 n_c (dx<=3.33nm) incl. acc. length # (>=6x V100) @@ -94,6 +97,8 @@ particles.species_names = electrons hydrogen hydrogen.species_type = hydrogen hydrogen.injection_style = NUniformPerCell hydrogen.num_particles_per_cell_each_dim = 2 2 +# for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c +#hydrogen.num_particles_per_cell_each_dim = 4 8 hydrogen.momentum_distribution_type = at_rest # minimum and maximum z position between which particles are initialized # --> should be set for dense targets limit memory consumption during initialization @@ -107,6 +112,8 @@ hydrogen.attribute.orig_z(x,y,z,ux,uy,uz,t) = "z" electrons.species_type = electron electrons.injection_style = NUniformPerCell electrons.num_particles_per_cell_each_dim = 2 2 +# for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c +#electrons.num_particles_per_cell_each_dim = 4 8 electrons.momentum_distribution_type = "gaussian" electrons.ux_th = .01 electrons.uz_th = .01 @@ -200,22 +207,19 @@ diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen # reduce resolution of output fields diag1.coarsening_ratio = 4 4 -diag1.electrons.variables = w ux uy uz -diag1.hydrogen.variables = w ux uz orig_x orig_z # demonstration of a spatial and momentum filter diag1.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) diag1.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) +diag1.format = openpmd +diag1.openpmd_backend = h5 openPMDfw.intervals = 100 openPMDfw.diag_type = Full openPMDfw.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen # reduce resolution of output fields openPMDfw.coarsening_ratio = 4 4 -openPMDfw.electrons.variables = w ux uy uz -openPMDfw.hydrogen.variables = w ux uy uz orig_x orig_z openPMDfw.format = openpmd openPMDfw.openpmd_backend = h5 -openPMDfw.species = electrons hydrogen # demonstration of a spatial and momentum filter openPMDfw.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) openPMDfw.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) @@ -225,11 +229,8 @@ openPMDbw.diag_type = Full openPMDbw.fields_to_plot = rho_hydrogen # reduce resolution of output fields openPMDbw.coarsening_ratio = 4 4 -openPMDbw.electrons.variables = w ux uy uz -openPMDbw.hydrogen.variables = w ux uy uz orig_x orig_z openPMDbw.format = openpmd openPMDbw.openpmd_backend = h5 -openPMDbw.species = electrons hydrogen # demonstration of a momentum filter openPMDbw.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz<0) openPMDbw.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz<0) @@ -246,21 +247,19 @@ warpx.reduced_diags_names = histuH histue histuzAll FieldProbe histuH.type = ParticleHistogram histuH.intervals = 100 -histuH.path = "./" histuH.species = hydrogen histuH.bin_number = 1000 histuH.bin_min = 0.0 -histuH.bin_max = 35.0 +histuH.bin_max = 0.474 # 100 MeV protons histuH.histogram_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)" histuH.filter_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)" histue.type = ParticleHistogram histue.intervals = 100 -histue.path = "./" histue.species = electrons histue.bin_number = 1000 histue.bin_min = 0.0 -histue.bin_max = 0.1 +histue.bin_max = 197 # 100 MeV electrons histue.histogram_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)" histue.filter_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)" @@ -268,11 +267,10 @@ histue.filter_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(a # this one just records uz of all hydrogen ions, independent of their pointing histuzAll.type = ParticleHistogram histuzAll.intervals = 100 -histuzAll.path = "./" histuzAll.species = hydrogen histuzAll.bin_number = 1000 -histuzAll.bin_min = -35.0 -histuzAll.bin_max = 35.0 +histuzAll.bin_min = -0.474 +histuzAll.bin_max = 0.474 histuzAll.histogram_function(t,x,y,z,ux,uy,uz) = "uz" FieldProbe_Z.type = FieldProbe @@ -313,8 +311,8 @@ PhaseSpaceIons.bin_number_abs = 1000 PhaseSpaceIons.bin_number_ord = 1000 PhaseSpaceIons.bin_min_abs = -5.e-6 PhaseSpaceIons.bin_max_abs = 25.e-6 -PhaseSpaceIons.bin_min_ord = -20.e-6 -PhaseSpaceIons.bin_max_ord = 20.e-6 +PhaseSpaceIons.bin_min_ord = -0.474 +PhaseSpaceIons.bin_max_ord = 0.474 PhaseSpaceIons.histogram_function_abs(t,x,y,z,ux,uy,uz,w) = "z" PhaseSpaceIons.histogram_function_ord(t,x,y,z,ux,uy,uz,w) = "uz" PhaseSpaceIons.value_function(t,x,y,z,ux,uy,uz,w) = "w" @@ -327,8 +325,8 @@ PhaseSpaceElectrons.bin_number_abs = 1000 PhaseSpaceElectrons.bin_number_ord = 1000 PhaseSpaceElectrons.bin_min_abs = -5.e-6 PhaseSpaceElectrons.bin_max_abs = 25.e-6 -PhaseSpaceElectrons.bin_min_ord = -20 -PhaseSpaceElectrons.bin_max_ord = 20 +PhaseSpaceElectrons.bin_min_ord = -197 +PhaseSpaceElectrons.bin_max_ord = 197 PhaseSpaceElectrons.histogram_function_abs(t,x,y,z,ux,uy,uz,w) = "z" PhaseSpaceElectrons.histogram_function_ord(t,x,y,z,ux,uy,uz,w) = "uz" PhaseSpaceElectrons.value_function(t,x,y,z,ux,uy,uz,w) = "w" diff --git a/Examples/Physics_applications/laser_ion/plot_2d.py b/Examples/Physics_applications/laser_ion/plot_2d.py new file mode 100644 index 00000000000..e85782a7d23 --- /dev/null +++ b/Examples/Physics_applications/laser_ion/plot_2d.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 + +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Marco Garten +# License: BSD-3-Clause-LBNL +# +# This script plots the densities and fields of a 2D laser-ion acceleration simulation. + + +import argparse +import os +import re + +from matplotlib.colors import TwoSlopeNorm +import matplotlib.pyplot as plt +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +import pandas as pd +import scipy.constants as sc + +plt.rcParams.update({'font.size':16}) + +def create_analysis_dir(directory): + if not os.path.exists(directory): + os.makedirs(directory) + + +def visualize_density_iteration(ts, iteration, out_dir): + """ + Visualize densities and fields of a single iteration. + + :param ts: OpenPMDTimeSeries + :param iteration: Output iteration (simulation timestep) + :param out_dir: Directory for PNG output + :return: + """ + # Physics parameters + lambda_L = 800e-9 # Laser wavelength in meters + omega_L = 2 * np.pi * sc.c / lambda_L # Laser frequency in seconds + n_c = sc.m_e * sc.epsilon_0 * omega_L**2 / sc.elementary_charge**2 # Critical plasma density in meters^(-3) + micron = 1e-6 + + # Simulation parameters + n_e0 = 30 + n_max = 2 * n_e0 + nr = 1 # Number to decrease resolution + + # Data fetching + it = iteration + ii = np.where(ts.iterations == it)[0][0] + + time = ts.t[ii] + rho_e, rho_e_info = ts.get_field(field="rho_electrons", iteration=it) + rho_d, rho_d_info = ts.get_field(field="rho_hydrogen", iteration=it) + + # Rescale to critical density + rho_e = rho_e / (sc.elementary_charge * n_c) + rho_d = rho_d / (sc.elementary_charge * n_c) + + # Axes setup + fig, axs = plt.subplots(3, 1, figsize=(5, 8)) + xax, zax = rho_e_info.x, rho_e_info.z + + # Plotting + # Electron density + im0 = axs[0].pcolormesh(zax[::nr]/micron, xax[::nr]/micron, -rho_e.T[::nr, ::nr], + vmin=0, vmax=n_max, cmap="Reds", rasterized=True) + plt.colorbar(im0, ax=axs[0], label=r"$n_\mathrm{\,e}\ (n_\mathrm{c})$") + + # Hydrogen density + im1 = axs[1].pcolormesh(zax[::nr]/micron, xax[::nr]/micron, rho_d.T[::nr, ::nr], + vmin=0, vmax=n_max, cmap="Blues", rasterized=True) + plt.colorbar(im1, ax=axs[1], label=r"$n_\mathrm{\,H}\ (n_\mathrm{c})$") + + # Masked electron density + divnorm = TwoSlopeNorm(vmin=-7., vcenter=0., vmax=2) + masked_data = np.ma.masked_where(rho_e.T == 0, rho_e.T) + my_cmap = plt.cm.PiYG_r.copy() + my_cmap.set_bad(color='black') + im2 = axs[2].pcolormesh(zax[::nr]/micron, xax[::nr]/micron, np.log(-masked_data[::nr, ::nr]), + norm=divnorm, cmap=my_cmap, rasterized=True) + plt.colorbar(im2, ax=axs[2], ticks=[-6, -3, 0, 1, 2], extend='both', + label=r"$\log n_\mathrm{\,e}\ (n_\mathrm{c})$") + + # Axis labels and title + for ax in axs: + ax.set_aspect(1.0) + ax.set_ylabel(r"$x$ ($\mu$m)") + for ax in axs[:-1]: + ax.set_xticklabels([]) + axs[2].set_xlabel(r"$z$ ($\mu$m)") + fig.suptitle(f"Iteration: {it}, Time: {time/1e-15:.1f} fs") + + plt.tight_layout() + + plt.savefig(f"{out_dir}/densities_{it:06d}.png") + +def visualize_field_iteration(ts, iteration, out_dir): + + # Additional parameters + nr = 1 # Number to decrease resolution + micron = 1e-6 + + # Data fetching + it = iteration + ii = np.where(ts.iterations == it)[0][0] + time = ts.t[ii] + + Ex, Ex_info = ts.get_field(field="E", coord="x", iteration=it) + Exmax = np.max(np.abs([np.min(Ex),np.max(Ex)])) + By, By_info = ts.get_field(field="B", coord="y", iteration=it) + Bymax = np.max(np.abs([np.min(By),np.max(By)])) + Ez, Ez_info = ts.get_field(field="E", coord="z", iteration=it) + Ezmax = np.max(np.abs([np.min(Ez),np.max(Ez)])) + + # Axes setup + fig,axs = plt.subplots(3, 1, figsize=(5, 8)) + xax, zax = Ex_info.x, Ex_info.z + + # Plotting + im0 = axs[0].pcolormesh( + zax[::nr]/micron,xax[::nr]/micron,Ex.T[::nr,::nr], + vmin=-Exmax, vmax=Exmax, + cmap="RdBu", rasterized=True) + + plt.colorbar(im0,ax=axs[00], label=r"$E_x$ (V/m)") + + im1 = axs[1].pcolormesh( + zax[::nr]/micron,xax[::nr]/micron,By.T[::nr,::nr], + vmin=-Bymax, vmax=Bymax, + cmap="RdBu", rasterized=True) + plt.colorbar(im1,ax=axs[1], label=r"$B_y$ (T)") + + im2 = axs[2].pcolormesh( + zax[::nr]/micron,xax[::nr]/micron,Ez.T[::nr,::nr], + vmin=-Ezmax, vmax=Ezmax, + cmap="RdBu", rasterized=True) + plt.colorbar(im2,ax=axs[2],label=r"$E_z$ (V/m)") + + # Axis labels and title + for ax in axs: + ax.set_aspect(1.0) + ax.set_ylabel(r"$x$ ($\mu$m)") + for ax in axs[:-1]: + ax.set_xticklabels([]) + axs[2].set_xlabel(r"$z$ ($\mu$m)") + fig.suptitle(f"Iteration: {it}, Time: {time/1e-15:.1f} fs") + + plt.tight_layout() + + plt.savefig(f"{out_dir}/fields_{it:06d}.png") + +def visualize_particle_histogram_iteration(diag_name="histuH", species="hydrogen", iteration=1000, out_dir="./analysis"): + + it = iteration + + if species == "hydrogen": + # proton rest energy in eV + mc2 = sc.m_p/sc.electron_volt * sc.c**2 + elif species == "electron": + mc2 = sc.m_e/sc.electron_volt * sc.c**2 + else: + raise NotImplementedError("The only implemented presets for this analysis script are `electron` or `hydrogen`.") + + fs = 1.e-15 + MeV = 1.e6 + + df = pd.read_csv(f"./diags/reducedfiles/{diag_name}.txt",delimiter=r'\s+') + # the columns look like this: + # #[0]step() [1]time(s) [2]bin1=0.000220() [3]bin2=0.000660() [4]bin3=0.001100() + + # matches words, strings surrounded by " ' ", dots, minus signs and e for scientific notation in numbers + nested_list = [re.findall(r"[\w'\.]+",col) for col in df.columns] + + index = pd.MultiIndex.from_tuples(nested_list, names=('column#', 'name', 'bin value')) + + df.columns = (index) + + steps = df.values[:, 0].astype(int) + ii = np.where(steps == it)[0][0] + time = df.values[:, 1] + data = df.values[:, 2:] + edge_vals = np.array([float(row[2]) for row in df.columns[2:]]) + edges_MeV = (np.sqrt(edge_vals**2 + 1)-1) * mc2 / MeV + + time_fs = time / fs + + fig,ax = plt.subplots(1,1) + + ax.plot(edges_MeV, data[ii, :]) + ax.set_yscale("log") + ax.set_ylabel(r"d$N$/d$\mathcal{E}$ (arb. u.)") + ax.set_xlabel(r"$\mathcal{E}$ (MeV)") + + fig.suptitle(f"{species} - Iteration: {it}, Time: {time_fs[ii]:.1f} fs") + + plt.tight_layout() + plt.savefig(f"./{out_dir}/{diag_name}_{it:06d}.png") + + +if __name__ == "__main__": + + # Argument parsing + parser = argparse.ArgumentParser(description='Visualize Laser-Ion Accelerator Densities and Fields') + parser.add_argument('-d', '--diag_dir', type=str, default='./diags/diag1', help='Directory containing density and field diagnostics') + parser.add_argument('-i', '--iteration', type=int, default=None, help='Specific iteration to visualize') + parser.add_argument('-hn', '--histogram_name', type=str, default='histuH', help='Name of histogram diagnostic to visualize') + parser.add_argument('-hs', '--histogram_species', type=str, default='hydrogen', help='Particle species in the visualized histogram diagnostic') + args = parser.parse_args() + + # Create analysis directory + analysis_dir = 'analysis' + create_analysis_dir(analysis_dir) + + # Loading the time series + ts = OpenPMDTimeSeries(args.diag_dir) + + if args.iteration is not None: + visualize_density_iteration(ts, args.iteration, analysis_dir) + visualize_field_iteration(ts, args.iteration, analysis_dir) + visualize_particle_histogram_iteration(args.histogram_name, args.histogram_species, args.iteration, analysis_dir) + else: + for it in ts.iterations: + visualize_density_iteration(ts, it, analysis_dir) + visualize_field_iteration(ts, it, analysis_dir) + visualize_particle_histogram_iteration(args.histogram_name, args.histogram_species, it, analysis_dir) diff --git a/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json index 1678e794683..81850192121 100644 --- a/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json +++ b/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json @@ -1,33 +1,36 @@ { - "lev=0": { - "Bx": 0.0, - "By": 11411047.898351602, - "Bz": 0.0, - "Ex": 2031641076084948.5, - "Ey": 0.0, - "Ez": 334777106874158.8, - "jx": 1.6602050255725978e+19, - "jy": 0.0, - "jz": 9.545451466990307e+18, - "rho": 67237998613.86053, - "rho_electrons": 17449705826002.385, - "rho_hydrogen": 17441792654743.146 - }, - "hydrogen": { - "particle_momentum_x": 2.2575255298569512e-18, - "particle_momentum_z": 1.0773391029868316e-18, - "particle_orig_x": 0.007763427734375002, - "particle_orig_z": 0.0351975439453125, - "particle_position_x": 0.007763034484095496, - "particle_position_y": 0.035195761508116416, - "particle_weight": 2.6792730619156992e+17 - }, "electrons": { "particle_momentum_x": 3.8463518636930147e-19, "particle_momentum_y": 0.0, "particle_momentum_z": 1.6287398567676136e-18, "particle_position_x": 0.008139313907538123, - "particle_position_y": 0.0308605164193468, + "particle_position_y": 0.0, + "particle_position_z": 0.0308605164193468, "particle_weight": 2.647983436428149e+17 + }, + "hydrogen": { + "particle_momentum_x": 2.2575255298569516e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.0773391029868316e-18, + "particle_origX": 0.007763427734375002, + "particle_origZ": 0.0351975439453125, + "particle_position_x": 0.007763034484095496, + "particle_position_y": 0.0, + "particle_position_z": 0.035195761508116416, + "particle_weight": 2.6792730619156992e+17 + }, + "lev=0": { + "Bx": 0.0, + "By": 11411047.898351604, + "Bz": 0.0, + "Ex": 2031641076084948.8, + "Ey": 0.0, + "Ez": 334777106874158.75, + "jx": 1.660205025572598e+19, + "jy": 0.0, + "jz": 9.545451466990307e+18, + "rho": 67237998613.860535, + "rho_electrons": 17449705826002.387, + "rho_hydrogen": 17441792654743.145 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json new file mode 100644 index 00000000000..baaf29bec59 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json @@ -0,0 +1,34 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 11406516.737324182, + "Bz": 0.0, + "Ex": 2032785026975781.0, + "Ey": 0.0, + "Ez": 315269171271122.75, + "jx": 1.6224680921022521e+19, + "jy": 0.0, + "jz": 8.875043590548824e+18, + "rho": 61329928310.45101, + "rho_electrons": 17450329551552.684, + "rho_hydrogen": 17441799909767.852 + }, + "electrons": { + "particle_position_x": 0.008152777855374778, + "particle_position_y": 0.0, + "particle_position_z": 0.030714485915215906, + "particle_momentum_x": 3.594599848069649e-19, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.6339243089017708e-18, + "particle_weight": 2.6507336926909222e+17 + }, + "hydrogen": { + "particle_position_x": 0.008197892199782453, + "particle_position_y": 0.0, + "particle_position_z": 0.0365646600930625, + "particle_momentum_x": 2.2464037026384752e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.0873094324185116e-18, + "particle_weight": 2.703612070965676e+17 + } +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index c9feea370af..4d1435e61de 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -2184,7 +2184,8 @@ doVis = 0 [LaserIonAcc2d] buildDir = . inputFile = Examples/Physics_applications/laser_ion/inputs_2d -runtime_params = amr.n_cell=384 512 max_step=100 +outputFile = LaserIonAcc2d_plt +runtime_params = dim = 2 addToCompileString = USE_OPENPMD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON @@ -2195,7 +2196,7 @@ useOMP = 1 numthreads = 1 compileTest = 0 doVis = 0 -analysisRoutine = Examples/analysis_default_regression.py +analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserOnFine] buildDir = . @@ -3393,6 +3394,25 @@ compareParticles = 1 particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py +[Python_LaserIonAcc2d] +buildDir = . +inputFile = Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py +outputFile= diags/Python_LaserIonAcc2d_plt +runtime_params = +customRunCmd = python3 PICMI_inputs_2d.py +dim = 2 +addToCompileString = USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +analysisRoutine = Examples/analysis_default_openpmd_regression.py + [Python_LoadExternalField3D] buildDir = . inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d.py From 9a017a67e5495263223da42db47657693b25bbd2 Mon Sep 17 00:00:00 2001 From: Eya D <81635404+EyaDammak@users.noreply.github.com> Date: Fri, 23 Feb 2024 21:55:45 -0800 Subject: [PATCH 170/176] Update of the time saved for scraped particles on the boundaries (#4710) * modifications of the names and change of timeScraped into delta * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update ParticleBoundaryBuffer.cpp * Update analysis.py * updates * Update parameters.rst * Update parameters.rst * Update parameters.rst * Update parameters.rst * Update parameters.rst * Update parameters.rst * Update parameters.rst * Update analysis.py * Update parameters.rst * Update particle_containers.py * Update ParticleBoundaryBuffer.cpp * Update analysis.py * Update ParticleBoundaryBuffer.cpp * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * update name * Update Source/Particles/ParticleBoundaryBuffer.cpp Co-authored-by: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> * Update Source/Particles/ParticleBoundaryBuffer.cpp Co-authored-by: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp * Update Docs/source/usage/parameters.rst Co-authored-by: Remi Lehe * Update analysis.py * Update analysis.py * Update particle_containers.py * Update particle_containers.py * Update analysis.py * Update analysis.py --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Co-authored-by: Remi Lehe --- Docs/source/usage/parameters.rst | 5 +-- .../PICMI_inputs_reflection.py | 4 +-- .../PICMI_inputs_scrape.py | 2 +- .../Tests/point_of_contact_EB/analysis.py | 20 ++++++------ Python/pywarpx/particle_containers.py | 2 +- Source/Particles/ParticleBoundaryBuffer.cpp | 31 ++++++++++--------- 6 files changed, 34 insertions(+), 30 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index f7462351800..0c5fe85973f 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2757,8 +2757,9 @@ By default, all of the collected particle data is written out at the end of the This can be important if a large number of particles are lost, avoiding filling up memory with the accumulated lost particle data. In addition to their usual attributes, the saved particles have - an integer attribute ``step_scraped``, which indicates the PIC iteration at which each particle was absorbed at the boundary, - a real attribute ``time_scraped``, which indicates the exact time calculated when each particle hit the boundary, + an integer attribute ``stepScraped``, which indicates the PIC iteration at which each particle was absorbed at the boundary, + a real attribute ``deltaTimeScraped``, which indicates the time between the time associated to `stepScraped` + and the exact time when each particle hits the boundary. 3 real attributes ``nx``, ``ny``, ``nz``, which represents the three components of the normal to the boundary on the point of contact of the particles (not saved if they reach non-EB boundaries) ``BoundaryScrapingDiagnostics`` can be used with ``..random_fraction``, ``..uniform_stride``, and ``..plot_filter_function``, which have the same behavior as for ``FullDiagnostics``. For ``BoundaryScrapingDiagnostics``, these filters are applied at the time the data is written to file. An implication of this is that more particles may initially be accumulated in memory than are ultimately written. ``t`` in ``plot_filter_function`` refers to the time the diagnostic is written rather than the time the particle crossed the boundary. diff --git a/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py b/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py index 9ac40818e12..bb1ebd73082 100755 --- a/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py +++ b/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py @@ -131,12 +131,12 @@ print("Number of electrons in lower buffer:", n) assert n == 67 -scraped_steps = buffer.get_particle_boundary_buffer("electrons", 'z_hi', 'step_scraped', 0) +scraped_steps = buffer.get_particle_boundary_buffer("electrons", 'z_hi', 'stepScraped', 0) for arr in scraped_steps: # print(arr) assert all(arr == 4) -scraped_steps = buffer.get_particle_boundary_buffer("electrons", 'z_lo', 'step_scraped', 0) +scraped_steps = buffer.get_particle_boundary_buffer("electrons", 'z_lo', 'stepScraped', 0) for arr in scraped_steps: # print(arr) assert all(arr == 8) diff --git a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py index 9871bdac655..e5a9a58f597 100755 --- a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py +++ b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py @@ -133,7 +133,7 @@ print(f"Number of electrons in buffer (proc #{my_id}): {n}") assert n == 612 -scraped_steps = particle_buffer.get_particle_boundary_buffer("electrons", 'eb', 'step_scraped', 0) +scraped_steps = particle_buffer.get_particle_boundary_buffer("electrons", 'eb', 'stepScraped', 0) for arr in scraped_steps: assert all(np.array(arr, copy=False) > 40) diff --git a/Examples/Tests/point_of_contact_EB/analysis.py b/Examples/Tests/point_of_contact_EB/analysis.py index 9bf7ce1aae5..cb1fc23ee62 100755 --- a/Examples/Tests/point_of_contact_EB/analysis.py +++ b/Examples/Tests/point_of_contact_EB/analysis.py @@ -26,8 +26,8 @@ ts_scraping = OpenPMDTimeSeries('./diags/diag2/particles_at_eb/') it=ts_scraping.iterations -step_scraped, time_scraped, x, y, z, nx, ny, nz=ts_scraping.get_particle( ['stepScraped','timeScraped','x','y','z', 'nx', 'ny', 'nz'], species='electron', iteration=it ) -time_scraped_reduced=time_scraped[0]*1e10 +step_scraped, delta, x, y, z, nx, ny, nz=ts_scraping.get_particle( ['stepScraped','deltaTimeScraped','x','y','z', 'nx', 'ny', 'nz'], species='electron', iteration=it ) +delta_reduced=delta[0]*1e10 # Analytical results calculated x_analytic=-0.1983 @@ -38,24 +38,24 @@ nz_analytic=0.0 #result obtained by analysis of simulations -step=3 -time_reduced=3.58 +step_ref=3 +delta_reduced_ref=0.59 print('NUMERICAL coordinates of the point of contact:') -print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f' % (step_scraped[0],time_reduced,x[0], y[0], z[0], nx[0], ny[0], nz[0])) +print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f' % (step_scraped[0],delta_reduced,x[0], y[0], z[0], nx[0], ny[0], nz[0])) print('\n') print('ANALYTICAL coordinates of the point of contact:') -print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f' % (step, time_reduced, x_analytic, y_analytic, z_analytic, nx_analytic, ny_analytic, nz_analytic)) +print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f' % (step_ref, delta_reduced_ref, x_analytic, y_analytic, z_analytic, nx_analytic, ny_analytic, nz_analytic)) tolerance=0.001 -tolerance_t=0.003 +tolerance_t=0.01 tolerance_n=0.01 print("tolerance = "+ str(tolerance *100) + '%') print("tolerance for the time = "+ str(tolerance_t *100) + '%') print("tolerance for the normal components = "+ str(tolerance_n *100) + '%') -diff_step=np.abs((step_scraped[0]-step)/step) -diff_time=np.abs((time_scraped_reduced-time_reduced)/time_reduced) +diff_step=np.abs((step_scraped[0]-step_ref)/step_ref) +diff_delta=np.abs((delta_reduced-delta_reduced_ref)/delta_reduced_ref) diff_x=np.abs((x[0]-x_analytic)/x_analytic) diff_y=np.abs((y[0]-y_analytic)/y_analytic) diff_nx=np.abs((nx[0]-nx_analytic)/nx_analytic) @@ -67,4 +67,4 @@ print("percentage error for ny = %5.2f %%" %(diff_ny *100)) print("nz = %5.2f " %(nz[0])) -assert (diff_x < tolerance) and (diff_y < tolerance) and (np.abs(z[0]) < 1e-8) and (diff_step < 1e-8) and (diff_time < tolerance_t) and (diff_nx < tolerance_n) and (diff_ny < tolerance_n) and (np.abs(nz) < 1e-8) , 'Test point_of_contact did not pass' +assert (diff_x < tolerance) and (diff_y < tolerance) and (np.abs(z[0]) < 1e-8) and (diff_step < 1e-8) and (diff_delta < tolerance_t) and (diff_nx < tolerance_n) and (diff_ny < tolerance_n) and (np.abs(nz) < 1e-8) , 'Test point_of_contact did not pass' diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index 9526e9322a2..19b0a1ab6c4 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -774,7 +774,7 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) comp_name : str The component of the array data that will be returned. "x", "y", "z", "ux", "uy", "uz", "w" - "step_scraped","time_scraped", + "stepScraped","deltaTimeScraped", if boundary='eb': "nx", "ny", "nz" level : int diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index f91f7a348e4..f991f211d28 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -22,6 +22,7 @@ #include #include #include +using namespace amrex::literals; struct IsOutsideDomainBoundary { amrex::GpuArray m_plo; @@ -47,7 +48,7 @@ struct IsOutsideDomainBoundary { #ifdef AMREX_USE_EB struct FindEmbeddedBoundaryIntersection { const int m_step_index; - const int m_time_index; + const int m_delta_index; const int m_normal_index; const int m_step; const amrex::Real m_dt; @@ -103,7 +104,7 @@ struct FindEmbeddedBoundaryIntersection { // Also record the real time on the destination dst.m_runtime_idata[m_step_index][dst_i] = m_step; - dst.m_runtime_rdata[m_time_index][dst_i] = m_step*m_dt + (1- dt_fraction)*m_dt; + dst.m_runtime_rdata[m_delta_index][dst_i] = (1._rt- dt_fraction)*m_dt; // Now that dt_fraction has be obtained (with bisect) // Save the corresponding position of the particle at the boundary @@ -162,7 +163,7 @@ struct FindEmbeddedBoundaryIntersection { struct CopyAndTimestamp { int m_step_index; - int m_time_index; + int m_delta_index; int m_step; const amrex::Real m_dt; @@ -182,7 +183,7 @@ struct CopyAndTimestamp { dst.m_runtime_idata[j][dst_i] = src.m_runtime_idata[j][src_i]; } dst.m_runtime_idata[m_step_index][dst_i] = m_step; - dst.m_runtime_rdata[m_time_index][dst_i] = m_step*m_dt; + dst.m_runtime_rdata[m_delta_index][dst_i] = 0._rt; //delta_fraction is initialized to zero } }; @@ -364,8 +365,8 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, if (!buffer[i].isDefined()) { buffer[i] = pc.make_alike(); - buffer[i].AddIntComp("step_scraped", false); - buffer[i].AddRealComp("time_scraped", false); + buffer[i].AddIntComp("stepScraped", false); + buffer[i].AddRealComp("deltaTimeScraped", false); } auto& species_buffer = buffer[i]; for (int lev = 0; lev < pc.numLevels(); ++lev) @@ -408,14 +409,14 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, auto& warpx = WarpX::GetInstance(); const auto dt = warpx.getdt(pti.GetLevel()); auto string_to_index_intcomp = buffer[i].getParticleRuntimeiComps(); - const int step_scraped_index = string_to_index_intcomp.at("step_scraped"); + const int step_scraped_index = string_to_index_intcomp.at("stepScraped"); auto string_to_index_realcomp = buffer[i].getParticleRuntimeComps(); - const int time_scraped_index = string_to_index_realcomp.at("time_scraped"); + const int delta_index = string_to_index_realcomp.at("deltaTimeScraped"); const int step = warpx_instance.getistep(0); amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, - CopyAndTimestamp{step_scraped_index,time_scraped_index, step,dt}, + CopyAndTimestamp{step_scraped_index, delta_index, step, dt}, 0, dst_index); } } @@ -435,11 +436,12 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, if (!buffer[i].isDefined()) { buffer[i] = pc.make_alike(); - buffer[i].AddIntComp("step_scraped", false); - buffer[i].AddRealComp("time_scraped", false); + buffer[i].AddIntComp("stepScraped", false); + buffer[i].AddRealComp("deltaTimeScraped", false); buffer[i].AddRealComp("nx", false); buffer[i].AddRealComp("ny", false); buffer[i].AddRealComp("nz", false); + } auto& species_buffer = buffer[i]; for (int lev = 0; lev < pc.numLevels(); ++lev) @@ -493,16 +495,17 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, auto& warpx = WarpX::GetInstance(); const auto dt = warpx.getdt(pti.GetLevel()); auto string_to_index_intcomp = buffer[i].getParticleRuntimeiComps(); - const int step_scraped_index = string_to_index_intcomp.at("step_scraped"); + const int step_scraped_index = string_to_index_intcomp.at("stepScraped"); auto string_to_index_realcomp = buffer[i].getParticleRuntimeComps(); - const int time_scraped_index = string_to_index_realcomp.at("time_scraped"); + const int delta_index = string_to_index_realcomp.at("deltaTimeScraped"); const int normal_index = string_to_index_realcomp.at("nx"); const int step = warpx_instance.getistep(0); { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterTransformEB"); amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, - FindEmbeddedBoundaryIntersection{step_scraped_index,time_scraped_index, normal_index, step, dt, phiarr, dxi, plo}, 0, dst_index); + FindEmbeddedBoundaryIntersection{step_scraped_index,delta_index, normal_index, step, dt, phiarr, dxi, plo}, 0, dst_index); + } } } From a0a6b72d2a48cb8d43791ac83b5f4571922e086e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 26 Feb 2024 16:09:09 -0800 Subject: [PATCH 171/176] AMReX: Weekly Update (#4728) --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- run_test.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index f16b405973e..368221efa9c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 2230caa24c7d4bd07edb08b54e9368f9c73eae6e && cd - + cd ../amrex && git checkout --detach 2ecafcff40132f56eb2b494e1a374684ff97117a && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 13cbd2c22d5..ef25ee0dfbe 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 2230caa24c7d4bd07edb08b54e9368f9c73eae6e +branch = 2ecafcff40132f56eb2b494e1a374684ff97117a [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 4d1435e61de..f6f35aa873e 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 2230caa24c7d4bd07edb08b54e9368f9c73eae6e +branch = 2ecafcff40132f56eb2b494e1a374684ff97117a [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 6c4570a4078..9657cec146d 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "2230caa24c7d4bd07edb08b54e9368f9c73eae6e" +set(WarpX_amrex_branch "2ecafcff40132f56eb2b494e1a374684ff97117a" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/run_test.sh b/run_test.sh index 2b53571163a..ff41399a9c8 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 2230caa24c7d4bd07edb08b54e9368f9c73eae6e && cd - +cd amrex && git checkout --detach 2ecafcff40132f56eb2b494e1a374684ff97117a && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 1d87c26f5ae06a6e97847493df9ee2ea0df4026d Mon Sep 17 00:00:00 2001 From: Marco Garten Date: Mon, 26 Feb 2024 16:12:19 -0800 Subject: [PATCH 172/176] Position keyword now adds position variables in PICMI ParticleDiagnostic (#4721) * Position keyword in ParticleDiags now adds pos Prior to recently, the `'position'` string in the input of the PICMI `ParticleDiagnostic` did not add the positions because they were written out automatically. That has been changed and so we are adding them to the list of variables now. Also added the class `LabFrameParticleDiagnostic` to the docs. * Fix dimension-specific cases and examples - in picmi.py, the dimensions were not checked and so it would try to add variables that do not exist for a species in a lower dimension - in examples, some coordinates were named wrongly for lower dimensions than 2 but it did not matter before since positions were added automatically --- Docs/source/usage/python.rst | 2 + Examples/Tests/ionization/PICMI_inputs_2d.py | 2 +- .../PICMI_inputs.py | 2 +- .../PICMI_inputs.py | 2 +- .../PICMI_inputs.py | 2 +- Python/pywarpx/picmi.py | 88 ++++++++++++++----- 6 files changed, 71 insertions(+), 27 deletions(-) diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index b18606c9a53..639b57d219e 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -94,6 +94,8 @@ Diagnostics Lab-frame diagnostics diagnostics are used when running boosted-frame simulations. +.. autoclass:: pywarpx.picmi.LabFrameParticleDiagnostic + .. autoclass:: pywarpx.picmi.LabFrameFieldDiagnostic Particles diff --git a/Examples/Tests/ionization/PICMI_inputs_2d.py b/Examples/Tests/ionization/PICMI_inputs_2d.py index 86d24181a75..a076361bf50 100644 --- a/Examples/Tests/ionization/PICMI_inputs_2d.py +++ b/Examples/Tests/ionization/PICMI_inputs_2d.py @@ -88,7 +88,7 @@ name = 'diag1', period = 10000, species = [electrons, ions], - data_list = ['ux', 'uy', 'uz', 'x', 'y', 'weighting'], + data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting'], write_dir = '.', warpx_file_prefix = 'Python_ionization_plt') field_diag = picmi.FieldDiagnostic( diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py index aa0cc96d97e..d545195a9e5 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py @@ -222,7 +222,7 @@ def setup_run(self): period=100, write_dir='.', species=[self.ions], - data_list = ['ux', 'uy', 'uz', 'x', 'y', 'weighting'], + data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting'], warpx_file_prefix=f'Python_ohms_law_solver_landau_damping_{self.dim}d_plt', ) simulation.add_diagnostic(particle_diag) diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py index 514f336c482..d9a71df6ac4 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py @@ -259,7 +259,7 @@ def setup_run(self): name='diag1', period=1250, species=[self.ions, self.beam_ions], - data_list = ['ux', 'uy', 'uz', 'x', 'weighting'], + data_list = ['ux', 'uy', 'uz', 'z', 'weighting'], write_dir='.', warpx_file_prefix='Python_ohms_law_solver_ion_beam_1d_plt', ) diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py index 9bcd0584e0f..a7f2e0da12d 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py @@ -255,7 +255,7 @@ def setup_run(self): period=self.total_steps, write_dir='.', species=[self.ions], - data_list=['ux', 'uy', 'uz', 'x', 'y', 'weighting'], + data_list=['ux', 'uy', 'uz', 'x', 'z', 'weighting'], warpx_file_prefix='Python_ohms_law_solver_magnetic_reconnection_2d_plt', # warpx_format='openpmd', # warpx_openpmd_backend='h5', diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 02c9e38c8d5..959db6c701d 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2442,8 +2442,13 @@ def diagnostic_initialize_inputs(self): if self.data_list is not None: for dataname in self.data_list: if dataname == 'position': - # --- The positions are alway written out anyway - pass + if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z + variables.add('x') + if pywarpx.geometry.dims == '3': + variables.add('y') + variables.add('z') + if pywarpx.geometry.dims == 'RZ': + variables.add('theta') elif dataname == 'momentum': variables.add('ux') variables.add('uy') @@ -2457,8 +2462,24 @@ def diagnostic_initialize_inputs(self): variables.add('Bx') variables.add('By') variables.add('Bz') - elif dataname in ['ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz']: - variables.add(dataname) + elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: + if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): + raise RuntimeError( + f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" + f"chosen by dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != '3' and dataname == 'y': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + else: + variables.add(dataname) # --- Convert the set to a sorted list so that the order # --- is the same on all processors. @@ -2666,25 +2687,46 @@ def diagnostic_initialize_inputs(self): # --- Use a set to ensure that fields don't get repeated. variables = set() - if self.data_list is not None: - for dataname in self.data_list: - if dataname == 'position': - # --- The positions are alway written out anyway - pass - elif dataname == 'momentum': - variables.add('ux') - variables.add('uy') - variables.add('uz') - elif dataname == 'weighting': - variables.add('w') - elif dataname == 'fields': - variables.add('Ex') - variables.add('Ey') - variables.add('Ez') - variables.add('Bx') - variables.add('By') - variables.add('Bz') - elif dataname in ['ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: + + for dataname in self.data_list: + if dataname == 'position': + if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z + variables.add('x') + if pywarpx.geometry.dims == '3': + variables.add('y') + variables.add('z') + if pywarpx.geometry.dims == 'RZ': + variables.add('theta') + elif dataname == 'momentum': + variables.add('ux') + variables.add('uy') + variables.add('uz') + elif dataname == 'weighting': + variables.add('w') + elif dataname == 'fields': + variables.add('Ex') + variables.add('Ey') + variables.add('Ez') + variables.add('Bx') + variables.add('By') + variables.add('Bz') + elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: + if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): + raise RuntimeError( + f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" + f"chosen by dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != '3' and dataname == 'y': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + else: variables.add(dataname) # --- Convert the set to a sorted list so that the order From ac2e64943d04781e8bc6d3a952983437ef225ece Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 26 Feb 2024 16:12:53 -0800 Subject: [PATCH 173/176] openPMD: Fix Particle Weight Dims in 1D/2D (#4717) * openPMD: Fix Particle Weight Dims Weighting of a macro particle is a count in 3D and RZ sims, but a count per line in 2D and a count per surface in 1D. This reflects this now in the `weighting` attribute `unitDimensions`. * Simplify * Spaces --- Source/Diagnostics/WarpXOpenPMD.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index e284e254d1e..2569e6cd1e8 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -316,7 +316,15 @@ namespace detail return {{openPMD::UnitDimension::T, 1.}, {openPMD::UnitDimension::I, 1.}}; } else if( record_name == "mass" ) { - return {{openPMD::UnitDimension::M, 1.}}; + return {{openPMD::UnitDimension::M, 1.}}; + } else if( record_name == "weighting" ) { +#if defined(WARPX_DIM_1D_Z) + return {{openPMD::UnitDimension::L, -2.}}; +#elif defined(WARPX_DIM_XZ) + return {{openPMD::UnitDimension::L, -1.}}; +#else // 3D and RZ + return {}; +#endif } else if( record_name == "E" ) { return {{openPMD::UnitDimension::L, 1.}, {openPMD::UnitDimension::M, 1.}, From 1af03a1828665688d30f88c48a6f7f9eae2cf462 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 27 Feb 2024 09:52:03 -0800 Subject: [PATCH 174/176] Clang-Tidy: Fix Diagnostics Branch (#4731) Keep for readability as is. --- Source/Diagnostics/WarpXOpenPMD.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 2569e6cd1e8..ec8daa52dd9 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -317,7 +317,7 @@ namespace detail {openPMD::UnitDimension::I, 1.}}; } else if( record_name == "mass" ) { return {{openPMD::UnitDimension::M, 1.}}; - } else if( record_name == "weighting" ) { + } else if( record_name == "weighting" ) { // NOLINT(bugprone-branch-clone) #if defined(WARPX_DIM_1D_Z) return {{openPMD::UnitDimension::L, -2.}}; #elif defined(WARPX_DIM_XZ) @@ -334,7 +334,7 @@ namespace detail return {{openPMD::UnitDimension::M, 1.}, {openPMD::UnitDimension::I, -1.}, {openPMD::UnitDimension::T, -2.}}; - } else { + } else { // NOLINT(bugprone-branch-clone) return {}; } } From 5e66d0edb716dc657e5841e5f088eb057bcc32ec Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 27 Feb 2024 10:46:50 -0800 Subject: [PATCH 175/176] Exchange guard cells with the electrostatic solver (#4713) * Exchange guard cells for electrostatic solver * Update calls to fillboundary * Update checksums * Add automated test * Update benchmark * Add automated test for energy conservation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Examples/Tests/energy_conserving_thermal_plasma/analysis.py * Update Regression/WarpX-tests.ini * Fix automated test * Update benchmarks * Apply suggestions from code review Co-authored-by: Luca Fedeli * Update input script --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Luca Fedeli --- .../analysis.py | 38 +++++ .../inputs_2d_electrostatic | 83 ++++++++++ .../benchmarks_json/BeamBeamCollision.json | 142 +++++++++--------- .../benchmarks_json/ElectrostaticSphere.json | 20 +-- .../ElectrostaticSphereLabFrame.json | 20 +-- .../ElectrostaticSphereRZ.json | 18 +-- .../EnergyConservingThermalPlasma.json | 29 ++++ Regression/WarpX-tests.ini | 16 ++ Source/Evolve/WarpXEvolve.cpp | 26 ++-- 9 files changed, 279 insertions(+), 113 deletions(-) create mode 100755 Examples/Tests/energy_conserving_thermal_plasma/analysis.py create mode 100644 Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic create mode 100644 Regression/Checksum/benchmarks_json/EnergyConservingThermalPlasma.json diff --git a/Examples/Tests/energy_conserving_thermal_plasma/analysis.py b/Examples/Tests/energy_conserving_thermal_plasma/analysis.py new file mode 100755 index 00000000000..43e9b6d9822 --- /dev/null +++ b/Examples/Tests/energy_conserving_thermal_plasma/analysis.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +# Copyright 2024, Remi Lehe +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests the conservation of energy for a thermal plasma with periodic boundary. +# Here, we use energy-converving gather and an electrostatic solver. The energy +# is not expected to be exactly conserved, but it is expected to be better conserved +# than other gathering scheme. This tests checks that the energy does not increase by +# more than 0.3% over the duration of the simulatoin. + +import os +import sys + +import numpy as np + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Get energy as a function of time, from reduced diagnostics +EFdata = np.genfromtxt('./diags/reducedfiles/EF.txt') # Field energy +EPdata = np.genfromtxt('./diags/reducedfiles/EP.txt') # Particle energy +field_energy = EFdata[:,2] +particle_energy = EPdata[:,2] +E = field_energy + particle_energy +print(abs(E-E[0])/E[0]) +# Check that the energy is conserved to 0.3% +assert np.all( abs(E-E[0])/E[0] < 0.003 ) + +# Checksum test +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic b/Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic new file mode 100644 index 00000000000..dab1b77db50 --- /dev/null +++ b/Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic @@ -0,0 +1,83 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 500 +amr.n_cell = 8 8 +amr.max_level = 0 +geometry.dims = 2 + +warpx.do_electrostatic = labframe +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ CONSTANTS ############# +################################# +my_constants.n0 = 1.e30 # plasma density, m^-3 +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.dt = ( 0.2 )/wpe # time step size, s +warpx.const_dt = dt + +################################# +####### GENERAL PARAMETERS ###### +################################# +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 +geometry.prob_hi = 10.*de0 10.*de0 +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 + +################################# +###### BOUNDARY CONDITIONS ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic +boundary.particle_lo = periodic periodic +boundary.particle_hi = periodic periodic + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = 2 2 +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight +electrons.xmin = 0 +electrons.xmax = 10.*de0 +electrons.zmin = 0 +electrons.zmax =10.*de0 + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = 2 2 +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight +protons.xmin = 0 +protons.xmax = 10.*de0 +protons.zmin = 0 +protons.zmax =10.*de0 + +diagnostics.diags_names = diag1 +diag1.intervals = 500 +diag1.diag_type = Full + +warpx.reduced_diags_names = EP EF +EP.type = ParticleEnergy +EP.intervals = 100 +EF.type = FieldEnergy +EF.intervals =100 diff --git a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json index 16c27055ab5..693faf43882 100644 --- a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json +++ b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json @@ -1,96 +1,96 @@ { "lev=0": { - "Bx": 971135657171.612, - "By": 971078812454.5405, - "Bz": 20140193.235893946, - "Ex": 2.9111756162943966e+20, - "Ey": 2.9113725115697712e+20, - "Ez": 1.0213536367191107e+17, - "rho_beam1": 7.970337929028706e+16, - "rho_beam2": 7.969213804851568e+16, - "rho_ele1": 343600677331163.7, - "rho_ele2": 302746939366837.25, - "rho_pos1": 333855946230626.06, - "rho_pos2": 310879461124837.44 + "Bx": 970573570349.2142, + "By": 970493693614.8823, + "Bz": 18743533.08373785, + "Ex": 2.9100804648817213e+20, + "Ey": 2.910188719414779e+20, + "Ez": 1.309938057448976e+17, + "rho_beam1": 7.969171294602906e+16, + "rho_beam2": 7.969079911431987e+16, + "rho_ele1": 234837475722889.97, + "rho_ele2": 277285508564124.12, + "rho_pos1": 229824415763789.62, + "rho_pos2": 286513388076434.4 }, "beam1": { - "particle_opticalDepthQSR": 104909.59461909423, - "particle_position_x": 0.001500222221634118, - "particle_position_y": 0.0015002445303035634, - "particle_position_z": 0.0049656251227976015, - "particle_momentum_x": 6.205341799808723e-15, - "particle_momentum_y": 6.1592257603817594e-15, - "particle_momentum_z": 6.806886719670214e-12, - "particle_weight": 635949610.5135971 + "particle_opticalDepthQSR": 104947.58821047074, + "particle_position_x": 0.0015001846430437182, + "particle_position_y": 0.0015002200838688795, + "particle_position_z": 0.004965556992831605, + "particle_momentum_x": 6.207861637949496e-15, + "particle_momentum_y": 6.16591876835476e-15, + "particle_momentum_z": 6.806755306915448e-12, + "particle_weight": 635859587.7915188 }, "beam2": { - "particle_opticalDepthQSR": 104164.848014815, - "particle_position_x": 0.0015001011957527532, - "particle_position_y": 0.001500139975740741, - "particle_position_z": 0.004965479176845744, - "particle_momentum_x": 6.200690794877584e-15, - "particle_momentum_y": 6.186048913459861e-15, - "particle_momentum_z": 6.7990490255176515e-12, - "particle_weight": 635863144.251134 + "particle_opticalDepthQSR": 104180.42997257302, + "particle_position_x": 0.0015001132145418016, + "particle_position_y": 0.001500162338878405, + "particle_position_z": 0.004965528365042041, + "particle_momentum_x": 6.201791193788119e-15, + "particle_momentum_y": 6.188946480983165e-15, + "particle_momentum_z": 6.796967564632488e-12, + "particle_weight": 635853166.1373858 }, "ele1": { - "particle_opticalDepthQSR": 435.73003117907257, - "particle_position_x": 4.882183530045367e-06, - "particle_position_y": 4.841391483672882e-06, - "particle_position_z": 1.8449175055560687e-05, - "particle_momentum_x": 5.6656608489971696e-18, - "particle_momentum_y": 5.724425295258085e-18, - "particle_momentum_z": 2.6277553331470036e-15, - "particle_weight": 2696555.9200674472 + "particle_opticalDepthQSR": 385.3959944370225, + "particle_position_x": 5.063867198487091e-06, + "particle_position_y": 5.323267522706671e-06, + "particle_position_z": 1.81974337459613e-05, + "particle_momentum_x": 5.220808698440275e-18, + "particle_momentum_y": 5.2833645967924376e-18, + "particle_momentum_z": 2.4960852396311436e-15, + "particle_weight": 1883777.3071500752 }, "ele2": { - "particle_opticalDepthQSR": 340.82229684726735, - "particle_position_x": 5.233059654483856e-06, - "particle_position_y": 4.781569085220371e-06, - "particle_position_z": 1.6293559425324337e-05, - "particle_momentum_x": 4.802611971470525e-18, - "particle_momentum_y": 4.3556825243407754e-18, - "particle_momentum_z": 2.41587659230925e-15, - "particle_weight": 2481137.860727036 + "particle_opticalDepthQSR": 391.0032239817431, + "particle_position_x": 5.442400321293304e-06, + "particle_position_y": 5.3621200149906786e-06, + "particle_position_z": 1.716649458876156e-05, + "particle_momentum_x": 4.7056270741663116e-18, + "particle_momentum_y": 4.414438514376292e-18, + "particle_momentum_z": 2.3816287305827113e-15, + "particle_weight": 2255238.1204111334 }, "pho1": { - "particle_opticalDepthBW": 9894.64959724129, - "particle_position_x": 0.00014191369823397644, - "particle_position_y": 0.00014347545392717968, - "particle_position_z": 0.00047442826029322116, + "particle_opticalDepthBW": 9912.332305353064, + "particle_position_x": 0.0001424857999095925, + "particle_position_y": 0.00014366766674682975, + "particle_position_z": 0.0004764478273708734, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_weight": 61063948.68610941 + "particle_weight": 61671336.44071695 }, "pho2": { - "particle_opticalDepthBW": 10292.1955840901, - "particle_position_x": 0.00014731892710321073, - "particle_position_y": 0.00014515617182809124, - "particle_position_z": 0.00048756513452074315, + "particle_opticalDepthBW": 10361.142481614075, + "particle_position_x": 0.00014718841339786984, + "particle_position_y": 0.00014538267635727008, + "particle_position_z": 0.00048768591004702896, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_weight": 62299636.622087024 + "particle_weight": 61889405.34553001 }, "pos1": { - "particle_opticalDepthQSR": 387.7441392212553, - "particle_position_x": 5.1462880118803425e-06, - "particle_position_y": 5.2613832293016684e-06, - "particle_position_z": 1.7054223425917483e-05, - "particle_momentum_x": 4.6437665862693495e-18, - "particle_momentum_y": 4.761862836969051e-18, - "particle_momentum_z": 2.3776996599289627e-15, - "particle_weight": 2625121.7841375084 + "particle_opticalDepthQSR": 343.0370605565725, + "particle_position_x": 5.41949101836001e-06, + "particle_position_y": 5.5292865311090835e-06, + "particle_position_z": 1.7063207973991194e-05, + "particle_momentum_x": 4.76704175252458e-18, + "particle_momentum_y": 5.049007427826035e-18, + "particle_momentum_z": 2.533578387785534e-15, + "particle_weight": 1821809.4309160584 }, "pos2": { - "particle_opticalDepthQSR": 361.943365907597, - "particle_position_x": 4.969019565031149e-06, - "particle_position_y": 4.361394970806125e-06, - "particle_position_z": 1.7413304358612675e-05, - "particle_momentum_x": 5.6348322786528905e-18, - "particle_momentum_y": 4.8171439953214205e-18, - "particle_momentum_z": 2.1937254860708963e-15, - "particle_weight": 2529794.7740602638 + "particle_opticalDepthQSR": 390.0211924195479, + "particle_position_x": 5.053169658257247e-06, + "particle_position_y": 5.039302796539821e-06, + "particle_position_z": 1.8526669235892217e-05, + "particle_momentum_x": 5.755216173987019e-18, + "particle_momentum_y": 5.056618594918483e-18, + "particle_momentum_z": 2.52324147594341e-15, + "particle_weight": 2328558.5523011778 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json index 2d26136cc5e..4e9bde52a78 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json @@ -1,17 +1,17 @@ { "lev=0": { - "Ex": 6.504757806138167, - "Ey": 6.504757806138166, - "Ez": 6.501489620648247, - "rho": 2.60925680083338e-10 + "Ex": 6.504803536004636, + "Ey": 6.504803536004637, + "Ez": 6.504803536004637, + "rho": 2.609256800833379e-10 }, "electron": { - "particle_momentum_x": 1.0083594348819436e-23, - "particle_momentum_y": 1.0083594348819436e-23, - "particle_momentum_z": 1.0159789930728084e-23, - "particle_position_x": 518.4112403470117, - "particle_position_y": 518.4112403470117, - "particle_position_z": 520.2110302538686, + "particle_momentum_x": 1.0084159184928337e-23, + "particle_momentum_y": 1.008415918492834e-23, + "particle_momentum_z": 1.0084159184928338e-23, + "particle_position_x": 518.418176976446, + "particle_position_y": 518.4181769764461, + "particle_position_z": 518.418176976446, "particle_weight": 6212.501525878906 } } diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame.json index d46a8cfcda4..648e805c938 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame.json @@ -1,17 +1,17 @@ { "lev=0": { - "Ex": 6.504586436288052, - "Ey": 6.504586436288051, - "Ez": 6.501896077968277, - "rho": 2.6092568008333797e-10 + "Ex": 6.504631859055906, + "Ey": 6.504631859055904, + "Ez": 6.504631859055904, + "rho": 2.609256800833379e-10 }, "electron": { - "particle_momentum_x": 9.881704085619681e-24, - "particle_momentum_y": 9.881704085619682e-24, - "particle_momentum_z": 9.969862126470096e-24, - "particle_position_x": 513.5147715983729, - "particle_position_y": 513.5147715983729, - "particle_position_z": 515.4187595183639, + "particle_momentum_x": 9.882423407883042e-24, + "particle_momentum_y": 9.882423407883041e-24, + "particle_momentum_z": 9.882423407883041e-24, + "particle_position_x": 513.522905933945, + "particle_position_y": 513.522905933945, + "particle_position_z": 513.522905933945, "particle_weight": 6212.501525878906 } } diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRZ.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereRZ.json index 038ea5a6449..a94e8a4cd0b 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRZ.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereRZ.json @@ -1,17 +1,17 @@ { "lev=0": { - "Er": 0.11588932103656224, + "Er": 0.11588944816311227, "Et": 0.0, - "Ez": 0.11483033274834441, - "rho": 9.716216490680392e-12 + "Ez": 0.11492556391155323, + "rho": 9.71614990653583e-12 }, "electron": { - "particle_momentum_x": 4.54997005723341e-25, - "particle_momentum_y": 4.434893182689167e-25, - "particle_momentum_z": 6.989989085028144e-25, - "particle_position_x": 35.73642348640023, - "particle_position_y": 35.69851886414351, + "particle_momentum_x": 4.550172089304043e-25, + "particle_momentum_y": 4.435063230214811e-25, + "particle_momentum_z": 6.953391709067729e-25, + "particle_position_x": 35.73677703655427, + "particle_position_y": 35.61000307679542, "particle_theta": 823.0413054971611, "particle_weight": 6406.017620347038 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/EnergyConservingThermalPlasma.json b/Regression/Checksum/benchmarks_json/EnergyConservingThermalPlasma.json new file mode 100644 index 00000000000..3451ac42a9f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/EnergyConservingThermalPlasma.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 14828575131693.746, + "Ey": 0.0, + "Ez": 15137612614715.773, + "jx": 5.572707185530917e+18, + "jy": 7.091118076558307e+18, + "jz": 6.668532755465409e+18 + }, + "protons": { + "particle_momentum_x": 3.187482462935171e-20, + "particle_momentum_y": 3.397549094666928e-20, + "particle_momentum_z": 3.300591612645518e-20, + "particle_position_x": 6.8052980975094785e-06, + "particle_position_y": 6.8011432612301085e-06, + "particle_weight": 2823958719279159.5 + }, + "electrons": { + "particle_momentum_x": 7.610483420803999e-22, + "particle_momentum_y": 8.024587481438078e-22, + "particle_momentum_z": 8.138108658606551e-22, + "particle_position_x": 6.8822971884226895e-06, + "particle_position_y": 6.8608357509110685e-06, + "particle_weight": 2823958719279159.5 + } +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index f6f35aa873e..128495195da 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4648,6 +4648,22 @@ doVis = 0 compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py +[EnergyConservingThermalPlasma] +buildDir = . +inputFile = Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic +dim = 2 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=2 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/energy_conserving_thermal_plasma/analysis.py + [focusing_gaussian_beam] buildDir = . inputFile = Examples/Tests/gaussian_beam/inputs_focusing_beam diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index b062501ed1b..dd6acfd2694 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -122,12 +122,12 @@ WarpX::Evolve (int numsteps) // At the beginning, we have B^{n} and E^{n}. // Particles have p^{n} and x^{n}. // is_synchronized is true. + if (is_synchronized) { - if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { - // Not called at each iteration, so exchange all guard cells - FillBoundaryE(guard_cells.ng_alloc_EB); - FillBoundaryB(guard_cells.ng_alloc_EB); - } + // Not called at each iteration, so exchange all guard cells + FillBoundaryE(guard_cells.ng_alloc_EB); + FillBoundaryB(guard_cells.ng_alloc_EB); + UpdateAuxilaryData(); FillBoundaryAux(guard_cells.ng_UpdateAux); // on first step, push p by -0.5*dt @@ -140,15 +140,15 @@ WarpX::Evolve (int numsteps) is_synchronized = false; } else { + // Beyond one step, we have E^{n} and B^{n}. + // Particles have p^{n-1/2} and x^{n}. + // E and B: enough guard cells to update Aux or call Field Gather in fp and cp + // Need to update Aux on lower levels, to interpolate to higher levels. + + // E and B are up-to-date inside the domain only + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { - // Beyond one step, we have E^{n} and B^{n}. - // Particles have p^{n-1/2} and x^{n}. - - // E and B are up-to-date inside the domain only - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - // E and B: enough guard cells to update Aux or call Field Gather in fp and cp - // Need to update Aux on lower levels, to interpolate to higher levels. if (fft_do_time_averaging) { FillBoundaryE_avg(guard_cells.ng_FieldGather); From 6be87d6fa38290fb44756b9679846db38b615385 Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:46:44 -0800 Subject: [PATCH 176/176] Adding support for pml in domain for multiple patches (#4632) * this works only on single proc, but lifts the assumption of single patch for do pml in domain * fix cbl * building reduced ba and cba * Removing print statements * removing comment * assert that box length must be greater than size of pml when its grown inwards * same assertion for box length for coarse grid * simplified_list and no need for assert in coarse, but if fine works, coarse should * boxlist instead of simplified list * comments * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * CI test for double patch with particles entering and leaving pml inside the patch * checksum for new particles in pml 2d mr test with double patch * renaming plt to diag1 again * include level 0 maxfield to capture no j damping effect * removing incorect comment about heavy paritcle in input --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../analysis_particles_in_pml_2dmr.py | 68 ++++++++++++++ Examples/Tests/particles_in_pml/inputs_mr_2d | 15 ++-- .../particles_in_pml_2d_MR.json | 12 +-- Regression/WarpX-tests.ini | 2 +- Source/BoundaryConditions/PML.cpp | 89 +++++++++++-------- 5 files changed, 136 insertions(+), 50 deletions(-) create mode 100755 Examples/Tests/particles_in_pml/analysis_particles_in_pml_2dmr.py diff --git a/Examples/Tests/particles_in_pml/analysis_particles_in_pml_2dmr.py b/Examples/Tests/particles_in_pml/analysis_particles_in_pml_2dmr.py new file mode 100755 index 00000000000..81a981e70a6 --- /dev/null +++ b/Examples/Tests/particles_in_pml/analysis_particles_in_pml_2dmr.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2020 Luca Fedeli, Maxence Thevenet, Remi Lehe +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +""" +This script tests the absorption of particles in the PML. + +The input file inputs_2d/inputs is used: it features a positive and a +negative particle, going in opposite direction and eventually +leaving the box. This script tests that the field in the box +is close to 0 once the particles have left. With regular +PML, this test fails, since the particles leave a spurious +charge, with associated fields, behind them. +""" +import os +import sys + +import yt + +yt.funcs.mylog.setLevel(0) +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# Open plotfile specified in command line +filename = sys.argv[1] +ds = yt.load( filename ) + +# Check that the field is low enough +ad0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +Ex_array_lev0 = ad0[('mesh','Ex')].to_ndarray() +Ey_array_lev0 = ad0[('mesh','Ey')].to_ndarray() +Ez_array_lev0 = ad0[('mesh','Ez')].to_ndarray() +max_Efield_lev0 = max(Ex_array_lev0.max(), Ey_array_lev0.max(), Ez_array_lev0.max()) +print( "max_Efield level0 = %s" %max_Efield_lev0 ) + +ad1 = ds.covering_grid(level=1, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +Ex_array_lev1 = ad1[('mesh','Ex')].to_ndarray() +Ey_array_lev1 = ad1[('mesh','Ey')].to_ndarray() +Ez_array_lev1 = ad1[('mesh','Ez')].to_ndarray() +max_Efield_lev1 = max(Ex_array_lev1.max(), Ey_array_lev1.max(), Ez_array_lev1.max()) +print( "max_Efield level1 = %s" %max_Efield_lev1 ) + +# The field associated with the particle does not have +# the same amplitude in 2d and 3d +if ds.dimensionality == 2: + if ds.max_level == 0: + tolerance_abs = 0.0003 + elif ds.max_level == 1: + tolerance_abs = 0.0006 +elif ds.dimensionality == 3: + if ds.max_level == 0: + tolerance_abs = 10 + elif ds.max_level == 1: + tolerance_abs = 110 +else: + raise ValueError("Unknown dimensionality") + +print("tolerance_abs: " + str(tolerance_abs)) +assert max_Efield_lev0 < tolerance_abs +assert max_Efield_lev1 < tolerance_abs + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/particles_in_pml/inputs_mr_2d b/Examples/Tests/particles_in_pml/inputs_mr_2d index a6db42a3897..40e869a5e16 100644 --- a/Examples/Tests/particles_in_pml/inputs_mr_2d +++ b/Examples/Tests/particles_in_pml/inputs_mr_2d @@ -1,16 +1,15 @@ -max_step = 200 +max_step = 300 amr.n_cell = 128 128 amr.blocking_factor = 16 -amr.max_grid_size = 1024 +amr.max_grid_size = 64 amr.max_level = 1 # Geometry geometry.dims = 2 geometry.prob_lo = -32.e-6 -32.e-6 # physical domain geometry.prob_hi = 32.e-6 32.e-6 -warpx.fine_tag_lo = -8.e-6 -8.e-6 # physical domain -warpx.fine_tag_hi = 8.e-6 8.e-6 +warpx.ref_patch_function(x,y,z) = " ((x > -16.e-6)*(x<-10.e-6) + (x > 9.e-6)*(x<16.e-6))*(z>-8.e-6)*(z<8.e-6)" # Boundary condition boundary.field_lo = pml pml pml @@ -38,14 +37,14 @@ particles.species_names = electron proton electron.charge = -q_e electron.mass = m_e electron.injection_style = "singleparticle" -electron.single_particle_pos = 0. 0. 0. +electron.single_particle_pos = -12.e-6 0. 0. electron.single_particle_u = 2. 0. 0. electron.single_particle_weight = 1. proton.charge = q_e -proton.mass = m_p # Very heavy ; should not move +proton.mass = m_p proton.injection_style = "singleparticle" -proton.single_particle_pos = 0. 0. 0. +proton.single_particle_pos = -12.e-6 0. 0. proton.single_particle_u = -2. 0. 0. proton.single_particle_weight = 1. @@ -54,6 +53,6 @@ algo.particle_shape = 1 # Diagnostics diagnostics.diags_names = diag1 -diag1.intervals = 200 +diag1.intervals = 300 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz diff --git a/Regression/Checksum/benchmarks_json/particles_in_pml_2d_MR.json b/Regression/Checksum/benchmarks_json/particles_in_pml_2d_MR.json index 11c0a2a6108..dfe63ddc1c2 100644 --- a/Regression/Checksum/benchmarks_json/particles_in_pml_2d_MR.json +++ b/Regression/Checksum/benchmarks_json/particles_in_pml_2d_MR.json @@ -1,22 +1,22 @@ { "lev=0": { "Bx": 0.0, - "By": 3.578051588629885e-09, + "By": 1.959394057951299e-09, "Bz": 0.0, - "Ex": 1.9699822913484977, + "Ex": 1.2177330062543195, "Ey": 0.0, - "Ez": 0.5356004212513483, + "Ez": 0.28770171464461436, "jx": 0.0, "jy": 0.0, "jz": 0.0 }, "lev=1": { "Bx": 0.0, - "By": 2.7629151306453947e-09, + "By": 1.4999363537195907e-09, "Bz": 0.0, - "Ex": 2.4597363065789337, + "Ex": 1.5591272748392493, "Ey": 0.0, - "Ez": 0.45901250290130735, + "Ez": 0.29412053281248907, "jx": 0.0, "jy": 0.0, "jz": 0.0 diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 128495195da..b25d41db5d2 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -2525,7 +2525,7 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 -analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py +analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml_2dmr.py [particles_in_pml_3d_MR] buildDir = . diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 1f0b6f09a33..805b4fec181 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -560,40 +560,56 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri m_geom(geom), m_cgeom(cgeom) { - // When `do_pml_in_domain` is true, the PML overlap with the last `ncell` of the physical domain - // (instead of extending `ncell` outside of the physical domain) - // In order to implement this, a reduced domain is created here (decreased by ncells in all direction) - // and passed to `MakeBoxArray`, which surrounds it by PML boxes - // (thus creating the PML boxes at the right position, where they overlap with the original domain) - // minimalBox provides the bounding box around grid_ba for level, lev. - // Note that this is okay to build pml inside domain for a single patch, or joint patches - // with same [min,max]. But it does not support multiple disjoint refinement patches. - Box domain0 = grid_ba.minimalBox(); + // When `do_pml_in_domain` is true, the PML overlap with the last `ncell` of the physical domain or fine patch(es) + // (instead of extending `ncell` outside of the physical domain or fine patch(es)) + // In order to implement this, we define a new reduced Box Array ensuring that it does not + // include ncells from the edges of the physical domain or fine patch. + // (thus creating the PML boxes at the right position, where they overlap with the original domain or fine patch(es)) + + BoxArray grid_ba_reduced = grid_ba; if (do_pml_in_domain) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (do_pml_Lo[idim]){ - domain0.growLo(idim, -ncell); - } - if (do_pml_Hi[idim]){ - domain0.growHi(idim, -ncell); + BoxList bl = grid_ba.boxList(); + // Here we loop over all the boxes in the original grid_ba BoxArray + // For each box, we find if its in the edge (or boundary), and the size of those boxes are decreased by ncell + for (auto& b : bl) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (do_pml_Lo[idim]) { + // Get neighboring box on lower side in direction idim and check if it intersects with any of the boxes + // in grid_ba. If no intersection, then the box, b, in the boxlist, is in the edge and we decrase + // the size by ncells using growLo(idim,-ncell) + Box const& bb = amrex::adjCellLo(b, idim); + if ( ! grid_ba.intersects(bb) ) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(b.length(idim) > ncell, " box length must be greater that pml size"); + b.growLo(idim, -ncell); + } + } + if (do_pml_Hi[idim]) { + // Get neighboring box on higher side in direction idim and check if it intersects with any of the boxes + // in grid_ba. If no intersection, then the box, b, in the boxlist, is in the edge and we decrase + // the size by ncells using growHi(idim,-ncell) + Box const& bb = amrex::adjCellHi(b, idim); + if ( ! grid_ba.intersects(bb) ) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(b.length(idim) > ncell, " box length must be greater that pml size"); + b.growHi(idim, -ncell); + } + } } } + grid_ba_reduced = BoxArray(std::move(bl)); } - const BoxArray grid_ba_reduced = (do_pml_in_domain) ? - BoxArray(grid_ba.boxList().intersect(domain0)) : grid_ba; - + Box const domain0 = grid_ba_reduced.minimalBox(); const bool is_single_box_domain = domain0.numPts() == grid_ba_reduced.numPts(); const BoxArray& ba = MakeBoxArray(is_single_box_domain, domain0, *geom, grid_ba_reduced, IntVect(ncell), do_pml_in_domain, do_pml_Lo, do_pml_Hi); + if (ba.empty()) { m_ok = false; return; } else { m_ok = true; } - - // Define the number of guard cells in each direction, for E, B, and F + // Define the number of guard cells in each di;rection, for E, B, and F auto nge = IntVect(AMREX_D_DECL(2, 2, 2)); auto ngb = IntVect(AMREX_D_DECL(2, 2, 2)); int ngf_int = 0; @@ -760,24 +776,28 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri BoxArray grid_cba = grid_ba; grid_cba.coarsen(ref_ratio); - // assuming that the bounding box around grid_cba is a single patch, and not disjoint patches, similar to fine patch. - amrex::Box cdomain = grid_cba.minimalBox(); + BoxArray grid_cba_reduced = grid_cba; if (do_pml_in_domain) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (do_pml_Lo[idim]){ - // ncell is divided by refinement ratio to ensure that the - // physical width of the PML region is equal in fine and coarse patch - cdomain.growLo(idim, -ncell/ref_ratio[idim]); - } - if (do_pml_Hi[idim]){ - // ncell is divided by refinement ratio to ensure that the - // physical width of the PML region is equal in fine and coarse patch - cdomain.growHi(idim, -ncell/ref_ratio[idim]); + BoxList bl = grid_cba.boxList(); + for (auto& b : bl) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (do_pml_Lo[idim]) { + Box const& bb = amrex::adjCellLo(b, idim); + if ( ! grid_cba.intersects(bb) ) { + b.growLo(idim, -ncell/ref_ratio[idim]); + } + } + if (do_pml_Hi[idim]) { + Box const& bb = amrex::adjCellHi(b, idim); + if ( ! grid_cba.intersects(bb) ) { + b.growHi(idim, -ncell/ref_ratio[idim]); + } + } } } + grid_cba_reduced = BoxArray(std::move(bl)); } - const BoxArray grid_cba_reduced = (do_pml_in_domain) ? - BoxArray(grid_cba.boxList().intersect(cdomain)) : grid_cba; + Box const cdomain = grid_cba_reduced.minimalBox(); const IntVect cncells = IntVect(ncell)/ref_ratio; const IntVect cdelta = IntVect(delta)/ref_ratio; @@ -792,7 +812,6 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri } else { cdm.define(cba); } - const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,0).ixType().toIntVect()); const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,1).ixType().toIntVect()); const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,2).ixType().toIntVect());