MPI-AMRVAC  3.0
The MPI - Adaptive Mesh Refinement - Versatile Advection Code
m_octree_mg_3d.t
Go to the documentation of this file.
1 
2 
3 ! Single module generated from the octree-mg sources.
4 ! This file can be easier to include in existing projects.
5 !
6 ! Notes:
7 ! 1. The module name is here extended by _1d, _2d or _3d
8 ! 2. The free space Poisson solver is not included here.
9 ! 3. It is best to make changes in the original repository at
10 ! https://github.com/jannisteunissen/octree-mg
11 ! 4. This file can be generated as follows:
12 ! cd octree-mg/single_module
13 ! ./to_single_module.sh
14 !
15 ! The modules can be compiled with:
16 ! mpif90 -c m_octree_mg_1d.f90 [other options]
17 ! mpif90 -c m_octree_mg_2d.f90 [other options]
18 ! mpif90 -c m_octree_mg_3d.f90 [other options]
19 ! mpif90 -c m_octree_mg.f90 -cpp -DNDIM=<1,2,3> [other options]
20 
21 
23  use mpi
24  implicit none
25  private
26 
27  !! File ../src/m_data_structures.f90
28 
29  !> Type of reals
30  integer, parameter, public :: dp = kind(0.0d0)
31 
32  !> Type for 64-bit integers
33  integer, parameter, public :: i8 = selected_int_kind(18)
34 
35  !> Indicates a standard Laplacian
36  integer, parameter, public :: mg_laplacian = 1
37 
38  !> Indicates a variable-coefficient Laplacian
39  integer, parameter, public :: mg_vlaplacian = 2
40 
41  !> Indicates a constant-coefficient Helmholtz equation
42  integer, parameter, public :: mg_helmholtz = 3
43 
44  !> Indicates a variable-coefficient Helmholtz equation
45  integer, parameter, public :: mg_vhelmholtz = 4
46 
47  !> Indicates a anisotropic-coefficient Helmholtz equation
48  integer, parameter, public :: mg_ahelmholtz = 5
49 
50  !> Cartesian coordinate system
51  integer, parameter, public :: mg_cartesian = 1
52  !> Cylindrical coordinate system
53  integer, parameter, public :: mg_cylindrical = 2
54  !> Spherical coordinate system
55  integer, parameter, public :: mg_spherical = 3
56 
57  integer, parameter, public :: mg_smoother_gs = 1
58  integer, parameter, public :: mg_smoother_gsrb = 2
59  integer, parameter, public :: mg_smoother_jacobi = 3
60 
61  !> Problem dimension
62  integer, parameter, public :: mg_ndim = 3
63 
64  !> Number of predefined multigrid variables
65  integer, parameter, public :: mg_num_vars = 4
66  !> Maximum number of variables
67  integer, parameter, public :: mg_max_num_vars = 10
68  !> Index of solution
69  integer, parameter, public :: mg_iphi = 1
70  !> Index of right-hand side
71  integer, parameter, public :: mg_irhs = 2
72  !> Index of previous solution (used for correction)
73  integer, parameter, public :: mg_iold = 3
74  !> Index of residual
75  integer, parameter, public :: mg_ires = 4
76 
77  !> Index of the variable coefficient (at cell centers)
78  integer, parameter, public :: mg_iveps = 5
79 
80  !> Indexes of anisotropic variable coefficients
81  integer, parameter, public :: mg_iveps1 = 5
82  integer, parameter, public :: mg_iveps2 = 6
83  integer, parameter, public :: mg_iveps3 = 7
84 
85  !> Minimum allowed grid level
86  integer, parameter, public :: mg_lvl_lo = -20
87  !> Maximum allowed grid level
88  integer, parameter, public :: mg_lvl_hi = 20
89 
90  !> Value to indicate a Dirichlet boundary condition
91  integer, parameter, public :: mg_bc_dirichlet = -10
92 
93  !> Value to indicate a Neumann boundary condition
94  integer, parameter, public :: mg_bc_neumann = -11
95 
96  !> Value to indicate a continuous boundary condition
97  integer, parameter, public :: mg_bc_continuous = -12
98 
99  !> Special value that indicates there is no box
100  integer, parameter, public :: mg_no_box = 0
101  !> Special value that indicates there is a physical boundary
102  integer, parameter, public :: mg_physical_boundary = -1
103 
104  !> Maximum number of timers to use
105  integer, parameter, public :: mg_max_timers = 20
106 
107  ! Numbering of children (same location as **corners**)
108  integer, parameter, public :: mg_num_children = 8
109 
110  ! Index offset for each child
111  integer, parameter, public :: mg_child_dix(3, 8) = reshape( &
112  [0,0,0, 1,0,0, 0,1,0, 1,1,0, &
113  0,0,1, 1,0,1, 0,1,1, 1,1,1], [3,8])
114  ! Reverse child index in each direction
115  integer, parameter, public :: mg_child_rev(8, 3) = reshape( &
116  [2,1,4,3,6,5,8,7, 3,4,1,2,7,8,5,6, 5,6,7,8,1,2,3,4], [8,3])
117  ! Children adjacent to a neighbor
118  integer, parameter, public :: mg_child_adj_nb(4, 6) = reshape( &
119  [1,3,5,7, 2,4,6,8, 1,2,5,6, 3,4,7,8, 1,2,3,4, 5,6,7,8], [4,6])
120  ! Which children have a low index per dimension
121  logical, parameter, public :: mg_child_low(3, 8) = reshape([ &
122  .true., .true., .true., .false., .true., .true., &
123  .true., .false., .true., .false., .false., .true., &
124  .true., .true., .false., .false., .true., .false., &
125  .true., .false., .false., .false., .false., .false.], [3, 8])
126 
127  ! Neighbor topology information
128  integer, parameter, public :: mg_num_neighbors = 6
129  integer, parameter, public :: mg_neighb_lowx = 1
130  integer, parameter, public :: mg_neighb_highx = 2
131  integer, parameter, public :: mg_neighb_lowy = 3
132  integer, parameter, public :: mg_neighb_highy = 4
133  integer, parameter, public :: mg_neighb_lowz = 5
134  integer, parameter, public :: mg_neighb_highz = 6
135  ! Index offsets of neighbors
136  integer, parameter, public :: mg_neighb_dix(3, 6) = reshape( &
137  [-1,0,0, 1,0,0, 0,-1,0, 0,1,0, 0,0,-1, 0,0,1], [3,6])
138  ! Which neighbors have a lower index
139  logical, parameter, public :: mg_neighb_low(6) = &
140  [.true., .false., .true., .false., .true., .false.]
141  ! Opposite of nb_low, but now as -1,1 integers
142  integer, parameter, public :: mg_neighb_high_pm(6) = [-1, 1, -1, 1, -1, 1]
143  ! Reverse neighbors
144  integer, parameter, public :: mg_neighb_rev(6) = [2, 1, 4, 3, 6, 5]
145  ! Direction (dimension) for a neighbor
146  integer, parameter, public :: mg_neighb_dim(6) = [1, 1, 2, 2, 3, 3]
147 
148  !> Lists of blocks per refinement level
149  type, public :: mg_lvl_t
150  integer, allocatable :: leaves(:)
151  integer, allocatable :: parents(:)
152  integer, allocatable :: ref_bnds(:)
153  integer, allocatable :: ids(:)
154  integer, allocatable :: my_leaves(:)
155  integer, allocatable :: my_parents(:)
156  integer, allocatable :: my_ref_bnds(:)
157  integer, allocatable :: my_ids(:)
158  end type mg_lvl_t
159 
160  !> Box data structure
161  type, public :: mg_box_t
162  integer :: rank !< Which process owns this box
163  integer :: id !< Box id (index in boxes(:) array)
164  integer :: lvl !< Refinement level
165  integer :: ix(3) !< Spatial index
166  integer :: parent !< Id of parent
167  integer :: children(2**3) !< Ids of children
168  integer :: neighbors(2*3) !< Ids of neighbors
169  real(dp) :: r_min(3) !< Minimum coordinate
170  real(dp) :: dr(3) !< Grid spacing
171  !> Cell-centered data
172  real(dp), allocatable :: cc(:, :, :, :)
173  end type mg_box_t
174 
175  !> Buffer type (one is used for each pair of communicating processes)
176  type, public :: mg_buf_t
177  integer :: i_send !< Index in send array
178  integer :: i_recv
179  integer :: i_ix
180  integer, allocatable :: ix(:) !< Will be used to sort the data
181  real(dp), allocatable :: send(:)
182  real(dp), allocatable :: recv(:)
183  end type mg_buf_t
184 
185  type, public :: mg_comm_t
186  integer, allocatable :: n_send(:, :)
187  integer, allocatable :: n_recv(:, :)
188  end type mg_comm_t
189 
190  type, public :: mg_bc_t
191  integer :: bc_type = mg_bc_dirichlet !< Type of boundary condition
192  real(dp) :: bc_value = 0.0_dp !< Value (for e.g. Dirichlet or Neumann)
193  !> To set user-defined boundary conditions (overrides bc(:))
194  procedure(mg_subr_bc), pointer, nopass :: boundary_cond => null()
195  !> To set a user-defined refinement boundary method
196  procedure(mg_subr_rb), pointer, nopass :: refinement_bnd => null()
197  end type mg_bc_t
198 
199  type, public :: mg_timer_t
200  character(len=20) :: name
201  real(dp) :: t = 0.0_dp
202  real(dp) :: t0
203  end type mg_timer_t
204 
205  type, public :: mg_t
206  !> Whether the multigrid tree structure has been created
207  logical :: tree_created = .false.
208  !> Whether storage has been allocated for boxes
209  logical :: is_allocated = .false.
210  !> Number of extra cell-centered variable (e.g., for coefficients)
211  integer :: n_extra_vars = 0
212  !> MPI communicator
213  integer :: comm = -1
214  !> Number of MPI tasks
215  integer :: n_cpu = -1
216  !> MPI rank of this task
217  integer :: my_rank = -1
218  !> Size of boxes in cells, be equal for all dimensions
219  integer :: box_size = -1
220  !> Highest grid level in the tree
221  integer :: highest_lvl = -1
222  !> Lowest grid level in the tree
223  integer :: lowest_lvl = -1
224  !> First normal level of the quadtree/octree, at coarser levels parents
225  !> have only one child
226  integer :: first_normal_lvl = -1
227  !> Total number of boxes in the tree (among all processes)
228  integer :: n_boxes = 0
229  !> Size of boxes per level (differs for coarsest levels)
230  integer :: box_size_lvl(mg_lvl_lo:mg_lvl_hi)
231  !> Size of domain per level (if uniformly refined)
232  integer :: domain_size_lvl(3, mg_lvl_lo:mg_lvl_hi)
233  !> Grid spacing per level
234  real(dp) :: dr(3, mg_lvl_lo:mg_lvl_hi)
235  !> Minimum coordinates
236  real(dp) :: r_min(3)
237  !> List of all levels
238  type(mg_lvl_t) :: lvls(mg_lvl_lo:mg_lvl_hi)
239  !> Array with all boxes in the tree. Only boxes owned by this task are
240  !> actually allocated
241  type(mg_box_t), allocatable :: boxes(:)
242  !> Buffer for communication with each other task
243  type(mg_buf_t), allocatable :: buf(:)
244 
245  !> Communication info for restriction
246  type(mg_comm_t) :: comm_restrict
247  !> Communication info for prolongation
248  type(mg_comm_t) :: comm_prolong
249  !> Communication info for ghost cell filling
250  type(mg_comm_t) :: comm_ghostcell
251 
252  !> Whether boundary condition data has been stored for mg solution
253  logical :: phi_bc_data_stored = .false.
254 
255  !> Whether a dimension is periodic
256  logical :: periodic(3) = .false.
257 
258  !> To store pre-defined boundary conditions per direction per variable
260 
261  !> Type of operator
262  integer :: operator_type = mg_laplacian
263  !> Type of grid geometry
264  integer :: geometry_type = mg_cartesian
265 
266  !> Whether the mean has to be subtracted from the multigrid solution
267  logical :: subtract_mean = .false.
268  !> Type of multigrid smoother
269  integer :: smoother_type = mg_smoother_gs
270  !> Number of substeps for the smoother (for GSRB this is 2)
271  integer :: n_smoother_substeps = 1
272  !> Number of cycles when doing downwards relaxation
273  integer :: n_cycle_down = 2
274  !> Number of cycles when doing upwards relaxation
275  integer :: n_cycle_up = 2
276  !> Maximum number of cycles on the coarse grid
277  integer :: max_coarse_cycles = 1000
278  integer :: coarsest_grid(3) = 2
279  !> Stop coarse grid when max. residual is smaller than this
280  real(dp) :: residual_coarse_abs = 1e-8_dp
281  !> Stop coarse grid when residual has been reduced by this factor
282  real(dp) :: residual_coarse_rel = 1e-8_dp
283 
284  !> Multigrid operator (e.g., Laplacian)
285  procedure(mg_box_op), pointer, nopass :: box_op => null()
286 
287  !> Multigrid smoother
288  procedure(mg_box_gsrb), pointer, nopass :: box_smoother => null()
289 
290  !> Multigrid prolongation method
291  procedure(mg_box_prolong), pointer, nopass :: box_prolong => null()
292 
293  !> Number of timers
294  integer :: n_timers = 0
295  !> Values for the timers
296  type(mg_timer_t) :: timers(mg_max_timers)
297  end type mg_t
298 
299  interface
300  !> To fill ghost cells near physical boundaries
301  subroutine mg_subr_bc(box, nc, iv, nb, bc_type, bc)
302  import
303  type(mg_box_t), intent(in) :: box
304  integer, intent(in) :: nc
305  integer, intent(in) :: iv !< Index of variable
306  integer, intent(in) :: nb !< Direction
307  integer, intent(out) :: bc_type !< Type of b.c.
308  !> Boundary values
309  real(dp), intent(out) :: bc(nc, nc)
310  end subroutine mg_subr_bc
311 
312  !> To fill ghost cells near refinement boundaries
313  subroutine mg_subr_rb(box, nc, iv, nb, cgc)
314  import
315  type(mg_box_t), intent(inout) :: box
316  integer, intent(in) :: nc
317  integer, intent(in) :: iv !< Index of variable
318  integer, intent(in) :: nb !< Direction
319  !> Coarse data
320  real(dp), intent(in) :: cgc(nc, nc)
321  end subroutine mg_subr_rb
322 
323  !> Subroutine that performs A * cc(..., i_in) = cc(..., i_out)
324  subroutine mg_box_op(mg, id, nc, i_out)
325  import
326  type(mg_t), intent(inout) :: mg
327  integer, intent(in) :: id
328  integer, intent(in) :: nc
329  integer, intent(in) :: i_out
330  end subroutine mg_box_op
331 
332  !> Subroutine that performs Gauss-Seidel relaxation
333  subroutine mg_box_gsrb(mg, id, nc, redblack_cntr)
334  import
335  type(mg_t), intent(inout) :: mg
336  integer, intent(in) :: id
337  integer, intent(in) :: nc
338  integer, intent(in) :: redblack_cntr
339  end subroutine mg_box_gsrb
340 
341  !> Subroutine that performs prolongation to a single child
342  subroutine mg_box_prolong(mg, p_id, dix, nc, iv, fine)
343  import
344  type(mg_t), intent(inout) :: mg
345  integer, intent(in) :: p_id !< Id of parent
346  integer, intent(in) :: dix(3) !< Offset of child in parent grid
347  integer, intent(in) :: nc !< Child grid size
348  integer, intent(in) :: iv !< Prolong from this variable
349  real(dp), intent(out) :: fine(nc, nc, nc) !< Prolonged values
350  end subroutine mg_box_prolong
351  end interface
352 
353  ! Public methods
354  public :: mg_subr_bc
355  public :: mg_subr_rb
356  public :: mg_box_op
357  public :: mg_box_gsrb
358  public :: mg_box_prolong
359  public :: mg_has_children
360  public :: mg_ix_to_ichild
361  public :: mg_get_child_offset
362  public :: mg_highest_uniform_lvl
363  public :: mg_number_of_unknowns
364  public :: mg_get_face_coords
365  public :: mg_add_timer
366  public :: mg_timer_start
367  public :: mg_timer_end
368  public :: mg_timers_show
369 
370  !! File ../src/m_allocate_storage.f90
371 
372  public :: mg_allocate_storage
373  public :: mg_deallocate_storage
374 
375  !! File ../src/m_diffusion.f90
376 
377 !> Module for implicitly solving diffusion equations
378 
379  public :: diffusion_solve
380  public :: diffusion_solve_vcoeff
381  public :: diffusion_solve_acoeff
382 
383  !! File ../src/m_laplacian.f90
384 
385 !> Module which contains multigrid procedures for a Laplacian operator
386 
387  public :: laplacian_set_methods
388 
389  !! File ../src/m_vhelmholtz.f90
390 
391 !> Module which contains multigrid procedures for a Helmholtz operator of the
392 !> form: div(D grad(phi)) - lambda*phi = f, where D has a smooth spatial
393 !> variation
394 
395  !> The lambda used for the Helmholtz equation (should be >= 0)
396  real(dp), public, protected :: vhelmholtz_lambda = 0.0_dp
397 
398  public :: vhelmholtz_set_methods
399  public :: vhelmholtz_set_lambda
400 
401  !! File ../src/m_build_tree.f90
402 
403  ! Public methods
404  public :: mg_build_rectangle
405  public :: mg_add_children
406  public :: mg_set_leaves_parents
407  public :: mg_set_next_level_ids
409  public :: mg_set_neighbors_lvl
410 
411  !! File ../src/m_load_balance.f90
412 !> Module for load balancing a tree (that already has been constructed). The
413 !> load balancing determines which ranks (MPI processes) allocated physical
414 !> storage for boxes. The tree structure itself is present on all processes.
415 
416  ! Public methods
417  public :: mg_load_balance_simple
418  public :: mg_load_balance
419  public :: mg_load_balance_parents
420 
421  !! File ../src/m_vlaplacian.f90
422 
423 !> Module which contains multigrid procedures for a variable-coefficient
424 !> Laplacian operator, assuming the variation is smooth
425 
426  public :: vlaplacian_set_methods
427 
428  !! File ../src/m_communication.f90
429 
430  ! Public methods
431  public :: mg_comm_init
432  public :: sort_and_transfer_buffers
433 
434  !! File ../src/m_ghost_cells.f90
435 
436  ! Public methods
437  public :: mg_ghost_cell_buffer_size
438  public :: mg_fill_ghost_cells
439  public :: mg_fill_ghost_cells_lvl
440  public :: mg_phi_bc_store
441 
442  !! File ../src/m_prolong.f90
443 
444  ! Public methods
445  public :: mg_prolong
446  public :: mg_prolong_buffer_size
447  public :: mg_prolong_sparse
448 
449  !! File ../src/m_helmholtz.f90
450 
451 !> Module which contains multigrid procedures for a Helmholtz operator of the
452 !> form: laplacian(phi) - lambda*phi = f
453 
454  !> The lambda used for the Helmholtz equation (should be >= 0)
455  real(dp), public, protected :: helmholtz_lambda = 0.0_dp
456 
457  public :: helmholtz_set_methods
458  public :: helmholtz_set_lambda
459 
460  !! File ../src/m_multigrid.f90
461 
462  integer :: timer_total_vcycle = -1
463  integer :: timer_total_fmg = -1
464  integer :: timer_smoother = -1
465  integer :: timer_smoother_gc = -1
466  integer :: timer_coarse = -1
467  integer :: timer_correct = -1
468  integer :: timer_update_coarse = -1
469 
470  ! Public methods
471  public :: mg_fas_vcycle
472  public :: mg_fas_fmg
473  public :: mg_set_methods
474  public :: mg_apply_op
475 
476  !! File ../src/m_restrict.f90
477 
478  ! Public methods
479  public :: mg_restrict
480  public :: mg_restrict_lvl
481  public :: mg_restrict_buffer_size
482 
483  !! File ../src/m_mrgrnk.f90
484 
485  Integer, Parameter :: kdp = selected_real_kind(15)
486  public :: mrgrnk
487  private :: kdp
488  private :: i_mrgrnk
489  interface mrgrnk
490  module procedure i_mrgrnk
491  end interface mrgrnk
492 
493  !! File ../src/m_ahelmholtz.f90
494 
495 !> Module which contains multigrid procedures for a Helmholtz operator of the
496 !> form: div(D grad(phi)) - lambda*phi = f, where D has a smooth spatial
497 !> variation and a component in each spatial direction
498 
499  !> The lambda used for the Helmholtz equation (should be >= 0)
500  real(dp), public, protected :: ahelmholtz_lambda = 0.0_dp
501 
502  public :: ahelmholtz_set_methods
503  public :: ahelmholtz_set_lambda
504 contains
505 
506  !! File ../src/m_data_structures.f90
507 
508  !> Return .true. if a box has children
509  elemental logical function mg_has_children(box)
510  type(mg_box_t), intent(in) :: box
511 
512  ! Boxes are either fully refined or not, so we only need to check one of the
513  ! children
514  mg_has_children = (box%children(1) /= mg_no_box)
515  end function mg_has_children
516 
517  !> Compute the child index for a box with spatial index ix. With child index
518  !> we mean the index in the children(:) array of its parent.
519  integer function mg_ix_to_ichild(ix)
520  integer, intent(in) :: ix(3) !< Spatial index of the box
521  ! The index can range from 1 (all ix odd) and 2**$D (all ix even)
522  mg_ix_to_ichild = 8 - 4 * iand(ix(3), 1) - &
523  2 * iand(ix(2), 1) - iand(ix(1), 1)
524  end function mg_ix_to_ichild
525 
526  !> Get the offset of a box with respect to its parent (e.g. in 2d, there can
527  !> be a child at offset 0,0, one at n_cell/2,0, one at 0,n_cell/2 and one at
528  !> n_cell/2, n_cell/2)
529  pure function mg_get_child_offset(mg, id) result(ix_offset)
530  type(mg_t), intent(in) :: mg
531  integer, intent(in) :: id
532  integer :: ix_offset(3)
533 
534  if (mg%boxes(id)%lvl <= mg%first_normal_lvl) then
535  ix_offset(:) = 0
536  else
537  ix_offset = iand(mg%boxes(id)%ix-1, 1) * &
538  ishft(mg%box_size, -1) ! * n_cell / 2
539  end if
540  end function mg_get_child_offset
541 
542  pure function mg_highest_uniform_lvl(mg) result(lvl)
543  type(mg_t), intent(in) :: mg
544  integer :: lvl
545 
546  do lvl = mg%first_normal_lvl, mg%highest_lvl-1
547  ! Exit if a grid is partially refined
548  if (size(mg%lvls(lvl)%leaves) /= 0 .and. &
549  size(mg%lvls(lvl)%parents) /= 0) exit
550  end do
551  ! If the loop did not exit, we get lvl equals mg%highest_lvl
552  end function mg_highest_uniform_lvl
553 
554  !> Determine total number of unknowns (on leaves)
555  function mg_number_of_unknowns(mg) result(n_unknowns)
556  type(mg_t), intent(in) :: mg
557  integer :: lvl
558  integer(i8) :: n_unknowns
559 
560  n_unknowns = 0
561  do lvl = mg%first_normal_lvl, mg%highest_lvl
562  n_unknowns = n_unknowns + size(mg%lvls(lvl)%leaves)
563  end do
564  n_unknowns = n_unknowns * int(mg%box_size**3, i8)
565  end function mg_number_of_unknowns
566 
567  !> Get coordinates at the face of a box
568  subroutine mg_get_face_coords(box, nb, nc, x)
569  type(mg_box_t), intent(in) :: box
570  integer, intent(in) :: nb
571  integer, intent(in) :: nc
572  real(dp), intent(out) :: x(nc, nc, 3)
573  integer :: i, j, ixs(3-1)
574  integer :: nb_dim
575  real(dp) :: rmin(3)
576 
577  nb_dim = mg_neighb_dim(nb)
578 
579  ! Determine directions perpendicular to neighbor
580  ixs = [(i, i = 1, 3-1)]
581  ixs(nb_dim:) = ixs(nb_dim:) + 1
582 
583  rmin = box%r_min
584  if (.not. mg_neighb_low(nb)) then
585  rmin(nb_dim) = rmin(nb_dim) + box%dr(nb_dim) * nc
586  end if
587 
588  do j = 1, nc
589  do i = 1, nc
590  x(i, j, :) = rmin
591  x(i, j, ixs) = x(i, j, ixs) + ([i, j] - 0.5d0) * box%dr(ixs)
592  end do
593  end do
594  end subroutine mg_get_face_coords
595 
596  integer function mg_add_timer(mg, name)
597  type(mg_t), intent(inout) :: mg
598  character(len=*), intent(in) :: name
599 
600  mg%n_timers = mg%n_timers + 1
601  mg_add_timer = mg%n_timers
602  mg%timers(mg_add_timer)%name = name
603  end function mg_add_timer
604 
605  subroutine mg_timer_start(timer)
606  use mpi
607  type(mg_timer_t), intent(inout) :: timer
608  timer%t0 = mpi_wtime()
609  end subroutine mg_timer_start
610 
611  subroutine mg_timer_end(timer)
612  use mpi
613  type(mg_timer_t), intent(inout) :: timer
614  timer%t = timer%t + mpi_wtime() - timer%t0
615  end subroutine mg_timer_end
616 
617  subroutine mg_timers_show(mg)
618  use mpi
619  type(mg_t), intent(in) :: mg
620  integer :: n, ierr
621  real(dp) :: tmin(mg%n_timers)
622  real(dp) :: tmax(mg%n_timers)
623 
624  call mpi_reduce(mg%timers(1:mg%n_timers)%t, tmin, mg%n_timers, &
625  mpi_double, mpi_min, 0, mg%comm, ierr)
626  call mpi_reduce(mg%timers(1:mg%n_timers)%t, tmax, mg%n_timers, &
627  mpi_double, mpi_max, 0, mg%comm, ierr)
628 
629  if (mg%my_rank == 0) then
630  write(*, "(A20,2A16)") "name ", "min(s)", "max(s)"
631  do n = 1, mg%n_timers
632  write(*, "(A20,2F16.6)") mg%timers(n)%name, &
633  tmin(n), tmax(n)
634  end do
635  end if
636  end subroutine mg_timers_show
637 
638  !! File ../src/m_allocate_storage.f90
639 
640  !> Deallocate all allocatable arrays
641  subroutine mg_deallocate_storage(mg)
642  type(mg_t), intent(inout) :: mg
643  integer :: lvl
644 
645  if (.not. mg%is_allocated) &
646  error stop "deallocate_storage: tree is not allocated"
647 
648  deallocate(mg%boxes)
649  deallocate(mg%buf)
650 
651  deallocate(mg%comm_restrict%n_send)
652  deallocate(mg%comm_restrict%n_recv)
653 
654  deallocate(mg%comm_prolong%n_send)
655  deallocate(mg%comm_prolong%n_recv)
656 
657  deallocate(mg%comm_ghostcell%n_send)
658  deallocate(mg%comm_ghostcell%n_recv)
659 
660  do lvl = mg%lowest_lvl, mg%highest_lvl
661  deallocate(mg%lvls(lvl)%ids)
662  deallocate(mg%lvls(lvl)%leaves)
663  deallocate(mg%lvls(lvl)%parents)
664  deallocate(mg%lvls(lvl)%ref_bnds)
665  deallocate(mg%lvls(lvl)%my_ids)
666  deallocate(mg%lvls(lvl)%my_leaves)
667  deallocate(mg%lvls(lvl)%my_parents)
668  deallocate(mg%lvls(lvl)%my_ref_bnds)
669  end do
670 
671  mg%is_allocated = .false.
672  mg%n_boxes = 0
673  mg%phi_bc_data_stored = .false.
674  end subroutine mg_deallocate_storage
675 
676  !> Allocate communication buffers and local boxes for a tree that has already
677  !> been created
678  subroutine mg_allocate_storage(mg)
679 
680  type(mg_t), intent(inout) :: mg
681  integer :: i, id, lvl, nc
682  integer :: n_send(0:mg%n_cpu-1, 3)
683  integer :: n_recv(0:mg%n_cpu-1, 3)
684  integer :: dsize(3)
685  integer :: n_in, n_out, n_id
686 
687  if (.not. mg%tree_created) &
688  error stop "allocate_storage: tree is not yet created"
689 
690  if (mg%is_allocated) &
691  error stop "allocate_storage: tree is already allocated"
692 
693  do lvl = mg%lowest_lvl, mg%highest_lvl
694  nc = mg%box_size_lvl(lvl)
695  do i = 1, size(mg%lvls(lvl)%my_ids)
696  id = mg%lvls(lvl)%my_ids(i)
697  allocate(mg%boxes(id)%cc(0:nc+1, 0:nc+1, 0:nc+1, &
698  mg_num_vars + mg%n_extra_vars))
699 
700  ! Set all initial values to zero
701  mg%boxes(id)%cc(:, :, :, :) = 0.0_dp
702  end do
703  end do
704 
705  allocate(mg%buf(0:mg%n_cpu-1))
706 
707  call mg_ghost_cell_buffer_size(mg, n_send(:, 1), &
708  n_recv(:, 1), dsize(1))
709  call mg_restrict_buffer_size(mg, n_send(:, 2), &
710  n_recv(:, 2), dsize(2))
711  call mg_prolong_buffer_size(mg, n_send(:, 3), &
712  n_recv(:, 3), dsize(3))
713 
714  do i = 0, mg%n_cpu-1
715  n_out = maxval(n_send(i, :) * dsize(:))
716  n_in = maxval(n_recv(i, :) * dsize(:))
717  n_id = maxval(n_send(i, :))
718  allocate(mg%buf(i)%send(n_out))
719  allocate(mg%buf(i)%recv(n_in))
720  allocate(mg%buf(i)%ix(n_id))
721  end do
722 
723  mg%is_allocated = .true.
724  end subroutine mg_allocate_storage
725 
726  !! File ../src/m_diffusion.f90
727 
728  !> Solve a diffusion equation implicitly, assuming a constant diffusion
729  !> coefficient. The solution at time t should be stored in mg_iphi, which is
730  !> on output replaced by the solution at time t+dt.
731  subroutine diffusion_solve(mg, dt, diffusion_coeff, order, max_res)
732 
733  type(mg_t), intent(inout) :: mg
734  real(dp), intent(in) :: dt
735  real(dp), intent(in) :: diffusion_coeff
736  integer, intent(in) :: order
737  real(dp), intent(in) :: max_res
738  integer, parameter :: max_its = 10
739  integer :: n
740  real(dp) :: res
741 
742  mg%operator_type = mg_helmholtz
743  call mg_set_methods(mg)
744 
745  select case (order)
746  case (1)
747  call helmholtz_set_lambda(1/(dt * diffusion_coeff))
748  call set_rhs(mg, -1/(dt * diffusion_coeff), 0.0_dp)
749  case (2)
750  call helmholtz_set_lambda(0.0d0)
751  call mg_apply_op(mg, mg_irhs)
752  call helmholtz_set_lambda(2/(dt * diffusion_coeff))
753  call set_rhs(mg, -2/(dt * diffusion_coeff), -1.0_dp)
754  case default
755  error stop "diffusion_solve: order should be 1 or 2"
756  end select
757 
758  ! Start with an FMG cycle
759  call mg_fas_fmg(mg, .true., max_res=res)
760 
761  ! Add V-cycles if necessary
762  do n = 1, max_its
763  if (res <= max_res) exit
764  call mg_fas_vcycle(mg, max_res=res)
765  end do
766 
767  if (n == max_its + 1) then
768  if (mg%my_rank == 0) &
769  print *, "Did you specify boundary conditions correctly?"
770  error stop "diffusion_solve: no convergence"
771  end if
772  end subroutine diffusion_solve
773 
774  !> Solve a diffusion equation implicitly, assuming a variable diffusion
775  !> coefficient which has been stored in mg_iveps (also on coarse grids). The
776  !> solution at time t should be stored in mg_iphi, which is on output replaced
777  !> by the solution at time t+dt.
778  subroutine diffusion_solve_vcoeff(mg, dt, order, max_res)
779 
780  type(mg_t), intent(inout) :: mg
781  real(dp), intent(in) :: dt
782  integer, intent(in) :: order
783  real(dp), intent(in) :: max_res
784  integer, parameter :: max_its = 10
785  integer :: n
786  real(dp) :: res
787 
788  mg%operator_type = mg_vhelmholtz
789  call mg_set_methods(mg)
790 
791  select case (order)
792  case (1)
793  call vhelmholtz_set_lambda(1/dt)
794  call set_rhs(mg, -1/dt, 0.0_dp)
795  case (2)
796  call vhelmholtz_set_lambda(0.0d0)
797  call mg_apply_op(mg, mg_irhs)
798  call vhelmholtz_set_lambda(2/dt)
799  call set_rhs(mg, -2/dt, -1.0_dp)
800  case default
801  error stop "diffusion_solve: order should be 1 or 2"
802  end select
803 
804  ! Start with an FMG cycle
805  call mg_fas_fmg(mg, .true., max_res=res)
806 
807  ! Add V-cycles if necessary
808  do n = 1, max_its
809  if (res <= max_res) exit
810  call mg_fas_vcycle(mg, max_res=res)
811  end do
812 
813  if (n == max_its + 1) then
814  if (mg%my_rank == 0) then
815  print *, "Did you specify boundary conditions correctly?"
816  print *, "Or is the variation in diffusion too large?"
817  end if
818  error stop "diffusion_solve: no convergence"
819  end if
820  end subroutine diffusion_solve_vcoeff
821 
822  !> Solve a diffusion equation implicitly, assuming anisotropic diffusion
823  !> coefficient which has been stored in mg_iveps1, mg_iveps2, mg_iveps3
824  !> (also on coarse grids). The
825  !> solution at time t should be stored in mg_iphi, which is on output replaced
826  !> by the solution at time t+dt.
827  subroutine diffusion_solve_acoeff(mg, dt, order, max_res)
828 
829  type(mg_t), intent(inout) :: mg
830  real(dp), intent(in) :: dt
831  integer, intent(in) :: order
832  real(dp), intent(in) :: max_res
833  integer, parameter :: max_its = 10
834  integer :: n
835  real(dp) :: res
836 
837  mg%operator_type = mg_ahelmholtz
838  call mg_set_methods(mg)
839 
840  select case (order)
841  case (1)
842  call ahelmholtz_set_lambda(1/dt)
843  call set_rhs(mg, -1/dt, 0.0_dp)
844  case (2)
845  call ahelmholtz_set_lambda(0.0d0)
846  call mg_apply_op(mg, mg_irhs)
847  call ahelmholtz_set_lambda(2/dt)
848  call set_rhs(mg, -2/dt, -1.0_dp)
849  case default
850  error stop "diffusion_solve: order should be 1 or 2"
851  end select
852 
853  ! Start with an FMG cycle
854  call mg_fas_fmg(mg, .true., max_res=res)
855 
856  ! Add V-cycles if necessary
857  do n = 1, max_its
858  if (res <= max_res) exit
859  call mg_fas_vcycle(mg, max_res=res)
860  end do
861 
862  if (n == max_its + 1) then
863  if (mg%my_rank == 0) then
864  print *, "Did you specify boundary conditions correctly?"
865  print *, "Or is the variation in diffusion too large?"
866  end if
867  error stop "diffusion_solve: no convergence"
868  end if
869  end subroutine diffusion_solve_acoeff
870 
871  subroutine set_rhs(mg, f1, f2)
872  type(mg_t), intent(inout) :: mg
873  real(dp), intent(in) :: f1, f2
874  integer :: lvl, i, id, nc
875 
876  do lvl = 1, mg%highest_lvl
877  nc = mg%box_size_lvl(lvl)
878  do i = 1, size(mg%lvls(lvl)%my_leaves)
879  id = mg%lvls(lvl)%my_leaves(i)
880  mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, mg_irhs) = &
881  f1 * mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, mg_iphi) + &
882  f2 * mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, mg_irhs)
883  end do
884  end do
885  end subroutine set_rhs
886 
887  !! File ../src/m_laplacian.f90
888 
889  subroutine laplacian_set_methods(mg)
890  type(mg_t), intent(inout) :: mg
891 
892  if (all(mg%periodic)) then
893  ! For a fully periodic Laplacian, remove the mean from the rhs and phi so
894  ! that a unique and periodic solution can be found
895  mg%subtract_mean = .true.
896  end if
897 
898  select case (mg%geometry_type)
899  case (mg_cartesian)
900  mg%box_op => box_lpl
901 
902  select case (mg%smoother_type)
903  ! case (smoother_jacobi)
904  ! mg%box_smoother => box_jacobi_lpl
906  mg%box_smoother => box_gs_lpl
907  case default
908  error stop "laplacian_set_methods: unsupported smoother type"
909  end select
910  case default
911  error stop "laplacian_set_methods: unsupported geometry"
912  end select
913 
914  end subroutine laplacian_set_methods
915 
916  !> Perform Gauss-Seidel relaxation on box for a Laplacian operator
917  subroutine box_gs_lpl(mg, id, nc, redblack_cntr)
918  type(mg_t), intent(inout) :: mg
919  integer, intent(in) :: id
920  integer, intent(in) :: nc
921  integer, intent(in) :: redblack_cntr !< Iteration counter
922  integer :: i, j, k, i0, di
923  real(dp) :: idr2(3), fac
924  logical :: redblack
925  real(dp), parameter :: sixth = 1/6.0_dp
926 
927  idr2 = 1/mg%dr(:, mg%boxes(id)%lvl)**2
928  fac = 0.5_dp / sum(idr2)
929  i0 = 1
930 
931  redblack = (mg%smoother_type == mg_smoother_gsrb)
932  if (redblack) then
933  di = 2
934  else
935  di = 1
936  end if
937 
938  ! The parity of redblack_cntr determines which cells we use. If
939  ! redblack_cntr is even, we use the even cells and vice versa.
940  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
941  do k = 1, nc
942  do j = 1, nc
943  if (redblack) &
944  i0 = 2 - iand(ieor(redblack_cntr, k+j), 1)
945  do i = i0, nc, di
946  cc(i, j, k, n) = fac * ( &
947  idr2(1) * (cc(i+1, j, k, n) + cc(i-1, j, k, n)) + &
948  idr2(2) * (cc(i, j+1, k, n) + cc(i, j-1, k, n)) + &
949  idr2(3) * (cc(i, j, k+1, n) + cc(i, j, k-1, n)) - &
950  cc(i, j, k, mg_irhs))
951  end do
952  end do
953  end do
954  end associate
955  end subroutine box_gs_lpl
956 
957 ! !> Perform Jacobi relaxation on box for a Laplacian operator
958 ! subroutine box_jacobi_lpl(mg, id, nc, cntr)
959 ! type(mg_t), intent(inout) :: mg
960 ! integer, intent(in) :: id
961 ! integer, intent(in) :: nc
962 ! integer, intent(in) :: cntr !< Not used
963 ! integer :: i, j, k
964 ! real(dp), parameter :: w = 2.0_dp / 3
965 ! real(dp) :: tmp(0:nc+1, 0:nc+1, 0:nc+1)
966 ! real(dp) :: dr2
967 ! #if 3 == 3
968 ! real(dp), parameter :: sixth = 1/6.0_dp
969 ! #endif
970 
971 ! dr2 = mg%dr(mg%boxes(id)%lvl)**2
972 
973 ! associate (box => mg%boxes(id))
974 ! tmp = box%cc(:, :, :, mg_iphi)
975 ! do j=1, nc; do i=1, nc; do k=1, nc
976 ! #if 3 == 2
977 ! box%cc(i, j, mg_iphi) = (1-w) * box%cc(i, j, mg_iphi) + &
978 ! 0.25_dp * w * ( &
979 ! tmp(i+1, j) + tmp(i-1, j) + &
980 ! tmp(i, j+1) + tmp(i, j-1) - &
981 ! dr2 * box%cc(i, j, mg_irhs))
982 ! #elif 3 == 3
983 ! box%cc(i, j, k, mg_iphi) = (1-w) * &
984 ! box%cc(i, j, k, mg_iphi) + &
985 ! sixth * w * ( &
986 ! tmp(i+1, j, k) + tmp(i-1, j, k) + &
987 ! tmp(i, j+1, k) + tmp(i, j-1, k) + &
988 ! tmp(i, j, k+1) + tmp(i, j, k-1) - &
989 ! dr2 * box%cc(i, j, k, mg_irhs))
990 ! #endif
991 ! end do; end do; end do
992 ! end associate
993 ! end subroutine box_jacobi_lpl
994 
995  !> Perform Laplacian operator on a box
996  subroutine box_lpl(mg, id, nc, i_out)
997  type(mg_t), intent(inout) :: mg
998  integer, intent(in) :: id
999  integer, intent(in) :: nc
1000  integer, intent(in) :: i_out !< Index of variable to store Laplacian in
1001  integer :: i, j, k
1002  real(dp) :: idr2(3)
1003 
1004  idr2 = 1 / mg%dr(:, mg%boxes(id)%lvl)**2
1005 
1006  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
1007  do k = 1, nc
1008  do j = 1, nc
1009  do i = 1, nc
1010  cc(i, j, k, i_out) = &
1011  idr2(1) * (cc(i-1, j, k, n) + cc(i+1, j, k, n) &
1012  - 2 * cc(i, j, k, n)) &
1013  + idr2(2) * (cc(i, j-1, k, n) + cc(i, j+1, k, n) &
1014  - 2 * cc(i, j, k, n)) &
1015  + idr2(3) * (cc(i, j, k-1, n) + cc(i, j, k+1, n) &
1016  - 2 * cc(i, j, k, n))
1017  end do
1018  end do
1019  end do
1020  end associate
1021  end subroutine box_lpl
1022 
1023 
1024  !! File ../src/m_vhelmholtz.f90
1025 
1026  subroutine vhelmholtz_set_methods(mg)
1027  type(mg_t), intent(inout) :: mg
1028 
1029  if (mg%n_extra_vars == 0 .and. mg%is_allocated) then
1030  error stop "vhelmholtz_set_methods: mg%n_extra_vars == 0"
1031  else
1032  mg%n_extra_vars = max(1, mg%n_extra_vars)
1033  end if
1034 
1035  mg%subtract_mean = .false.
1036 
1037  ! Use Neumann zero boundary conditions for the variable coefficient, since
1038  ! it is needed in ghost cells.
1039  mg%bc(:, mg_iveps)%bc_type = mg_bc_neumann
1040  mg%bc(:, mg_iveps)%bc_value = 0.0_dp
1041 
1042  select case (mg%geometry_type)
1043  case (mg_cartesian)
1044  mg%box_op => box_vhelmh
1045 
1046  select case (mg%smoother_type)
1048  mg%box_smoother => box_gs_vhelmh
1049  case default
1050  error stop "vhelmholtz_set_methods: unsupported smoother type"
1051  end select
1052  case default
1053  error stop "vhelmholtz_set_methods: unsupported geometry"
1054  end select
1055 
1056  end subroutine vhelmholtz_set_methods
1057 
1058  subroutine vhelmholtz_set_lambda(lambda)
1059  real(dp), intent(in) :: lambda
1060 
1061  if (lambda < 0) &
1062  error stop "vhelmholtz_set_lambda: lambda < 0 not allowed"
1063 
1064  vhelmholtz_lambda = lambda
1065  end subroutine vhelmholtz_set_lambda
1066 
1067  !> Perform Gauss-Seidel relaxation on box for a Helmholtz operator
1068  subroutine box_gs_vhelmh(mg, id, nc, redblack_cntr)
1069  type(mg_t), intent(inout) :: mg
1070  integer, intent(in) :: id
1071  integer, intent(in) :: nc
1072  integer, intent(in) :: redblack_cntr !< Iteration counter
1073  integer :: i, j, k, i0, di
1074  logical :: redblack
1075  real(dp) :: idr2(2*3), u(2*3)
1076  real(dp) :: a0, a(2*3), c(2*3)
1077 
1078  ! Duplicate 1/dr^2 array to multiply neighbor values
1079  idr2(1:2*3:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1080  idr2(2:2*3:2) = idr2(1:2*3:2)
1081  i0 = 1
1082 
1083  redblack = (mg%smoother_type == mg_smoother_gsrb)
1084  if (redblack) then
1085  di = 2
1086  else
1087  di = 1
1088  end if
1089 
1090  ! The parity of redblack_cntr determines which cells we use. If
1091  ! redblack_cntr is even, we use the even cells and vice versa.
1092  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1093  i_eps => mg_iveps)
1094  do k = 1, nc
1095  do j = 1, nc
1096  if (redblack) &
1097  i0 = 2 - iand(ieor(redblack_cntr, k+j), 1)
1098  do i = i0, nc, di
1099  a0 = cc(i, j, k, i_eps)
1100  u(1:2) = cc(i-1:i+1:2, j, k, n)
1101  a(1:2) = cc(i-1:i+1:2, j, k, i_eps)
1102  u(3:4) = cc(i, j-1:j+1:2, k, n)
1103  a(3:4) = cc(i, j-1:j+1:2, k, i_eps)
1104  u(5:6) = cc(i, j, k-1:k+1:2, n)
1105  a(5:6) = cc(i, j, k-1:k+1:2, i_eps)
1106  c(:) = 2 * a0 * a(:) / (a0 + a(:)) * idr2
1107 
1108  cc(i, j, k, n) = (sum(c(:) * u(:)) - &
1109  cc(i, j, k, mg_irhs)) / &
1110  (sum(c(:)) + vhelmholtz_lambda)
1111  end do
1112  end do
1113  end do
1114  end associate
1115  end subroutine box_gs_vhelmh
1116 
1117  !> Perform Helmholtz operator on a box
1118  subroutine box_vhelmh(mg, id, nc, i_out)
1119  type(mg_t), intent(inout) :: mg
1120  integer, intent(in) :: id
1121  integer, intent(in) :: nc
1122  integer, intent(in) :: i_out !< Index of variable to store Helmholtz in
1123  integer :: i, j, k
1124  real(dp) :: idr2(2*3), a0, u0, u(2*3), a(2*3)
1125 
1126  ! Duplicate 1/dr^2 array to multiply neighbor values
1127  idr2(1:2*3:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1128  idr2(2:2*3:2) = idr2(1:2*3:2)
1129 
1130  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1131  i_eps => mg_iveps)
1132  do k = 1, nc
1133  do j = 1, nc
1134  do i = 1, nc
1135  u0 = cc(i, j, k, n)
1136  a0 = cc(i, j, k, i_eps)
1137  u(1:2) = cc(i-1:i+1:2, j, k, n)
1138  u(3:4) = cc(i, j-1:j+1:2, k, n)
1139  u(5:6) = cc(i, j, k-1:k+1:2, n)
1140  a(1:2) = cc(i-1:i+1:2, j, k, i_eps)
1141  a(3:4) = cc(i, j-1:j+1:2, k, i_eps)
1142  a(5:6) = cc(i, j, k-1:k+1:2, i_eps)
1143 
1144  cc(i, j, k, i_out) = sum(2 * idr2 * &
1145  a0*a(:)/(a0 + a(:)) * (u(:) - u0)) - &
1146  vhelmholtz_lambda * u0
1147  end do
1148  end do
1149  end do
1150  end associate
1151  end subroutine box_vhelmh
1152 
1153  !! File ../src/m_build_tree.f90
1154 
1155  subroutine mg_build_rectangle(mg, domain_size, box_size, dx, r_min, &
1156  periodic, n_finer)
1157  type(mg_t), intent(inout) :: mg
1158  integer, intent(in) :: domain_size(3)
1159  integer, intent(in) :: box_size
1160  real(dp), intent(in) :: dx(3)
1161  real(dp), intent(in) :: r_min(3)
1162  logical, intent(in) :: periodic(3)
1163  integer, intent(in) :: n_finer
1164  integer :: i, j, k, lvl, n, id, nx(3)
1165  integer :: boxes_per_dim(3, mg_lvl_lo:1)
1166  integer :: periodic_offset(3)
1167 
1168  if (modulo(box_size, 2) /= 0) &
1169  error stop "box_size should be even"
1170  if (any(modulo(domain_size, box_size) /= 0)) &
1171  error stop "box_size does not divide domain_size"
1172 
1173  if (all(periodic)) then
1174  mg%subtract_mean = .true.
1175  end if
1176 
1177  nx = domain_size
1178  mg%box_size = box_size
1179  mg%box_size_lvl(1) = box_size
1180  mg%domain_size_lvl(:, 1) = domain_size
1181  mg%first_normal_lvl = 1
1182  mg%dr(:, 1) = dx
1183  mg%r_min(:) = r_min
1184  mg%periodic = periodic
1185  boxes_per_dim(:, :) = 0
1186  boxes_per_dim(:, 1) = domain_size / box_size
1187 
1188  do lvl = 1, mg_lvl_lo+1, -1
1189  ! For a Gauss-Seidel (non red-black) smoother, we should avoid boxes
1190  ! containing a single cell
1191  if (any(modulo(nx, 2) == 1 .or. nx == mg%coarsest_grid .or. &
1192  (mg%box_size_lvl(lvl) == mg%coarsest_grid .and. &
1193  mg%smoother_type == mg_smoother_gs))) exit
1194 
1195  if (all(modulo(nx/mg%box_size_lvl(lvl), 2) == 0)) then
1196  mg%box_size_lvl(lvl-1) = mg%box_size_lvl(lvl)
1197  boxes_per_dim(:, lvl-1) = boxes_per_dim(:, lvl)/2
1198  mg%first_normal_lvl = lvl-1
1199  else
1200  mg%box_size_lvl(lvl-1) = mg%box_size_lvl(lvl)/2
1201  boxes_per_dim(:, lvl-1) = boxes_per_dim(:, lvl)
1202  end if
1203 
1204  mg%dr(:, lvl-1) = mg%dr(:, lvl) * 2
1205  nx = nx / 2
1206  mg%domain_size_lvl(:, lvl-1) = nx
1207  end do
1208 
1209  mg%lowest_lvl = lvl
1210  mg%highest_lvl = 1
1211 
1212  do lvl = 2, mg_lvl_hi
1213  mg%dr(:, lvl) = mg%dr(:, lvl-1) * 0.5_dp
1214  mg%box_size_lvl(lvl) = box_size
1215  mg%domain_size_lvl(:, lvl) = 2 * mg%domain_size_lvl(:, lvl-1)
1216  end do
1217 
1218  n = sum(product(boxes_per_dim, dim=1)) + n_finer
1219  allocate(mg%boxes(n))
1220 
1221  ! Create lowest level
1222  nx = boxes_per_dim(:, mg%lowest_lvl)
1223  periodic_offset = [nx(1)-1, (nx(2)-1)*nx(1), &
1224  (nx(3)-1) * nx(2) * nx(1)]
1225 
1226  do k=1,nx(3); do j=1,nx(2); do i=1,nx(1)
1227  mg%n_boxes = mg%n_boxes + 1
1228  n = mg%n_boxes
1229 
1230  mg%boxes(n)%rank = 0
1231  mg%boxes(n)%id = n
1232  mg%boxes(n)%lvl = mg%lowest_lvl
1233  mg%boxes(n)%ix(:) = [i, j, k]
1234  mg%boxes(n)%r_min(:) = r_min + (mg%boxes(n)%ix(:) - 1) * &
1235  mg%box_size_lvl(mg%lowest_lvl) * mg%dr(:, mg%lowest_lvl)
1236  mg%boxes(n)%dr(:) = mg%dr(:, mg%lowest_lvl)
1237  mg%boxes(n)%parent = mg_no_box
1238  mg%boxes(n)%children(:) = mg_no_box
1239 
1240  ! Set default neighbors
1241  mg%boxes(n)%neighbors(:) = [n-1, n+1, n-nx(1), n+nx(1), &
1242  n-nx(1)*nx(2), n+nx(1)*nx(2)]
1243 
1244  ! Handle boundaries
1245  where ([i, j, k] == 1 .and. .not. periodic)
1246  mg%boxes(n)%neighbors(1:mg_num_neighbors:2) = &
1248  end where
1249  where ([i, j, k] == 1 .and. periodic)
1250  mg%boxes(n)%neighbors(1:mg_num_neighbors:2) = &
1251  n + periodic_offset
1252  end where
1253 
1254  where ([i, j, k] == nx .and. .not. periodic)
1255  mg%boxes(n)%neighbors(2:mg_num_neighbors:2) = &
1257  end where
1258  where ([i, j, k] == nx .and. periodic)
1259  mg%boxes(n)%neighbors(2:mg_num_neighbors:2) = &
1260  n - periodic_offset
1261  end where
1262  end do; end do; end do
1263 
1264  mg%lvls(mg%lowest_lvl)%ids = [(n, n=1, mg%n_boxes)]
1265 
1266  ! Add higher levels
1267  do lvl = mg%lowest_lvl, 0
1268  if (mg%box_size_lvl(lvl+1) == mg%box_size_lvl(lvl)) then
1269  do i = 1, size(mg%lvls(lvl)%ids)
1270  id = mg%lvls(lvl)%ids(i)
1271  call mg_add_children(mg, id)
1272  end do
1273 
1274  call mg_set_leaves_parents(mg%boxes, mg%lvls(lvl))
1275  call mg_set_next_level_ids(mg, lvl)
1276  call mg_set_neighbors_lvl(mg, lvl+1)
1277  else
1278  do i = 1, size(mg%lvls(lvl)%ids)
1279  id = mg%lvls(lvl)%ids(i)
1280  call add_single_child(mg, id, size(mg%lvls(lvl)%ids))
1281  end do
1282 
1283  call mg_set_leaves_parents(mg%boxes, mg%lvls(lvl))
1284  call mg_set_next_level_ids(mg, lvl)
1285  end if
1286  end do
1287 
1288  call mg_set_leaves_parents(mg%boxes, mg%lvls(1))
1289 
1290  ! No refinement boundaries
1291  do lvl = mg%lowest_lvl, 1
1292  if (allocated(mg%lvls(lvl)%ref_bnds)) &
1293  deallocate(mg%lvls(lvl)%ref_bnds)
1294  allocate(mg%lvls(lvl)%ref_bnds(0))
1295  end do
1296 
1297  mg%tree_created = .true.
1298  end subroutine mg_build_rectangle
1299 
1300  subroutine mg_set_neighbors_lvl(mg, lvl)
1301  type(mg_t), intent(inout) :: mg
1302  integer, intent(in) :: lvl
1303  integer :: i, id
1304 
1305  do i = 1, size(mg%lvls(lvl)%ids)
1306  id = mg%lvls(lvl)%ids(i)
1307  call set_neighbs(mg%boxes, id)
1308  end do
1309  end subroutine mg_set_neighbors_lvl
1310 
1311  subroutine mg_set_next_level_ids(mg, lvl)
1312  type(mg_t), intent(inout) :: mg
1313  integer, intent(in) :: lvl
1314  integer :: n, i, id
1315 
1316  if (allocated(mg%lvls(lvl+1)%ids)) &
1317  deallocate(mg%lvls(lvl+1)%ids)
1318 
1319  ! Set next level ids to children of this level
1320  if (mg%box_size_lvl(lvl+1) == mg%box_size_lvl(lvl)) then
1321  n = mg_num_children * size(mg%lvls(lvl)%parents)
1322  allocate(mg%lvls(lvl+1)%ids(n))
1323 
1324  n = mg_num_children
1325  do i = 1, size(mg%lvls(lvl)%parents)
1326  id = mg%lvls(lvl)%parents(i)
1327  mg%lvls(lvl+1)%ids(n*(i-1)+1:n*i) = mg%boxes(id)%children
1328  end do
1329  else
1330  n = size(mg%lvls(lvl)%parents)
1331  allocate(mg%lvls(lvl+1)%ids(n))
1332 
1333  n = 1
1334  do i = 1, size(mg%lvls(lvl)%parents)
1335  id = mg%lvls(lvl)%parents(i)
1336  mg%lvls(lvl+1)%ids(i) = mg%boxes(id)%children(1)
1337  end do
1338  end if
1339 
1340  end subroutine mg_set_next_level_ids
1341 
1342  ! Set the neighbors of id (using their parent)
1343  subroutine set_neighbs(boxes, id)
1344  type(mg_box_t), intent(inout) :: boxes(:)
1345  integer, intent(in) :: id
1346  integer :: nb, nb_id
1347 
1348  do nb = 1, mg_num_neighbors
1349  if (boxes(id)%neighbors(nb) == mg_no_box) then
1350  nb_id = find_neighb(boxes, id, nb)
1351  if (nb_id > mg_no_box) then
1352  boxes(id)%neighbors(nb) = nb_id
1353  boxes(nb_id)%neighbors(mg_neighb_rev(nb)) = id
1354  end if
1355  end if
1356  end do
1357  end subroutine set_neighbs
1358 
1359  !> Get the id of neighbor nb of boxes(id), through its parent
1360  function find_neighb(boxes, id, nb) result(nb_id)
1361  type(mg_box_t), intent(in) :: boxes(:) !< List with all the boxes
1362  integer, intent(in) :: id !< Box whose neighbor we are looking for
1363  integer, intent(in) :: nb !< Neighbor index
1364  integer :: nb_id, p_id, c_ix, d, old_pid
1365 
1366  p_id = boxes(id)%parent
1367  old_pid = p_id
1368  c_ix = mg_ix_to_ichild(boxes(id)%ix)
1369  d = mg_neighb_dim(nb)
1370 
1371  ! Check if neighbor is in same direction as ix is (low/high). If so,
1372  ! use neighbor of parent
1373  if (mg_child_low(d, c_ix) .eqv. mg_neighb_low(nb)) then
1374  p_id = boxes(p_id)%neighbors(nb)
1375  end if
1376 
1377  ! The child ix of the neighbor is reversed in direction d
1378  nb_id = boxes(p_id)%children(mg_child_rev(c_ix, d))
1379  end function find_neighb
1380 
1381  !> Create a list of leaves and a list of parents for a level
1382  subroutine mg_set_leaves_parents(boxes, level)
1383  type(mg_box_t), intent(in) :: boxes(:) !< List of boxes
1384  type(mg_lvl_t), intent(inout) :: level !< Level type which contains the indices of boxes
1385  integer :: i, id, i_leaf, i_parent
1386  integer :: n_parents, n_leaves
1387 
1388  n_parents = count(mg_has_children(boxes(level%ids)))
1389  n_leaves = size(level%ids) - n_parents
1390 
1391  if (.not. allocated(level%parents)) then
1392  allocate(level%parents(n_parents))
1393  else if (n_parents /= size(level%parents)) then
1394  deallocate(level%parents)
1395  allocate(level%parents(n_parents))
1396  end if
1397 
1398  if (.not. allocated(level%leaves)) then
1399  allocate(level%leaves(n_leaves))
1400  else if (n_leaves /= size(level%leaves)) then
1401  deallocate(level%leaves)
1402  allocate(level%leaves(n_leaves))
1403  end if
1404 
1405  i_leaf = 0
1406  i_parent = 0
1407  do i = 1, size(level%ids)
1408  id = level%ids(i)
1409  if (mg_has_children(boxes(id))) then
1410  i_parent = i_parent + 1
1411  level%parents(i_parent) = id
1412  else
1413  i_leaf = i_leaf + 1
1414  level%leaves(i_leaf) = id
1415  end if
1416  end do
1417  end subroutine mg_set_leaves_parents
1418 
1419  !> Create a list of refinement boundaries (from the coarse side)
1420  subroutine mg_set_refinement_boundaries(boxes, level)
1421  type(mg_box_t), intent(in) :: boxes(:)
1422  type(mg_lvl_t), intent(inout) :: level
1423  integer, allocatable :: tmp(:)
1424  integer :: i, id, nb, nb_id, ix
1425 
1426  if (allocated(level%ref_bnds)) deallocate(level%ref_bnds)
1427 
1428  if (size(level%parents) == 0) then
1429  ! There are no refinement boundaries
1430  allocate(level%ref_bnds(0))
1431  else
1432  allocate(tmp(size(level%leaves)))
1433  ix = 0
1434  do i = 1, size(level%leaves)
1435  id = level%leaves(i)
1436 
1437  do nb = 1, mg_num_neighbors
1438  nb_id = boxes(id)%neighbors(nb)
1439  if (nb_id > mg_no_box) then
1440  if (mg_has_children(boxes(nb_id))) then
1441  ix = ix + 1
1442  tmp(ix) = id
1443  exit
1444  end if
1445  end if
1446  end do
1447  end do
1448 
1449  allocate(level%ref_bnds(ix))
1450  level%ref_bnds(:) = tmp(1:ix)
1451  end if
1452  end subroutine mg_set_refinement_boundaries
1453 
1454  subroutine mg_add_children(mg, id)
1455  type(mg_t), intent(inout) :: mg
1456  integer, intent(in) :: id !< Id of box that gets children
1457  integer :: lvl, i, nb, child_nb(2**(3-1))
1458  integer :: c_ids(mg_num_children)
1459  integer :: c_id, c_ix_base(3)
1460 
1461  if (mg%n_boxes + mg_num_children > size(mg%boxes)) then
1462  error stop "mg_add_children: not enough space"
1463  end if
1464 
1465  c_ids = [(mg%n_boxes+i, i=1,mg_num_children)]
1466  mg%n_boxes = mg%n_boxes + mg_num_children
1467  mg%boxes(id)%children = c_ids
1468  c_ix_base = 2 * mg%boxes(id)%ix - 1
1469  lvl = mg%boxes(id)%lvl+1
1470 
1471  do i = 1, mg_num_children
1472  c_id = c_ids(i)
1473  mg%boxes(c_id)%rank = mg%boxes(id)%rank
1474  mg%boxes(c_id)%ix = c_ix_base + mg_child_dix(:, i)
1475  mg%boxes(c_id)%lvl = lvl
1476  mg%boxes(c_id)%parent = id
1477  mg%boxes(c_id)%children = mg_no_box
1478  mg%boxes(c_id)%neighbors = mg_no_box
1479  mg%boxes(c_id)%r_min = mg%boxes(id)%r_min + &
1480  mg%dr(:, lvl) * mg_child_dix(:, i) * mg%box_size
1481  mg%boxes(c_id)%dr(:) = mg%dr(:, lvl)
1482  end do
1483 
1484  ! Set boundary conditions at children
1485  do nb = 1, mg_num_neighbors
1486  if (mg%boxes(id)%neighbors(nb) < mg_no_box) then
1487  child_nb = c_ids(mg_child_adj_nb(:, nb)) ! Neighboring children
1488  mg%boxes(child_nb)%neighbors(nb) = mg%boxes(id)%neighbors(nb)
1489  end if
1490  end do
1491  end subroutine mg_add_children
1492 
1493  subroutine add_single_child(mg, id, n_boxes_lvl)
1494  type(mg_t), intent(inout) :: mg
1495  integer, intent(in) :: id !< Id of box that gets children
1496  integer, intent(in) :: n_boxes_lvl
1497  integer :: lvl, c_id
1498 
1499  c_id = mg%n_boxes + 1
1500  mg%n_boxes = mg%n_boxes + 1
1501  mg%boxes(id)%children(1) = c_id
1502  lvl = mg%boxes(id)%lvl+1
1503 
1504  mg%boxes(c_id)%rank = mg%boxes(id)%rank
1505  mg%boxes(c_id)%ix = mg%boxes(id)%ix
1506  mg%boxes(c_id)%lvl = lvl
1507  mg%boxes(c_id)%parent = id
1508  mg%boxes(c_id)%children = mg_no_box
1509  where (mg%boxes(id)%neighbors == mg_physical_boundary)
1510  mg%boxes(c_id)%neighbors = mg%boxes(id)%neighbors
1511  elsewhere
1512  mg%boxes(c_id)%neighbors = mg%boxes(id)%neighbors + n_boxes_lvl
1513  end where
1514  mg%boxes(c_id)%r_min = mg%boxes(id)%r_min
1515  mg%boxes(c_id)%dr(:) = mg%dr(:, lvl)
1516 
1517  end subroutine add_single_child
1518 
1519  !! File ../src/m_load_balance.f90
1520 
1521  !> Load balance all boxes in the multigrid tree, by simply distributing the
1522  !> load per grid level. This method will only work well for uniform grids.
1523  !>
1524  !> Note that in a typical application the load balancing of the leaves is
1525  !> already determined, then mg_load_balance_parents can be used.
1526  subroutine mg_load_balance_simple(mg)
1527  type(mg_t), intent(inout) :: mg
1528  integer :: i, id, lvl, single_cpu_lvl
1529  integer :: work_left, my_work, i_cpu
1530 
1531  ! Up to this level, all boxes have to be on a single processor because they
1532  ! have a different size and the communication routines do not support this
1533  single_cpu_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
1534 
1535  do lvl = mg%lowest_lvl, single_cpu_lvl
1536  do i = 1, size(mg%lvls(lvl)%ids)
1537  id = mg%lvls(lvl)%ids(i)
1538  mg%boxes(id)%rank = 0
1539  end do
1540  end do
1541 
1542  ! Distribute the boxes equally. Due to the way the mesh is constructed, the
1543  ! mg%lvls(lvl)%ids array already contains a Morton-like ordering.
1544  do lvl = single_cpu_lvl+1, mg%highest_lvl
1545  work_left = size(mg%lvls(lvl)%ids)
1546  my_work = 0
1547  i_cpu = 0
1548 
1549  do i = 1, size(mg%lvls(lvl)%ids)
1550  if ((mg%n_cpu - i_cpu - 1) * my_work >= work_left) then
1551  i_cpu = i_cpu + 1
1552  my_work = 0
1553  end if
1554 
1555  my_work = my_work + 1
1556  work_left = work_left - 1
1557 
1558  id = mg%lvls(lvl)%ids(i)
1559  mg%boxes(id)%rank = i_cpu
1560  end do
1561  end do
1562 
1563  do lvl = mg%lowest_lvl, mg%highest_lvl
1564  call update_lvl_info(mg, mg%lvls(lvl))
1565  end do
1566 
1567  end subroutine mg_load_balance_simple
1568 
1569  !> Load balance all boxes in the multigrid tree. Compared to
1570  !> mg_load_balance_simple, this method does a better job of setting the ranks
1571  !> of parent boxes
1572  !>
1573  !> Note that in a typical application the load balancing of the leaves is
1574  !> already determined, then mg_load_balance_parents can be used.
1575  subroutine mg_load_balance(mg)
1576  type(mg_t), intent(inout) :: mg
1577  integer :: i, id, lvl, single_cpu_lvl
1578  integer :: work_left, my_work(0:mg%n_cpu), i_cpu
1579  integer :: c_ids(mg_num_children)
1580  integer :: c_ranks(mg_num_children)
1581  integer :: coarse_rank
1582 
1583  ! Up to this level, all boxes have to be on a single processor because they
1584  ! have a different size and the communication routines do not support this
1585  single_cpu_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
1586 
1587  ! Distribute the boxes equally. Due to the way the mesh is constructed, the
1588  ! mg%lvls(lvl)%ids array already contains a Morton-like ordering.
1589  do lvl = mg%highest_lvl, single_cpu_lvl+1, -1
1590  ! For parents determine the rank based on their child ranks
1591  my_work(:) = 0
1592 
1593  do i = 1, size(mg%lvls(lvl)%parents)
1594  id = mg%lvls(lvl)%parents(i)
1595 
1596  c_ids = mg%boxes(id)%children
1597  c_ranks = mg%boxes(c_ids)%rank
1598  i_cpu = most_popular(c_ranks, my_work, mg%n_cpu)
1599  mg%boxes(id)%rank = i_cpu
1600  my_work(i_cpu) = my_work(i_cpu) + 1
1601  end do
1602 
1603  work_left = size(mg%lvls(lvl)%leaves)
1604  i_cpu = 0
1605 
1606  do i = 1, size(mg%lvls(lvl)%leaves)
1607  ! Skip this CPU if it already has enough work
1608  if ((mg%n_cpu - i_cpu - 1) * my_work(i_cpu) >= &
1609  work_left + sum(my_work(i_cpu+1:))) then
1610  i_cpu = i_cpu + 1
1611  end if
1612 
1613  my_work(i_cpu) = my_work(i_cpu) + 1
1614  work_left = work_left - 1
1615 
1616  id = mg%lvls(lvl)%leaves(i)
1617  mg%boxes(id)%rank = i_cpu
1618  end do
1619  end do
1620 
1621  ! Determine most popular CPU for coarse grids
1622  if (single_cpu_lvl < mg%highest_lvl) then
1623  coarse_rank = most_popular(mg%boxes(&
1624  mg%lvls(single_cpu_lvl+1)%ids)%rank, my_work, mg%n_cpu)
1625  else
1626  coarse_rank = 0
1627  end if
1628 
1629  do lvl = mg%lowest_lvl, single_cpu_lvl
1630  do i = 1, size(mg%lvls(lvl)%ids)
1631  id = mg%lvls(lvl)%ids(i)
1632  mg%boxes(id)%rank = coarse_rank
1633  end do
1634  end do
1635 
1636  do lvl = mg%lowest_lvl, mg%highest_lvl
1637  call update_lvl_info(mg, mg%lvls(lvl))
1638  end do
1639 
1640  end subroutine mg_load_balance
1641 
1642  !> Load balance the parents (non-leafs). Assign them to the rank that has most
1643  !> children.
1645  type(mg_t), intent(inout) :: mg
1646  integer :: i, id, lvl
1647  integer :: c_ids(mg_num_children)
1648  integer :: c_ranks(mg_num_children)
1649  integer :: single_cpu_lvl, coarse_rank
1650  integer :: my_work(0:mg%n_cpu), i_cpu
1651 
1652  ! Up to this level, all boxes have to be on a single processor because they
1653  ! have a different size and the communication routines do not support this
1654  single_cpu_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
1655 
1656  do lvl = mg%highest_lvl-1, single_cpu_lvl+1, -1
1657  my_work(:) = 0
1658 
1659  ! Determine amount of work for the leaves
1660  do i = 1, size(mg%lvls(lvl)%leaves)
1661  id = mg%lvls(lvl)%leaves(i)
1662  i_cpu = mg%boxes(id)%rank
1663  my_work(i_cpu) = my_work(i_cpu) + 1
1664  end do
1665 
1666  do i = 1, size(mg%lvls(lvl)%parents)
1667  id = mg%lvls(lvl)%parents(i)
1668 
1669  c_ids = mg%boxes(id)%children
1670  c_ranks = mg%boxes(c_ids)%rank
1671  i_cpu = most_popular(c_ranks, my_work, mg%n_cpu)
1672  mg%boxes(id)%rank = i_cpu
1673  my_work(i_cpu) = my_work(i_cpu) + 1
1674  end do
1675 
1676  end do
1677 
1678  ! Determine most popular CPU for coarse grids
1679  if (single_cpu_lvl < mg%highest_lvl) then
1680  coarse_rank = most_popular(mg%boxes(&
1681  mg%lvls(single_cpu_lvl+1)%ids)%rank, my_work, mg%n_cpu)
1682  else
1683  coarse_rank = 0
1684  end if
1685 
1686  do lvl = mg%lowest_lvl, single_cpu_lvl
1687  do i = 1, size(mg%lvls(lvl)%ids)
1688  id = mg%lvls(lvl)%ids(i)
1689  mg%boxes(id)%rank = coarse_rank
1690  end do
1691  end do
1692 
1693  do lvl = mg%lowest_lvl, mg%highest_lvl
1694  call update_lvl_info(mg, mg%lvls(lvl))
1695  end do
1696 
1697  end subroutine mg_load_balance_parents
1698 
1699  !> Determine most popular rank in the list. In case of ties, assign the rank
1700  !> with the least work.
1701  pure integer function most_popular(list, work, n_cpu)
1702  integer, intent(in) :: list(:) !< List of MPI ranks
1703  integer, intent(in) :: n_cpu
1704  integer, intent(in) :: work(0:n_cpu-1) !< Existing work per rank
1705  integer :: i, best_count, current_count
1706  integer :: my_work, best_work
1707 
1708  best_count = 0
1709  best_work = 0
1710  most_popular = -1
1711 
1712  do i = 1, size(list)
1713  current_count = count(list == list(i))
1714  my_work = work(list(i))
1715 
1716  ! In case of ties, select task with lowest work
1717  if (current_count > best_count .or. &
1718  current_count == best_count .and. my_work < best_work) then
1719  best_count = current_count
1720  best_work = my_work
1721  most_popular = list(i)
1722  end if
1723  end do
1724 
1725  end function most_popular
1726 
1727  subroutine update_lvl_info(mg, lvl)
1728  type(mg_t), intent(inout) :: mg
1729  type(mg_lvl_t), intent(inout) :: lvl
1730 
1731  lvl%my_ids = pack(lvl%ids, &
1732  mg%boxes(lvl%ids)%rank == mg%my_rank)
1733  lvl%my_leaves = pack(lvl%leaves, &
1734  mg%boxes(lvl%leaves)%rank == mg%my_rank)
1735  lvl%my_parents = pack(lvl%parents, &
1736  mg%boxes(lvl%parents)%rank == mg%my_rank)
1737  lvl%my_ref_bnds = pack(lvl%ref_bnds, &
1738  mg%boxes(lvl%ref_bnds)%rank == mg%my_rank)
1739  end subroutine update_lvl_info
1740 
1741  !! File ../src/m_vlaplacian.f90
1742 
1743  subroutine vlaplacian_set_methods(mg)
1744  type(mg_t), intent(inout) :: mg
1745 
1746  if (mg%n_extra_vars == 0 .and. mg%is_allocated) then
1747  error stop "vlaplacian_set_methods: mg%n_extra_vars == 0"
1748  else
1749  mg%n_extra_vars = max(1, mg%n_extra_vars)
1750  end if
1751 
1752  mg%subtract_mean = .false.
1753 
1754  ! Use Neumann zero boundary conditions for the variable coefficient, since
1755  ! it is needed in ghost cells.
1756  mg%bc(:, mg_iveps)%bc_type = mg_bc_neumann
1757  mg%bc(:, mg_iveps)%bc_value = 0.0_dp
1758 
1759  select case (mg%geometry_type)
1760  case (mg_cartesian)
1761  mg%box_op => box_vlpl
1762 
1763  ! TODO: test whether a custom prolongation works better
1764  ! mg%box_prolong => vlpl_prolong
1765 
1766  select case (mg%smoother_type)
1768  mg%box_smoother => box_gs_vlpl
1769  case default
1770  error stop "vlaplacian_set_methods: unsupported smoother type"
1771  end select
1772 
1773  case default
1774  error stop "vlaplacian_set_methods: unsupported geometry"
1775  end select
1776 
1777  end subroutine vlaplacian_set_methods
1778 
1779  !> Perform Gauss-Seidel relaxation on box for a Laplacian operator
1780  subroutine box_gs_vlpl(mg, id, nc, redblack_cntr)
1781  type(mg_t), intent(inout) :: mg
1782  integer, intent(in) :: id
1783  integer, intent(in) :: nc
1784  integer, intent(in) :: redblack_cntr !< Iteration counter
1785  integer :: i, j, k, i0, di
1786  logical :: redblack
1787  real(dp) :: idr2(2*3), u(2*3)
1788  real(dp) :: a0, a(2*3), c(2*3)
1789 
1790  ! Duplicate 1/dr^2 array to multiply neighbor values
1791  idr2(1:2*3:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1792  idr2(2:2*3:2) = idr2(1:2*3:2)
1793  i0 = 1
1794 
1795  redblack = (mg%smoother_type == mg_smoother_gsrb)
1796  if (redblack) then
1797  di = 2
1798  else
1799  di = 1
1800  end if
1801 
1802  ! The parity of redblack_cntr determines which cells we use. If
1803  ! redblack_cntr is even, we use the even cells and vice versa.
1804  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1805  i_eps => mg_iveps)
1806  do k = 1, nc
1807  do j = 1, nc
1808  if (redblack) &
1809  i0 = 2 - iand(ieor(redblack_cntr, k+j), 1)
1810  do i = i0, nc, di
1811  a0 = cc(i, j, k, i_eps)
1812  u(1:2) = cc(i-1:i+1:2, j, k, n)
1813  a(1:2) = cc(i-1:i+1:2, j, k, i_eps)
1814  u(3:4) = cc(i, j-1:j+1:2, k, n)
1815  a(3:4) = cc(i, j-1:j+1:2, k, i_eps)
1816  u(5:6) = cc(i, j, k-1:k+1:2, n)
1817  a(5:6) = cc(i, j, k-1:k+1:2, i_eps)
1818  c(:) = 2 * a0 * a(:) / (a0 + a(:)) * idr2
1819 
1820  cc(i, j, k, n) = (sum(c(:) * u(:)) - &
1821  cc(i, j, k, mg_irhs)) / sum(c(:))
1822  end do
1823  end do
1824  end do
1825  end associate
1826  end subroutine box_gs_vlpl
1827 
1828  !> Perform Laplacian operator on a box
1829  subroutine box_vlpl(mg, id, nc, i_out)
1830  type(mg_t), intent(inout) :: mg
1831  integer, intent(in) :: id
1832  integer, intent(in) :: nc
1833  integer, intent(in) :: i_out !< Index of variable to store Laplacian in
1834  integer :: i, j, k
1835  real(dp) :: idr2(2*3), a0, u0, u(2*3), a(2*3)
1836 
1837  ! Duplicate 1/dr^2 array to multiply neighbor values
1838  idr2(1:2*3:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1839  idr2(2:2*3:2) = idr2(1:2*3:2)
1840 
1841  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1842  i_eps => mg_iveps)
1843  do k = 1, nc
1844  do j = 1, nc
1845  do i = 1, nc
1846  u0 = cc(i, j, k, n)
1847  a0 = cc(i, j, k, i_eps)
1848  u(1:2) = cc(i-1:i+1:2, j, k, n)
1849  u(3:4) = cc(i, j-1:j+1:2, k, n)
1850  u(5:6) = cc(i, j, k-1:k+1:2, n)
1851  a(1:2) = cc(i-1:i+1:2, j, k, i_eps)
1852  a(3:4) = cc(i, j-1:j+1:2, k, i_eps)
1853  a(5:6) = cc(i, j, k-1:k+1:2, i_eps)
1854 
1855  cc(i, j, k, i_out) = sum(2 * idr2 * &
1856  a0*a(:)/(a0 + a(:)) * (u(:) - u0))
1857  end do
1858  end do
1859  end do
1860  end associate
1861  end subroutine box_vlpl
1862 
1863  !> Prolong from a parent to a child with index offset dix. This method could
1864  !> sometimes work better than the default prolongation, which does not take
1865  !> the variation in epsilon into account
1866 ! subroutine vlpl_prolong(mg, p_id, dix, nc, iv, fine)
1867 ! type(mg_t), intent(inout) :: mg
1868 ! integer, intent(in) :: p_id !< Id of parent
1869 ! integer, intent(in) :: dix(3) !< Offset of child in parent grid
1870 ! integer, intent(in) :: nc !< Child grid size
1871 ! integer, intent(in) :: iv !< Prolong from this variable
1872 ! real(dp), intent(out) :: fine(nc, nc, nc) !< Prolonged values
1873 
1874 ! integer :: i, j, k, hnc
1875 ! #if 3 == 2
1876 ! integer :: ic, jc
1877 ! real(dp) :: f0, flx, fhx, fly, fhy
1878 ! real(dp) :: c0, clx, chx, cly, chy
1879 ! #elif 3 == 3
1880 ! integer :: ic, jc, kc
1881 ! real(dp) :: f0, flx, fhx, fly, fhy, flz, fhz
1882 ! real(dp) :: c0, clx, chx, cly, chy, clz, chz
1883 ! #endif
1884 
1885 ! hnc = nc/2
1886 
1887 ! associate (crs => mg%boxes(p_id)%cc, &
1888 ! i_eps => mg_iveps)
1889 ! #if 3 == 2
1890 ! do j = 1, hnc
1891 ! jc = j + dix(2)
1892 ! do i = 1, hnc
1893 ! ic = i + dix(1)
1894 
1895 ! f0 = crs(ic, jc, iv)
1896 ! flx = crs(ic-1, jc, iv)
1897 ! fhx = crs(ic+1, jc, iv)
1898 ! fly = crs(ic, jc-1, iv)
1899 ! fhy = crs(ic, jc+1, iv)
1900 
1901 ! c0 = 2 * crs(ic, jc, i_eps)
1902 ! clx = crs(ic-1, jc, i_eps)
1903 ! chx = crs(ic+1, jc, i_eps)
1904 ! cly = crs(ic, jc-1, i_eps)
1905 ! chy = crs(ic, jc+1, i_eps)
1906 
1907 ! fine(2*i-1, 2*j-1) = (f0*c0 + flx*clx + fly*cly) / &
1908 ! (c0 + clx + cly)
1909 ! fine(2*i , 2*j-1) = (f0*c0 + fhx*chx + fly*cly) / &
1910 ! (c0 + chx + cly)
1911 ! fine(2*i-1, 2*j) = (f0*c0 + flx*clx + fhy*chy) / &
1912 ! (c0 + clx + chy)
1913 ! fine(2*i , 2*j) = (f0*c0 + fhx*chx + fhy*chy) / &
1914 ! (c0 + chx + chy)
1915 ! end do
1916 ! end do
1917 ! #elif 3 == 3
1918 ! do k = 1, hnc
1919 ! kc = k + dix(3)
1920 ! do j = 1, hnc
1921 ! jc = j + dix(2)
1922 ! do i = 1, hnc
1923 ! ic = i + dix(1)
1924 
1925 ! f0 = crs(ic, jc, kc, iv)
1926 ! flx = crs(ic-1, jc, kc, iv)
1927 ! fhx = crs(ic+1, jc, kc, iv)
1928 ! fly = crs(ic, jc-1, kc, iv)
1929 ! fhy = crs(ic, jc+1, kc, iv)
1930 ! flz = crs(ic, jc, kc-1, iv)
1931 ! fhz = crs(ic, jc, kc+1, iv)
1932 
1933 ! c0 = crs(ic, jc, kc, i_eps)
1934 ! clx = crs(ic-1, jc, kc, i_eps)
1935 ! chx = crs(ic+1, jc, kc, i_eps)
1936 ! cly = crs(ic, jc-1, kc, i_eps)
1937 ! chy = crs(ic, jc+1, kc, i_eps)
1938 ! clz = crs(ic, jc, kc-1, i_eps)
1939 ! chz = crs(ic, jc, kc+1, i_eps)
1940 
1941 ! fine(2*i-1, 2*j-1, 2*k-1) = (f0*c0 + flx*clx + fly*cly + flz*clz) / &
1942 ! (c0 + clx + cly + clz)
1943 ! fine(2*i, 2*j-1, 2*k-1) = (f0*c0 + fhx*chx + fly*cly + flz*clz) / &
1944 ! (c0 + chx + cly + clz)
1945 ! fine(2*i-1, 2*j, 2*k-1) = (f0*c0 + flx*clx + fhy*chy + flz*clz) / &
1946 ! (c0 + clx + chy + clz)
1947 ! fine(2*i, 2*j, 2*k-1) = (f0*c0 + fhx*chx + fhy*chy + flz*clz) / &
1948 ! (c0 + chx + chy + clz)
1949 ! fine(2*i-1, 2*j-1, 2*k) = (f0*c0 + flx*clx + fly*cly + fhz*chz) / &
1950 ! (c0 + clx + cly + chz)
1951 ! fine(2*i, 2*j-1, 2*k) = (f0*c0 + fhx*chx + fly*cly + fhz*chz) / &
1952 ! (c0 + chx + cly + chz)
1953 ! fine(2*i-1, 2*j, 2*k) = (f0*c0 + flx*clx + fhy*chy + fhz*chz) / &
1954 ! (c0 + clx + chy + chz)
1955 ! fine(2*i, 2*j, 2*k) = (f0*c0 + fhx*chx + fhy*chy + fhz*chz) / &
1956 ! (c0 + chx + chy + chz)
1957 ! end do
1958 ! end do
1959 ! end do
1960 ! #endif
1961 ! end associate
1962 ! end subroutine vlpl_prolong
1963 
1964  !! File ../src/m_communication.f90
1965 
1966  !> Initialize MPI if needed, and store MPI information
1967  subroutine mg_comm_init(mg, comm)
1968  use mpi
1969  type(mg_t), intent(inout) :: mg
1970  !> MPI communicator (default: MPI_COMM_WORLD)
1971  integer, intent(in), optional :: comm
1972  integer :: ierr
1973  logical :: initialized
1974 
1975  call mpi_initialized(initialized, ierr)
1976  if (.not. initialized) then
1977  call mpi_init(ierr)
1978  end if
1979 
1980  if (present(comm)) then
1981  mg%comm = comm
1982  else
1983  mg%comm = mpi_comm_world
1984  end if
1985 
1986  call mpi_comm_rank(mg%comm, mg%my_rank, ierr)
1987  call mpi_comm_size(mg%comm, mg%n_cpu, ierr)
1988  end subroutine mg_comm_init
1989 
1990  subroutine sort_and_transfer_buffers(mg, dsize)
1991  use mpi
1992  type(mg_t), intent(inout) :: mg
1993  integer, intent(in) :: dsize
1994  integer :: i, n_send, n_recv
1995  integer :: send_req(mg%n_cpu)
1996  integer :: recv_req(mg%n_cpu)
1997  integer :: ierr
1998 
1999  n_send = 0
2000  n_recv = 0
2001 
2002  do i = 0, mg%n_cpu - 1
2003  if (mg%buf(i)%i_send > 0) then
2004  n_send = n_send + 1
2005  call sort_sendbuf(mg%buf(i), dsize)
2006  call mpi_isend(mg%buf(i)%send, mg%buf(i)%i_send, mpi_double, &
2007  i, 0, mg%comm, send_req(n_send), ierr)
2008  end if
2009  if (mg%buf(i)%i_recv > 0) then
2010  n_recv = n_recv + 1
2011  call mpi_irecv(mg%buf(i)%recv, mg%buf(i)%i_recv, mpi_double, &
2012  i, 0, mg%comm, recv_req(n_recv), ierr)
2013  end if
2014  end do
2015 
2016  call mpi_waitall(n_recv, recv_req(1:n_recv), mpi_statuses_ignore, ierr)
2017  call mpi_waitall(n_send, send_req(1:n_send), mpi_statuses_ignore, ierr)
2018 
2019  end subroutine sort_and_transfer_buffers
2020 
2021  !> Sort send buffers according to the idbuf array
2022  subroutine sort_sendbuf(gc, dsize)
2023 
2024  type(mg_buf_t), intent(inout) :: gc
2025  integer, intent(in) :: dsize !< Size of send buffer elements
2026  integer :: ix_sort(gc%i_ix)
2027  real(dp) :: buf_cpy(gc%i_send)
2028  integer :: i, j, n
2029 
2030  call mrgrnk(gc%ix(1:gc%i_ix), ix_sort)
2031 
2032  buf_cpy = gc%send(1:gc%i_send)
2033 
2034  do n = 1, gc%i_ix
2035  i = (n-1) * dsize
2036  j = (ix_sort(n)-1) * dsize
2037  gc%send(i+1:i+dsize) = buf_cpy(j+1:j+dsize)
2038  end do
2039  gc%ix(1:gc%i_ix) = gc%ix(ix_sort)
2040 
2041  end subroutine sort_sendbuf
2042 
2043  !! File ../src/m_ghost_cells.f90
2044 
2045  !> Specify minimum buffer size (per process) for communication
2046  subroutine mg_ghost_cell_buffer_size(mg, n_send, n_recv, dsize)
2047  type(mg_t), intent(inout) :: mg
2048  integer, intent(out) :: n_send(0:mg%n_cpu-1)
2049  integer, intent(out) :: n_recv(0:mg%n_cpu-1)
2050  integer, intent(out) :: dsize
2051  integer :: i, id, lvl, nc
2052 
2053  allocate(mg%comm_ghostcell%n_send(0:mg%n_cpu-1, &
2054  mg%first_normal_lvl:mg%highest_lvl))
2055  allocate(mg%comm_ghostcell%n_recv(0:mg%n_cpu-1, &
2056  mg%first_normal_lvl:mg%highest_lvl))
2057 
2058  dsize = mg%box_size**(3-1)
2059 
2060  do lvl = mg%first_normal_lvl, mg%highest_lvl
2061  nc = mg%box_size_lvl(lvl)
2062  mg%buf(:)%i_send = 0
2063  mg%buf(:)%i_recv = 0
2064  mg%buf(:)%i_ix = 0
2065 
2066  do i = 1, size(mg%lvls(lvl)%my_ids)
2067  id = mg%lvls(lvl)%my_ids(i)
2068  call buffer_ghost_cells(mg, id, nc, 1, dry_run=.true.)
2069  end do
2070 
2071  if (lvl > 1) then
2072  do i = 1, size(mg%lvls(lvl-1)%my_ref_bnds)
2073  id = mg%lvls(lvl-1)%my_ref_bnds(i)
2074  call buffer_refinement_boundaries(mg, id, nc, 1, dry_run=.true.)
2075  end do
2076  end if
2077 
2078  ! Set ghost cells to received data
2079  mg%buf(:)%i_recv = 0
2080  do i = 1, size(mg%lvls(lvl)%my_ids)
2081  id = mg%lvls(lvl)%my_ids(i)
2082  call set_ghost_cells(mg, id, nc, 1, dry_run=.true.)
2083  end do
2084 
2085  mg%comm_ghostcell%n_send(:, lvl) = mg%buf(:)%i_send/dsize
2086  mg%comm_ghostcell%n_recv(:, lvl) = mg%buf(:)%i_recv/dsize
2087  end do
2088 
2089  n_send = maxval(mg%comm_ghostcell%n_send, dim=2)
2090  n_recv = maxval(mg%comm_ghostcell%n_recv, dim=2)
2091  end subroutine mg_ghost_cell_buffer_size
2092 
2093  !> Store boundary conditions for the solution variable, this can speed up
2094  !> calculations if the same boundary conditions are re-used.
2095  subroutine mg_phi_bc_store(mg)
2096  type(mg_t), intent(inout) :: mg
2097  integer :: lvl, nc
2098 
2099  nc = mg%box_size
2100 
2101  do lvl = mg%lowest_lvl, mg%highest_lvl
2102  nc = mg%box_size_lvl(lvl)
2103  call mg_phi_bc_store_lvl(mg, lvl, nc)
2104  end do
2105 
2106  mg%phi_bc_data_stored = .true.
2107  end subroutine mg_phi_bc_store
2108 
2109  subroutine mg_phi_bc_store_lvl(mg, lvl, nc)
2110  type(mg_t), intent(inout) :: mg
2111  integer, intent(in) :: lvl
2112  integer, intent(in) :: nc
2113  real(dp) :: bc(nc, nc)
2114  integer :: i, id, nb, nb_id, bc_type
2115 
2116  do i = 1, size(mg%lvls(lvl)%my_ids)
2117  id = mg%lvls(lvl)%my_ids(i)
2118  do nb = 1, mg_num_neighbors
2119  nb_id = mg%boxes(id)%neighbors(nb)
2120  if (nb_id < mg_no_box) then
2121  ! Physical boundary
2122  if (associated(mg%bc(nb, mg_iphi)%boundary_cond)) then
2123  call mg%bc(nb, mg_iphi)%boundary_cond(mg%boxes(id), nc, &
2124  mg_iphi, nb, bc_type, bc)
2125  else
2126  bc_type = mg%bc(nb, mg_iphi)%bc_type
2127  bc = mg%bc(nb, mg_iphi)%bc_value
2128  end if
2129 
2130  ! Store the boundary condition type. This is not globally set in
2131  ! the tree, but all negative values are treated the same in
2132  ! other parts of the code
2133  mg%boxes(id)%neighbors(nb) = bc_type
2134 
2135  ! Store ghost cell data in the right-hand side
2136  call box_set_gc(mg%boxes(id), nb, nc, mg_irhs, bc)
2137  end if
2138  end do
2139  end do
2140  end subroutine mg_phi_bc_store_lvl
2141 
2142  !> Fill ghost cells at all grid levels
2143  subroutine mg_fill_ghost_cells(mg, iv)
2144  type(mg_t) :: mg
2145  integer, intent(in) :: iv !< Index of variable
2146  integer :: lvl
2147 
2148  do lvl = mg%lowest_lvl, mg%highest_lvl
2149  call mg_fill_ghost_cells_lvl(mg, lvl, iv)
2150  end do
2151  end subroutine mg_fill_ghost_cells
2152 
2153  !> Fill ghost cells at a grid level
2154  subroutine mg_fill_ghost_cells_lvl(mg, lvl, iv)
2155 
2156  type(mg_t) :: mg
2157  integer, intent(in) :: lvl
2158  integer, intent(in) :: iv !< Index of variable
2159  integer :: i, id, dsize, nc
2160 
2161  if (lvl < mg%lowest_lvl) &
2162  error stop "fill_ghost_cells_lvl: lvl < lowest_lvl"
2163  if (lvl > mg%highest_lvl) &
2164  error stop "fill_ghost_cells_lvl: lvl > highest_lvl"
2165 
2166  nc = mg%box_size_lvl(lvl)
2167 
2168  if (lvl >= mg%first_normal_lvl) then
2169  dsize = nc**(3-1)
2170  mg%buf(:)%i_send = 0
2171  mg%buf(:)%i_recv = 0
2172  mg%buf(:)%i_ix = 0
2173 
2174  do i = 1, size(mg%lvls(lvl)%my_ids)
2175  id = mg%lvls(lvl)%my_ids(i)
2176  call buffer_ghost_cells(mg, id, nc, iv, .false.)
2177  end do
2178 
2179  if (lvl > 1) then
2180  do i = 1, size(mg%lvls(lvl-1)%my_ref_bnds)
2181  id = mg%lvls(lvl-1)%my_ref_bnds(i)
2182  call buffer_refinement_boundaries(mg, id, nc, iv, .false.)
2183  end do
2184  end if
2185 
2186  ! Transfer data between processes
2187  mg%buf(:)%i_recv = mg%comm_ghostcell%n_recv(:, lvl) * dsize
2188  call sort_and_transfer_buffers(mg, dsize)
2189 
2190  ! Set ghost cells to received data
2191  mg%buf(:)%i_recv = 0
2192  end if
2193 
2194  do i = 1, size(mg%lvls(lvl)%my_ids)
2195  id = mg%lvls(lvl)%my_ids(i)
2196  call set_ghost_cells(mg, id, nc, iv, .false.)
2197  end do
2198  end subroutine mg_fill_ghost_cells_lvl
2199 
2200  subroutine buffer_ghost_cells(mg, id, nc, iv, dry_run)
2201  type(mg_t), intent(inout) :: mg
2202  integer, intent(in) :: id
2203  integer, intent(in) :: nc
2204  integer, intent(in) :: iv
2205  logical, intent(in) :: dry_run
2206  integer :: nb, nb_id, nb_rank
2207 
2208  do nb = 1, mg_num_neighbors
2209  nb_id = mg%boxes(id)%neighbors(nb)
2210 
2211  if (nb_id > mg_no_box) then
2212  ! There is a neighbor
2213  nb_rank = mg%boxes(nb_id)%rank
2214 
2215  if (nb_rank /= mg%my_rank) then
2216  call buffer_for_nb(mg, mg%boxes(id), nc, iv, nb_id, nb_rank, &
2217  nb, dry_run)
2218  end if
2219  end if
2220  end do
2221  end subroutine buffer_ghost_cells
2222 
2223  subroutine buffer_refinement_boundaries(mg, id, nc, iv, dry_run)
2224  type(mg_t), intent(inout) :: mg
2225  integer, intent(in) :: id
2226  integer, intent(in) :: nc
2227  integer, intent(in) :: iv
2228  logical, intent(in) :: dry_run
2229  integer :: nb, nb_id, c_ids(2**(3-1))
2230  integer :: n, c_id, c_rank
2231 
2232  do nb = 1, mg_num_neighbors
2233  nb_id = mg%boxes(id)%neighbors(nb)
2234  if (nb_id > mg_no_box) then
2235  if (mg_has_children(mg%boxes(nb_id))) then
2236  c_ids = mg%boxes(nb_id)%children(&
2238 
2239  do n = 1, mg_num_children/2
2240  c_id = c_ids(n)
2241  c_rank = mg%boxes(c_id)%rank
2242 
2243  if (c_rank /= mg%my_rank) then
2244  ! Send all coarse ghost cells
2245  call buffer_for_fine_nb(mg, mg%boxes(id), nc, iv, c_id, &
2246  c_rank, nb, dry_run)
2247  end if
2248  end do
2249  end if
2250  end if
2251  end do
2252  end subroutine buffer_refinement_boundaries
2253 
2254  !> The routine that actually fills the ghost cells
2255  subroutine set_ghost_cells(mg, id, nc, iv, dry_run)
2256  type(mg_t), intent(inout) :: mg
2257  integer, intent(in) :: id
2258  integer, intent(in) :: nc
2259  integer, intent(in) :: iv
2260  logical, intent(in) :: dry_run
2261  real(dp) :: bc(nc, nc)
2262  integer :: nb, nb_id, nb_rank, bc_type
2263 
2264  do nb = 1, mg_num_neighbors
2265  nb_id = mg%boxes(id)%neighbors(nb)
2266 
2267  if (nb_id > mg_no_box) then
2268  ! There is a neighbor
2269  nb_rank = mg%boxes(nb_id)%rank
2270 
2271  if (nb_rank /= mg%my_rank) then
2272  call fill_buffered_nb(mg, mg%boxes(id), nb_rank, &
2273  nb, nc, iv, dry_run)
2274  else if (.not. dry_run) then
2275  call copy_from_nb(mg%boxes(id), mg%boxes(nb_id), &
2276  nb, nc, iv)
2277  end if
2278  else if (nb_id == mg_no_box) then
2279  ! Refinement boundary
2280  call fill_refinement_bnd(mg, id, nb, nc, iv, dry_run)
2281  else if (.not. dry_run) then
2282  ! Physical boundary
2283  if (mg%phi_bc_data_stored .and. iv == mg_iphi) then
2284  ! Copy the boundary conditions stored in the ghost cells of the
2285  ! right-hand side
2286  call box_get_gc(mg%boxes(id), nb, nc, mg_irhs, bc)
2287  bc_type = nb_id
2288  else
2289  if (associated(mg%bc(nb, iv)%boundary_cond)) then
2290  call mg%bc(nb, iv)%boundary_cond(mg%boxes(id), nc, iv, &
2291  nb, bc_type, bc)
2292  else
2293  bc_type = mg%bc(nb, iv)%bc_type
2294  bc = mg%bc(nb, iv)%bc_value
2295  end if
2296  end if
2297 
2298  call box_set_gc(mg%boxes(id), nb, nc, iv, bc)
2299  call bc_to_gc(mg, id, nc, iv, nb, bc_type)
2300  end if
2301  end do
2302  end subroutine set_ghost_cells
2303 
2304  subroutine fill_refinement_bnd(mg, id, nb, nc, iv, dry_run)
2305  type(mg_t), intent(inout) :: mg
2306  integer, intent(in) :: id
2307  integer, intent(in) :: nc
2308  integer, intent(in) :: iv
2309  integer, intent(in) :: nb
2310  logical, intent(in) :: dry_run
2311  real(dp) :: gc(nc, nc)
2312  integer :: p_id, p_nb_id, ix_offset(3)
2313  integer :: i, dsize, p_nb_rank
2314 
2315  dsize = nc**(3-1)
2316  p_id = mg%boxes(id)%parent
2317  p_nb_id = mg%boxes(p_id)%neighbors(nb)
2318  p_nb_rank = mg%boxes(p_nb_id)%rank
2319 
2320  if (p_nb_rank /= mg%my_rank) then
2321  i = mg%buf(p_nb_rank)%i_recv
2322  if (.not. dry_run) then
2323  gc = reshape(mg%buf(p_nb_rank)%recv(i+1:i+dsize), shape(gc))
2324  end if
2325  mg%buf(p_nb_rank)%i_recv = mg%buf(p_nb_rank)%i_recv + dsize
2326  else if (.not. dry_run) then
2327  ix_offset = mg_get_child_offset(mg, id)
2328  call box_gc_for_fine_neighbor(mg%boxes(p_nb_id), mg_neighb_rev(nb), &
2329  ix_offset, nc, iv, gc)
2330  end if
2331 
2332  if (.not. dry_run) then
2333  if (associated(mg%bc(nb, iv)%refinement_bnd)) then
2334  call mg%bc(nb, iv)%refinement_bnd(mg%boxes(id), nc, iv, nb, gc)
2335  else
2336  call sides_rb(mg%boxes(id), nc, iv, nb, gc)
2337  end if
2338  end if
2339  end subroutine fill_refinement_bnd
2340 
2341  subroutine copy_from_nb(box, box_nb, nb, nc, iv)
2342  type(mg_box_t), intent(inout) :: box
2343  type(mg_box_t), intent(in) :: box_nb
2344  integer, intent(in) :: nb
2345  integer, intent(in) :: nc
2346  integer, intent(in) :: iv
2347  real(dp) :: gc(nc, nc)
2348 
2349  call box_gc_for_neighbor(box_nb, mg_neighb_rev(nb), nc, iv, gc)
2350  call box_set_gc(box, nb, nc, iv, gc)
2351  end subroutine copy_from_nb
2352 
2353  subroutine buffer_for_nb(mg, box, nc, iv, nb_id, nb_rank, nb, dry_run)
2354  type(mg_t), intent(inout) :: mg
2355  type(mg_box_t), intent(inout) :: box
2356  integer, intent(in) :: nc
2357  integer, intent(in) :: iv
2358  integer, intent(in) :: nb_id
2359  integer, intent(in) :: nb_rank
2360  integer, intent(in) :: nb
2361  logical, intent(in) :: dry_run
2362  integer :: i, dsize
2363  real(dp) :: gc(nc, nc)
2364 
2365  i = mg%buf(nb_rank)%i_send
2366  dsize = nc**(3-1)
2367 
2368  if (.not. dry_run) then
2369  call box_gc_for_neighbor(box, nb, nc, iv, gc)
2370  mg%buf(nb_rank)%send(i+1:i+dsize) = pack(gc, .true.)
2371  end if
2372 
2373  ! Later the buffer is sorted, using the fact that loops go from low to high
2374  ! box id, and we fill ghost cells according to the neighbor order
2375  i = mg%buf(nb_rank)%i_ix
2376  if (.not. dry_run) then
2377  mg%buf(nb_rank)%ix(i+1) = mg_num_neighbors * nb_id + mg_neighb_rev(nb)
2378  end if
2379 
2380  mg%buf(nb_rank)%i_send = mg%buf(nb_rank)%i_send + dsize
2381  mg%buf(nb_rank)%i_ix = mg%buf(nb_rank)%i_ix + 1
2382  end subroutine buffer_for_nb
2383 
2384  subroutine buffer_for_fine_nb(mg, box, nc, iv, fine_id, fine_rank, nb, dry_run)
2385  type(mg_t), intent(inout) :: mg
2386  type(mg_box_t), intent(inout) :: box
2387  integer, intent(in) :: nc
2388  integer, intent(in) :: iv
2389  integer, intent(in) :: fine_id
2390  integer, intent(in) :: fine_rank
2391  integer, intent(in) :: nb
2392  logical, intent(in) :: dry_run
2393  integer :: i, dsize, ix_offset(3)
2394  real(dp) :: gc(nc, nc)
2395 
2396  i = mg%buf(fine_rank)%i_send
2397  dsize = nc**(3-1)
2398 
2399  if (.not. dry_run) then
2400  ix_offset = mg_get_child_offset(mg, fine_id)
2401  call box_gc_for_fine_neighbor(box, nb, ix_offset, nc, iv, gc)
2402  mg%buf(fine_rank)%send(i+1:i+dsize) = pack(gc, .true.)
2403  end if
2404 
2405  ! Later the buffer is sorted, using the fact that loops go from low to high
2406  ! box id, and we fill ghost cells according to the neighbor order
2407  i = mg%buf(fine_rank)%i_ix
2408  if (.not. dry_run) then
2409  mg%buf(fine_rank)%ix(i+1) = mg_num_neighbors * fine_id + &
2410  mg_neighb_rev(nb)
2411  end if
2412 
2413  mg%buf(fine_rank)%i_send = mg%buf(fine_rank)%i_send + dsize
2414  mg%buf(fine_rank)%i_ix = mg%buf(fine_rank)%i_ix + 1
2415  end subroutine buffer_for_fine_nb
2416 
2417  subroutine fill_buffered_nb(mg, box, nb_rank, nb, nc, iv, dry_run)
2418  type(mg_t), intent(inout) :: mg
2419  type(mg_box_t), intent(inout) :: box
2420  integer, intent(in) :: nb_rank
2421  integer, intent(in) :: nb
2422  integer, intent(in) :: nc
2423  integer, intent(in) :: iv
2424  logical, intent(in) :: dry_run
2425  integer :: i, dsize
2426  real(dp) :: gc(nc, nc)
2427 
2428  i = mg%buf(nb_rank)%i_recv
2429  dsize = nc**(3-1)
2430 
2431  if (.not. dry_run) then
2432  gc = reshape(mg%buf(nb_rank)%recv(i+1:i+dsize), shape(gc))
2433  call box_set_gc(box, nb, nc, iv, gc)
2434  end if
2435  mg%buf(nb_rank)%i_recv = mg%buf(nb_rank)%i_recv + dsize
2436 
2437  end subroutine fill_buffered_nb
2438 
2439  subroutine box_gc_for_neighbor(box, nb, nc, iv, gc)
2440  type(mg_box_t), intent(in) :: box
2441  integer, intent(in) :: nb, nc, iv
2442  real(dp), intent(out) :: gc(nc, nc)
2443 
2444  select case (nb)
2445  case (mg_neighb_lowx)
2446  gc = box%cc(1, 1:nc, 1:nc, iv)
2447  case (mg_neighb_highx)
2448  gc = box%cc(nc, 1:nc, 1:nc, iv)
2449  case (mg_neighb_lowy)
2450  gc = box%cc(1:nc, 1, 1:nc, iv)
2451  case (mg_neighb_highy)
2452  gc = box%cc(1:nc, nc, 1:nc, iv)
2453  case (mg_neighb_lowz)
2454  gc = box%cc(1:nc, 1:nc, 1, iv)
2455  case (mg_neighb_highz)
2456  gc = box%cc(1:nc, 1:nc, nc, iv)
2457  end select
2458  end subroutine box_gc_for_neighbor
2459 
2460  !> Get ghost cells for a fine neighbor
2461  subroutine box_gc_for_fine_neighbor(box, nb, di, nc, iv, gc)
2462  type(mg_box_t), intent(in) :: box
2463  integer, intent(in) :: nb !< Direction of fine neighbor
2464  integer, intent(in) :: di(3) !< Index offset of fine neighbor
2465  integer, intent(in) :: nc, iv
2466  real(dp), intent(out) :: gc(nc, nc)
2467  real(dp) :: tmp(0:nc/2+1, 0:nc/2+1), grad(3-1)
2468  integer :: i, j, hnc
2469 
2470  hnc = nc/2
2471 
2472  ! First fill a temporary array with data next to the fine grid
2473  select case (nb)
2474  case (mg_neighb_lowx)
2475  tmp = box%cc(1, di(2):di(2)+hnc+1, di(3):di(3)+hnc+1, iv)
2476  case (mg_neighb_highx)
2477  tmp = box%cc(nc, di(2):di(2)+hnc+1, di(3):di(3)+hnc+1, iv)
2478  case (mg_neighb_lowy)
2479  tmp = box%cc(di(1):di(1)+hnc+1, 1, di(3):di(3)+hnc+1, iv)
2480  case (mg_neighb_highy)
2481  tmp = box%cc(di(1):di(1)+hnc+1, nc, di(3):di(3)+hnc+1, iv)
2482  case (mg_neighb_lowz)
2483  tmp = box%cc(di(1):di(1)+hnc+1, di(2):di(2)+hnc+1, 1, iv)
2484  case (mg_neighb_highz)
2485  tmp = box%cc(di(1):di(1)+hnc+1, di(2):di(2)+hnc+1, nc, iv)
2486  case default
2487  error stop
2488  end select
2489 
2490  ! Now interpolate the coarse grid data to obtain values 'straight' next to
2491  ! the fine grid points
2492  do j = 1, hnc
2493  do i = 1, hnc
2494  grad(1) = 0.125_dp * (tmp(i+1, j) - tmp(i-1, j))
2495  grad(2) = 0.125_dp * (tmp(i, j+1) - tmp(i, j-1))
2496  gc(2*i-1, 2*j-1) = tmp(i, j) - grad(1) - grad(2)
2497  gc(2*i, 2*j-1) = tmp(i, j) + grad(1) - grad(2)
2498  gc(2*i-1, 2*j) = tmp(i, j) - grad(1) + grad(2)
2499  gc(2*i, 2*j) = tmp(i, j) + grad(1) + grad(2)
2500  end do
2501  end do
2502  end subroutine box_gc_for_fine_neighbor
2503 
2504  subroutine box_get_gc(box, nb, nc, iv, gc)
2505  type(mg_box_t), intent(in) :: box
2506  integer, intent(in) :: nb, nc, iv
2507  real(dp), intent(out) :: gc(nc, nc)
2508 
2509  select case (nb)
2510  case (mg_neighb_lowx)
2511  gc = box%cc(0, 1:nc, 1:nc, iv)
2512  case (mg_neighb_highx)
2513  gc = box%cc(nc+1, 1:nc, 1:nc, iv)
2514  case (mg_neighb_lowy)
2515  gc = box%cc(1:nc, 0, 1:nc, iv)
2516  case (mg_neighb_highy)
2517  gc = box%cc(1:nc, nc+1, 1:nc, iv)
2518  case (mg_neighb_lowz)
2519  gc = box%cc(1:nc, 1:nc, 0, iv)
2520  case (mg_neighb_highz)
2521  gc = box%cc(1:nc, 1:nc, nc+1, iv)
2522  end select
2523  end subroutine box_get_gc
2524 
2525  subroutine box_set_gc(box, nb, nc, iv, gc)
2526  type(mg_box_t), intent(inout) :: box
2527  integer, intent(in) :: nb, nc, iv
2528  real(dp), intent(in) :: gc(nc, nc)
2529 
2530  select case (nb)
2531  case (mg_neighb_lowx)
2532  box%cc(0, 1:nc, 1:nc, iv) = gc
2533  case (mg_neighb_highx)
2534  box%cc(nc+1, 1:nc, 1:nc, iv) = gc
2535  case (mg_neighb_lowy)
2536  box%cc(1:nc, 0, 1:nc, iv) = gc
2537  case (mg_neighb_highy)
2538  box%cc(1:nc, nc+1, 1:nc, iv) = gc
2539  case (mg_neighb_lowz)
2540  box%cc(1:nc, 1:nc, 0, iv) = gc
2541  case (mg_neighb_highz)
2542  box%cc(1:nc, 1:nc, nc+1, iv) = gc
2543  end select
2544  end subroutine box_set_gc
2545 
2546  subroutine bc_to_gc(mg, id, nc, iv, nb, bc_type)
2547  type(mg_t), intent(inout) :: mg
2548  integer, intent(in) :: id
2549  integer, intent(in) :: nc
2550  integer, intent(in) :: iv
2551  integer, intent(in) :: nb !< Neighbor direction
2552  integer, intent(in) :: bc_type !< Type of b.c.
2553  real(dp) :: c0, c1, c2, dr
2554 
2555  ! If we call the interior point x1, x2 and the ghost point x0, then a
2556  ! Dirichlet boundary value b can be imposed as:
2557  ! x0 = -x1 + 2*b
2558  ! A Neumann b.c. can be imposed as:
2559  ! x0 = x1 +/- dx * b
2560  ! A continuous boundary (same slope) as:
2561  ! x0 = 2 * x1 - x2
2562  ! Below, we set coefficients to handle these cases
2563  select case (bc_type)
2564  case (mg_bc_dirichlet)
2565  c0 = 2
2566  c1 = -1
2567  c2 = 0
2568  case (mg_bc_neumann)
2569  dr = mg%dr(mg_neighb_dim(nb), mg%boxes(id)%lvl)
2570  c0 = dr * mg_neighb_high_pm(nb) ! This gives a + or - sign
2571  c1 = 1
2572  c2 = 0
2573  case (mg_bc_continuous)
2574  c0 = 0
2575  c1 = 2
2576  c2 = -1
2577  case default
2578  error stop "bc_to_gc: unknown boundary condition"
2579  end select
2580 
2581  select case (nb)
2582  case (mg_neighb_lowx)
2583  mg%boxes(id)%cc(0, 1:nc, 1:nc, iv) = &
2584  c0 * mg%boxes(id)%cc(0, 1:nc, 1:nc, iv) + &
2585  c1 * mg%boxes(id)%cc(1, 1:nc, 1:nc, iv) + &
2586  c2 * mg%boxes(id)%cc(2, 1:nc, 1:nc, iv)
2587  case (mg_neighb_highx)
2588  mg%boxes(id)%cc(nc+1, 1:nc, 1:nc, iv) = &
2589  c0 * mg%boxes(id)%cc(nc+1, 1:nc, 1:nc, iv) + &
2590  c1 * mg%boxes(id)%cc(nc, 1:nc, 1:nc, iv) + &
2591  c2 * mg%boxes(id)%cc(nc-1, 1:nc, 1:nc, iv)
2592  case (mg_neighb_lowy)
2593  mg%boxes(id)%cc(1:nc, 0, 1:nc, iv) = &
2594  c0 * mg%boxes(id)%cc(1:nc, 0, 1:nc, iv) + &
2595  c1 * mg%boxes(id)%cc(1:nc, 1, 1:nc, iv) + &
2596  c2 * mg%boxes(id)%cc(1:nc, 2, 1:nc, iv)
2597  case (mg_neighb_highy)
2598  mg%boxes(id)%cc(1:nc, nc+1, 1:nc, iv) = &
2599  c0 * mg%boxes(id)%cc(1:nc, nc+1, 1:nc, iv) + &
2600  c1 * mg%boxes(id)%cc(1:nc, nc, 1:nc, iv) + &
2601  c2 * mg%boxes(id)%cc(1:nc, nc-1, 1:nc, iv)
2602  case (mg_neighb_lowz)
2603  mg%boxes(id)%cc(1:nc, 1:nc, 0, iv) = &
2604  c0 * mg%boxes(id)%cc(1:nc, 1:nc, 0, iv) + &
2605  c1 * mg%boxes(id)%cc(1:nc, 1:nc, 1, iv) + &
2606  c2 * mg%boxes(id)%cc(1:nc, 1:nc, 2, iv)
2607  case (mg_neighb_highz)
2608  mg%boxes(id)%cc(1:nc, 1:nc, nc+1, iv) = &
2609  c0 * mg%boxes(id)%cc(1:nc, 1:nc, nc+1, iv) + &
2610  c1 * mg%boxes(id)%cc(1:nc, 1:nc, nc, iv) + &
2611  c2 * mg%boxes(id)%cc(1:nc, 1:nc, nc-1, iv)
2612  end select
2613  end subroutine bc_to_gc
2614 
2615  !> Fill ghost cells near refinement boundaries which preserves diffusive fluxes.
2616  subroutine sides_rb(box, nc, iv, nb, gc)
2617  type(mg_box_t), intent(inout) :: box
2618  integer, intent(in) :: nc
2619  integer, intent(in) :: iv
2620  integer, intent(in) :: nb !< Ghost cell direction
2621  !> Interpolated coarse grid ghost cell data (but not yet in the nb direction)
2622  real(dp), intent(in) :: gc(nc, nc)
2623  integer :: di, dj, dk
2624  integer :: i, j, k, ix, dix
2625 
2626  if (mg_neighb_low(nb)) then
2627  ix = 1
2628  dix = 1
2629  else
2630  ix = nc
2631  dix = -1
2632  end if
2633 
2634  select case (mg_neighb_dim(nb))
2635  case (1)
2636  i = ix
2637  di = dix
2638  do k = 1, nc
2639  dk = -1 + 2 * iand(k, 1)
2640  do j = 1, nc
2641  dj = -1 + 2 * iand(j, 1)
2642  box%cc(i-di, j, k, iv) = 0.5_dp * gc(j, k) &
2643  + 0.75_dp * box%cc(i, j, k, iv) &
2644  - 0.25_dp * box%cc(i+di, j, k, iv)
2645  end do
2646  end do
2647  case (2)
2648  j = ix
2649  dj = dix
2650  do k = 1, nc
2651  dk = -1 + 2 * iand(k, 1)
2652  do i = 1, nc
2653  di = -1 + 2 * iand(i, 1)
2654  box%cc(i, j-dj, k, iv) = 0.5_dp * gc(i, k) &
2655  + 0.75_dp * box%cc(i, j, k, iv) &
2656  - 0.25_dp * box%cc(i, j+dj, k, iv)
2657  end do
2658  end do
2659  case (3)
2660  k = ix
2661  dk = dix
2662  do j = 1, nc
2663  dj = -1 + 2 * iand(j, 1)
2664  do i = 1, nc
2665  di = -1 + 2 * iand(i, 1)
2666  box%cc(i, j, k-dk, iv) = 0.5_dp * gc(i, j) &
2667  + 0.75_dp * box%cc(i, j, k, iv) &
2668  - 0.25_dp * box%cc(i, j, k+dk, iv)
2669  end do
2670  end do
2671  end select
2672 
2673  end subroutine sides_rb
2674 
2675  !! File ../src/m_prolong.f90
2676 
2677  !> Specify minimum buffer size (per process) for communication
2678  subroutine mg_prolong_buffer_size(mg, n_send, n_recv, dsize)
2679  type(mg_t), intent(inout) :: mg
2680  integer, intent(out) :: n_send(0:mg%n_cpu-1)
2681  integer, intent(out) :: n_recv(0:mg%n_cpu-1)
2682  integer, intent(out) :: dsize
2683  integer :: lvl, min_lvl
2684 
2685  if (.not. allocated(mg%comm_restrict%n_send)) then
2686  error stop "Call restrict_buffer_size before prolong_buffer_size"
2687  end if
2688 
2689  min_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
2690  allocate(mg%comm_prolong%n_send(0:mg%n_cpu-1, &
2691  min_lvl:mg%highest_lvl))
2692  allocate(mg%comm_prolong%n_recv(0:mg%n_cpu-1, &
2693  min_lvl:mg%highest_lvl))
2694 
2695  mg%comm_prolong%n_recv(:, mg%highest_lvl) = 0
2696  mg%comm_prolong%n_send(:, mg%highest_lvl) = 0
2697 
2698  do lvl = min_lvl, mg%highest_lvl-1
2699  mg%comm_prolong%n_recv(:, lvl) = &
2700  mg%comm_restrict%n_send(:, lvl+1)
2701  mg%comm_prolong%n_send(:, lvl) = &
2702  mg%comm_restrict%n_recv(:, lvl+1)
2703  end do
2704 
2705  ! Send fine grid points, because this is more flexible than sending coarse
2706  ! grid points (e.g., when multiple variables are used for interpolation)
2707  dsize = (mg%box_size)**3
2708  n_send = maxval(mg%comm_prolong%n_send, dim=2)
2709  n_recv = maxval(mg%comm_prolong%n_recv, dim=2)
2710  end subroutine mg_prolong_buffer_size
2711 
2712  !> Prolong variable iv from lvl to variable iv_to at lvl+1
2713  subroutine mg_prolong(mg, lvl, iv, iv_to, method, add)
2714 
2715  type(mg_t), intent(inout) :: mg
2716  integer, intent(in) :: lvl !< Level to prolong from
2717  integer, intent(in) :: iv !< Source variable
2718  integer, intent(in) :: iv_to !< Target variable
2719  procedure(mg_box_prolong) :: method !< Prolongation method
2720  logical, intent(in) :: add !< If true, add to current values
2721  integer :: i, id, dsize, nc
2722 
2723  if (lvl == mg%highest_lvl) error stop "cannot prolong highest level"
2724  if (lvl < mg%lowest_lvl) error stop "cannot prolong below lowest level"
2725 
2726  ! Below the first normal level, all boxes are on the same CPU
2727  if (lvl >= mg%first_normal_lvl-1) then
2728  dsize = mg%box_size**3
2729  mg%buf(:)%i_send = 0
2730  mg%buf(:)%i_ix = 0
2731 
2732  do i = 1, size(mg%lvls(lvl)%my_ids)
2733  id = mg%lvls(lvl)%my_ids(i)
2734  call prolong_set_buffer(mg, id, mg%box_size, iv, method)
2735  end do
2736 
2737  mg%buf(:)%i_recv = mg%comm_prolong%n_recv(:, lvl) * dsize
2738  call sort_and_transfer_buffers(mg, dsize)
2739  mg%buf(:)%i_recv = 0
2740  end if
2741 
2742  nc = mg%box_size_lvl(lvl+1)
2743  do i = 1, size(mg%lvls(lvl+1)%my_ids)
2744  id = mg%lvls(lvl+1)%my_ids(i)
2745  call prolong_onto(mg, id, nc, iv, iv_to, add, method)
2746  end do
2747  end subroutine mg_prolong
2748 
2749  !> In case the fine grid is on a different CPU, perform the prolongation and
2750  !> store the fine-grid values in the send buffer.
2751  !>
2752  subroutine prolong_set_buffer(mg, id, nc, iv, method)
2753  type(mg_t), intent(inout) :: mg
2754  integer, intent(in) :: id
2755  integer, intent(in) :: nc
2756  integer, intent(in) :: iv
2757  procedure(mg_box_prolong) :: method
2758  integer :: i, dix(3)
2759  integer :: i_c, c_id, c_rank, dsize
2760  real(dp) :: tmp(nc, nc, nc)
2761 
2762  dsize = nc**3
2763 
2764  do i_c = 1, mg_num_children
2765  c_id = mg%boxes(id)%children(i_c)
2766  if (c_id > mg_no_box) then
2767  c_rank = mg%boxes(c_id)%rank
2768  if (c_rank /= mg%my_rank) then
2769  dix = mg_get_child_offset(mg, c_id)
2770  call method(mg, id, dix, nc, iv, tmp)
2771 
2772  i = mg%buf(c_rank)%i_send
2773  mg%buf(c_rank)%send(i+1:i+dsize) = pack(tmp, .true.)
2774  mg%buf(c_rank)%i_send = mg%buf(c_rank)%i_send + dsize
2775 
2776  i = mg%buf(c_rank)%i_ix
2777  mg%buf(c_rank)%ix(i+1) = c_id
2778  mg%buf(c_rank)%i_ix = mg%buf(c_rank)%i_ix + 1
2779  end if
2780  end if
2781  end do
2782  end subroutine prolong_set_buffer
2783 
2784  !> Prolong onto a child box
2785  subroutine prolong_onto(mg, id, nc, iv, iv_to, add, method)
2786  type(mg_t), intent(inout) :: mg
2787  integer, intent(in) :: id
2788  integer, intent(in) :: nc
2789  integer, intent(in) :: iv !< Prolong from this variable
2790  integer, intent(in) :: iv_to !< Prolong to this variable
2791  logical, intent(in) :: add !< If true, add to current values
2792  procedure(mg_box_prolong) :: method
2793  integer :: hnc, p_id, p_rank, i, dix(3), dsize
2794  real(dp) :: tmp(nc, nc, nc)
2795 
2796  hnc = nc/2
2797  p_id = mg%boxes(id)%parent
2798  p_rank = mg%boxes(p_id)%rank
2799 
2800  if (p_rank == mg%my_rank) then
2801  dix = mg_get_child_offset(mg, id)
2802  call method(mg, p_id, dix, nc, iv, tmp)
2803  else
2804  dsize = nc**3
2805  i = mg%buf(p_rank)%i_recv
2806  tmp = reshape(mg%buf(p_rank)%recv(i+1:i+dsize), [nc, nc, nc])
2807  mg%buf(p_rank)%i_recv = mg%buf(p_rank)%i_recv + dsize
2808  end if
2809 
2810  if (add) then
2811  mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, iv_to) = &
2812  mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, iv_to) + tmp
2813  else
2814  mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, iv_to) = tmp
2815  end if
2816 
2817  end subroutine prolong_onto
2818 
2819  !> Prolong from a parent to a child with index offset dix
2820  subroutine mg_prolong_sparse(mg, p_id, dix, nc, iv, fine)
2821  type(mg_t), intent(inout) :: mg
2822  integer, intent(in) :: p_id !< Id of parent
2823  integer, intent(in) :: dix(3) !< Offset of child in parent grid
2824  integer, intent(in) :: nc !< Child grid size
2825  integer, intent(in) :: iv !< Prolong from this variable
2826  real(dp), intent(out) :: fine(nc, nc, nc) !< Prolonged values
2827 
2828  integer :: i, j, k, hnc
2829  integer :: ic, jc, kc
2830  real(dp) :: f0, flx, fhx, fly, fhy, flz, fhz
2831 
2832  hnc = nc/2
2833 
2834  associate(crs => mg%boxes(p_id)%cc)
2835  do k = 1, hnc
2836  kc = k + dix(3)
2837  do j = 1, hnc
2838  jc = j + dix(2)
2839  do i = 1, hnc
2840  ic = i + dix(1)
2841 
2842  f0 = 0.25_dp * crs(ic, jc, kc, iv)
2843  flx = 0.25_dp * crs(ic-1, jc, kc, iv)
2844  fhx = 0.25_dp * crs(ic+1, jc, kc, iv)
2845  fly = 0.25_dp * crs(ic, jc-1, kc, iv)
2846  fhy = 0.25_dp * crs(ic, jc+1, kc, iv)
2847  flz = 0.25_dp * crs(ic, jc, kc-1, iv)
2848  fhz = 0.25_dp * crs(ic, jc, kc+1, iv)
2849 
2850  fine(2*i-1, 2*j-1, 2*k-1) = f0 + flx + fly + flz
2851  fine(2*i, 2*j-1, 2*k-1) = f0 + fhx + fly + flz
2852  fine(2*i-1, 2*j, 2*k-1) = f0 + flx + fhy + flz
2853  fine(2*i, 2*j, 2*k-1) = f0 + fhx + fhy + flz
2854  fine(2*i-1, 2*j-1, 2*k) = f0 + flx + fly + fhz
2855  fine(2*i, 2*j-1, 2*k) = f0 + fhx + fly + fhz
2856  fine(2*i-1, 2*j, 2*k) = f0 + flx + fhy + fhz
2857  fine(2*i, 2*j, 2*k) = f0 + fhx + fhy + fhz
2858  end do
2859  end do
2860  end do
2861  end associate
2862  end subroutine mg_prolong_sparse
2863 
2864  !! File ../src/m_helmholtz.f90
2865 
2866  subroutine helmholtz_set_methods(mg)
2867  type(mg_t), intent(inout) :: mg
2868 
2869  mg%subtract_mean = .false.
2870 
2871  select case (mg%geometry_type)
2872  case (mg_cartesian)
2873  mg%box_op => box_helmh
2874 
2875  select case (mg%smoother_type)
2877  mg%box_smoother => box_gs_helmh
2878  case default
2879  error stop "helmholtz_set_methods: unsupported smoother type"
2880  end select
2881  case default
2882  error stop "helmholtz_set_methods: unsupported geometry"
2883  end select
2884 
2885  end subroutine helmholtz_set_methods
2886 
2887  subroutine helmholtz_set_lambda(lambda)
2888  real(dp), intent(in) :: lambda
2889 
2890  if (lambda < 0) &
2891  error stop "helmholtz_set_lambda: lambda < 0 not allowed"
2892 
2893  helmholtz_lambda = lambda
2894  end subroutine helmholtz_set_lambda
2895 
2896  !> Perform Gauss-Seidel relaxation on box for a Helmholtz operator
2897  subroutine box_gs_helmh(mg, id, nc, redblack_cntr)
2898  type(mg_t), intent(inout) :: mg
2899  integer, intent(in) :: id
2900  integer, intent(in) :: nc
2901  integer, intent(in) :: redblack_cntr !< Iteration counter
2902  integer :: i, j, k, i0, di
2903  real(dp) :: idr2(3), fac
2904  logical :: redblack
2905  real(dp), parameter :: sixth = 1/6.0_dp
2906 
2907  idr2 = 1/mg%dr(:, mg%boxes(id)%lvl)**2
2908  fac = 1.0_dp / (2 * sum(idr2) + helmholtz_lambda)
2909  i0 = 1
2910 
2911  redblack = (mg%smoother_type == mg_smoother_gsrb)
2912  if (redblack) then
2913  di = 2
2914  else
2915  di = 1
2916  end if
2917 
2918  ! The parity of redblack_cntr determines which cells we use. If
2919  ! redblack_cntr is even, we use the even cells and vice versa.
2920  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
2921  do k = 1, nc
2922  do j = 1, nc
2923  if (redblack) &
2924  i0 = 2 - iand(ieor(redblack_cntr, k+j), 1)
2925  do i = i0, nc, di
2926  cc(i, j, k, n) = fac * ( &
2927  idr2(1) * (cc(i+1, j, k, n) + cc(i-1, j, k, n)) + &
2928  idr2(2) * (cc(i, j+1, k, n) + cc(i, j-1, k, n)) + &
2929  idr2(3) * (cc(i, j, k+1, n) + cc(i, j, k-1, n)) - &
2930  cc(i, j, k, mg_irhs))
2931  end do
2932  end do
2933  end do
2934  end associate
2935  end subroutine box_gs_helmh
2936 
2937  !> Perform Helmholtz operator on a box
2938  subroutine box_helmh(mg, id, nc, i_out)
2939  type(mg_t), intent(inout) :: mg
2940  integer, intent(in) :: id
2941  integer, intent(in) :: nc
2942  integer, intent(in) :: i_out !< Index of variable to store Helmholtz in
2943  integer :: i, j, k
2944  real(dp) :: idr2(3)
2945 
2946  idr2 = 1 / mg%dr(:, mg%boxes(id)%lvl)**2
2947 
2948  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
2949  do k = 1, nc
2950  do j = 1, nc
2951  do i = 1, nc
2952  cc(i, j, k, i_out) = &
2953  idr2(1) * (cc(i-1, j, k, n) + cc(i+1, j, k, n) &
2954  - 2 * cc(i, j, k, n)) &
2955  + idr2(2) * (cc(i, j-1, k, n) + cc(i, j+1, k, n) &
2956  - 2 * cc(i, j, k, n)) &
2957  + idr2(3) * (cc(i, j, k-1, n) + cc(i, j, k+1, n) &
2958  - 2 * cc(i, j, k, n)) - &
2959  helmholtz_lambda * cc(i, j, k, n)
2960  end do
2961  end do
2962  end do
2963  end associate
2964  end subroutine box_helmh
2965 
2966  !! File ../src/m_multigrid.f90
2967 
2968  subroutine mg_set_methods(mg)
2969 
2970  type(mg_t), intent(inout) :: mg
2971 
2972  ! Set default prolongation method (routines below can override this)
2973  mg%box_prolong => mg_prolong_sparse
2974 
2975  select case (mg%operator_type)
2976  case (mg_laplacian)
2977  call laplacian_set_methods(mg)
2978  case (mg_vlaplacian)
2979  call vlaplacian_set_methods(mg)
2980  case (mg_helmholtz)
2981  call helmholtz_set_methods(mg)
2982  case (mg_vhelmholtz)
2983  call vhelmholtz_set_methods(mg)
2984  case (mg_ahelmholtz)
2985  call ahelmholtz_set_methods(mg)
2986  case default
2987  error stop "mg_set_methods: unknown operator"
2988  end select
2989 
2990  ! For red-black, perform two smoothing sub-steps so that all unknowns are
2991  ! updated per cycle
2992  if (mg%smoother_type == mg_smoother_gsrb) then
2993  mg%n_smoother_substeps = 2
2994  else
2995  mg%n_smoother_substeps = 1
2996  end if
2997  end subroutine mg_set_methods
2998 
2999  subroutine check_methods(mg)
3000  type(mg_t), intent(inout) :: mg
3001 
3002  if (.not. associated(mg%box_op) .or. &
3003  .not. associated(mg%box_smoother)) then
3004  call mg_set_methods(mg)
3005  end if
3006 
3007  end subroutine check_methods
3008 
3009  subroutine mg_add_timers(mg)
3010  type(mg_t), intent(inout) :: mg
3011  timer_total_vcycle = mg_add_timer(mg, "mg total V-cycle")
3012  timer_total_fmg = mg_add_timer(mg, "mg total FMG cycle")
3013  timer_smoother = mg_add_timer(mg, "mg smoother")
3014  timer_smoother_gc = mg_add_timer(mg, "mg smoother g.c.")
3015  timer_coarse = mg_add_timer(mg, "mg coarse")
3016  timer_correct = mg_add_timer(mg, "mg correct")
3017  timer_update_coarse = mg_add_timer(mg, "mg update coarse")
3018  end subroutine mg_add_timers
3019 
3020  !> Perform FAS-FMG cycle (full approximation scheme, full multigrid).
3021  subroutine mg_fas_fmg(mg, have_guess, max_res)
3022  type(mg_t), intent(inout) :: mg
3023  logical, intent(in) :: have_guess !< If false, start from phi = 0
3024  real(dp), intent(out), optional :: max_res !< Store max(abs(residual))
3025  integer :: lvl, i, id
3026 
3027  call check_methods(mg)
3028  if (timer_smoother == -1) call mg_add_timers(mg)
3029 
3030  call mg_timer_start(mg%timers(timer_total_fmg))
3031 
3032  if (.not. have_guess) then
3033  do lvl = mg%highest_lvl, mg%lowest_lvl, -1
3034  do i = 1, size(mg%lvls(lvl)%my_ids)
3035  id = mg%lvls(lvl)%my_ids(i)
3036  mg%boxes(id)%cc(:, :, :, mg_iphi) = 0.0_dp
3037  end do
3038  end do
3039  end if
3040 
3041  ! Ensure ghost cells are filled correctly
3042  call mg_fill_ghost_cells_lvl(mg, mg%highest_lvl, mg_iphi)
3043 
3044  do lvl = mg%highest_lvl, mg%lowest_lvl+1, -1
3045  ! Set rhs on coarse grid and restrict phi
3046  call mg_timer_start(mg%timers(timer_update_coarse))
3047  call update_coarse(mg, lvl)
3048  call mg_timer_end(mg%timers(timer_update_coarse))
3049  end do
3050 
3051  if (mg%subtract_mean) then
3052  ! For fully periodic solutions, the mean source term has to be zero
3053  call subtract_mean(mg, mg_irhs, .false.)
3054  end if
3055 
3056  do lvl = mg%lowest_lvl, mg%highest_lvl
3057  ! Store phi_old
3058  do i = 1, size(mg%lvls(lvl)%my_ids)
3059  id = mg%lvls(lvl)%my_ids(i)
3060  mg%boxes(id)%cc(:, :, :, mg_iold) = &
3061  mg%boxes(id)%cc(:, :, :, mg_iphi)
3062  end do
3063 
3064  if (lvl > mg%lowest_lvl) then
3065  ! Correct solution at this lvl using lvl-1 data
3066  ! phi = phi + prolong(phi_coarse - phi_old_coarse)
3067  call mg_timer_start(mg%timers(timer_correct))
3068  call correct_children(mg, lvl-1)
3069  call mg_timer_end(mg%timers(timer_correct))
3070 
3071  ! Update ghost cells
3072  call mg_fill_ghost_cells_lvl(mg, lvl, mg_iphi)
3073  end if
3074 
3075  ! Perform V-cycle, possibly set residual on last iteration
3076  if (lvl == mg%highest_lvl) then
3077  call mg_fas_vcycle(mg, lvl, max_res, standalone=.false.)
3078  else
3079  call mg_fas_vcycle(mg, lvl, standalone=.false.)
3080  end if
3081  end do
3082 
3083  call mg_timer_end(mg%timers(timer_total_fmg))
3084  end subroutine mg_fas_fmg
3085 
3086  !> Perform FAS V-cycle (full approximation scheme).
3087  subroutine mg_fas_vcycle(mg, highest_lvl, max_res, standalone)
3088  use mpi
3089  type(mg_t), intent(inout) :: mg
3090  integer, intent(in), optional :: highest_lvl !< Maximum level for V-cycle
3091  real(dp), intent(out), optional :: max_res !< Store max(abs(residual))
3092  !> Whether the V-cycle is called by itself (default: true)
3093  logical, intent(in), optional :: standalone
3094  integer :: lvl, min_lvl, i, max_lvl, ierr
3095  real(dp) :: init_res, res
3096  logical :: is_standalone
3097 
3098  is_standalone = .true.
3099  if (present(standalone)) is_standalone = standalone
3100 
3101  call check_methods(mg)
3102  if (timer_smoother == -1) call mg_add_timers(mg)
3103 
3104  if (is_standalone) &
3105  call mg_timer_start(mg%timers(timer_total_vcycle))
3106 
3107  if (mg%subtract_mean .and. .not. present(highest_lvl)) then
3108  ! Assume that this is a stand-alone call. For fully periodic solutions,
3109  ! ensure the mean source term is zero.
3110  call subtract_mean(mg, mg_irhs, .false.)
3111  end if
3112 
3113  min_lvl = mg%lowest_lvl
3114  max_lvl = mg%highest_lvl
3115  if (present(highest_lvl)) max_lvl = highest_lvl
3116 
3117  ! Ensure ghost cells are filled correctly
3118  if (is_standalone) then
3119  call mg_fill_ghost_cells_lvl(mg, max_lvl, mg_iphi)
3120  end if
3121 
3122  do lvl = max_lvl, min_lvl+1, -1
3123  ! Downwards relaxation
3124  call smooth_boxes(mg, lvl, mg%n_cycle_down)
3125 
3126  ! Set rhs on coarse grid, restrict phi, and copy mg_iphi to mg_iold for the
3127  ! correction later
3128  call mg_timer_start(mg%timers(timer_update_coarse))
3129  call update_coarse(mg, lvl)
3130  call mg_timer_end(mg%timers(timer_update_coarse))
3131  end do
3132 
3133  call mg_timer_start(mg%timers(timer_coarse))
3134  if (.not. all(mg%boxes(mg%lvls(min_lvl)%ids)%rank == &
3135  mg%boxes(mg%lvls(min_lvl)%ids(1))%rank)) then
3136  error stop "Multiple CPUs for coarse grid (not implemented yet)"
3137  end if
3138 
3139  init_res = max_residual_lvl(mg, min_lvl)
3140  do i = 1, mg%max_coarse_cycles
3141  call smooth_boxes(mg, min_lvl, mg%n_cycle_up+mg%n_cycle_down)
3142  res = max_residual_lvl(mg, min_lvl)
3143  if (res < mg%residual_coarse_rel * init_res .or. &
3144  res < mg%residual_coarse_abs) exit
3145  end do
3146  call mg_timer_end(mg%timers(timer_coarse))
3147 
3148  ! Do the upwards part of the v-cycle in the tree
3149  do lvl = min_lvl+1, max_lvl
3150  ! Correct solution at this lvl using lvl-1 data
3151  ! phi = phi + prolong(phi_coarse - phi_old_coarse)
3152  call mg_timer_start(mg%timers(timer_correct))
3153  call correct_children(mg, lvl-1)
3154 
3155  ! Have to fill ghost cells after correction
3156  call mg_fill_ghost_cells_lvl(mg, lvl, mg_iphi)
3157  call mg_timer_end(mg%timers(timer_correct))
3158 
3159  ! Upwards relaxation
3160  call smooth_boxes(mg, lvl, mg%n_cycle_up)
3161  end do
3162 
3163  if (present(max_res)) then
3164  init_res = 0.0_dp
3165  do lvl = min_lvl, max_lvl
3166  res = max_residual_lvl(mg, lvl)
3167  init_res = max(res, init_res)
3168  end do
3169  call mpi_allreduce(init_res, max_res, 1, &
3170  mpi_double, mpi_max, mg%comm, ierr)
3171  end if
3172 
3173  ! Subtract mean(phi) from phi
3174  if (mg%subtract_mean) then
3175  call subtract_mean(mg, mg_iphi, .true.)
3176  end if
3177 
3178  if (is_standalone) &
3179  call mg_timer_end(mg%timers(timer_total_vcycle))
3180  end subroutine mg_fas_vcycle
3181 
3182  subroutine subtract_mean(mg, iv, include_ghostcells)
3183  use mpi
3184  type(mg_t), intent(inout) :: mg
3185  integer, intent(in) :: iv
3186  logical, intent(in) :: include_ghostcells
3187  integer :: i, id, lvl, nc, ierr
3188  real(dp) :: sum_iv, mean_iv, volume
3189 
3190  nc = mg%box_size
3191  sum_iv = get_sum(mg, iv)
3192  call mpi_allreduce(sum_iv, mean_iv, 1, &
3193  mpi_double, mpi_sum, mg%comm, ierr)
3194 
3195  ! Divide by total grid volume to get mean
3196  volume = nc**3 * product(mg%dr(:, 1)) * size(mg%lvls(1)%ids)
3197  mean_iv = mean_iv / volume
3198 
3199  do lvl = mg%lowest_lvl, mg%highest_lvl
3200  nc = mg%box_size_lvl(lvl)
3201 
3202  do i = 1, size(mg%lvls(lvl)%my_ids)
3203  id = mg%lvls(lvl)%my_ids(i)
3204  if (include_ghostcells) then
3205  mg%boxes(id)%cc(:, :, :, iv) = &
3206  mg%boxes(id)%cc(:, :, :, iv) - mean_iv
3207  else
3208  mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, iv) = &
3209  mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, iv) - mean_iv
3210  end if
3211  end do
3212  end do
3213  end subroutine subtract_mean
3214 
3215  real(dp) function get_sum(mg, iv)
3216  type(mg_t), intent(in) :: mg
3217  integer, intent(in) :: iv
3218  integer :: lvl, i, id, nc
3219  real(dp) :: w
3220 
3221  get_sum = 0.0_dp
3222  do lvl = 1, mg%highest_lvl
3223  nc = mg%box_size_lvl(lvl)
3224  w = product(mg%dr(:, lvl)) ! Adjust for non-Cartesian cases
3225  do i = 1, size(mg%lvls(lvl)%my_leaves)
3226  id = mg%lvls(lvl)%my_leaves(i)
3227  get_sum = get_sum + w * &
3228  sum(mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, iv))
3229  end do
3230  end do
3231  end function get_sum
3232 
3233  real(dp) function max_residual_lvl(mg, lvl)
3234  type(mg_t), intent(inout) :: mg
3235  integer, intent(in) :: lvl
3236  integer :: i, id, nc
3237  real(dp) :: res
3238 
3239  nc = mg%box_size_lvl(lvl)
3240  max_residual_lvl = 0.0_dp
3241 
3242  do i = 1, size(mg%lvls(lvl)%my_ids)
3243  id = mg%lvls(lvl)%my_ids(i)
3244  call residual_box(mg, id, nc)
3245  res = maxval(abs(mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, mg_ires)))
3246  max_residual_lvl = max(max_residual_lvl, res)
3247  end do
3248  end function max_residual_lvl
3249 
3250  ! subroutine solve_coarse_grid(mg)
3251  ! use m_fishpack
3252  ! type(mg_t), intent(inout) :: mg
3253 
3254  ! real(dp) :: rhs(mg%box_size, mg%box_size, mg%box_size)
3255  ! real(dp) :: rmin(3), rmax(3)
3256  ! integer :: nc, nx(3), my_boxes, total_boxes
3257 
3258  ! my_boxes = size(mg%lvls(1)%my_ids)
3259  ! total_boxes = size(mg%lvls(1)%ids)
3260  ! nc = mg%box_size
3261 
3262  ! if (my_boxes == total_boxes) then
3263  ! nx(:) = nc
3264  ! rmin = [0.0_dp, 0.0_dp, 0.0_dp]
3265  ! rmax = mg%dr(1) * [nc, nc, nc]
3266  ! rhs = mg%boxes(1)%cc(1:nc, 1:nc, 1:nc, mg_irhs)
3267 
3268  ! #if 3 == 2
3269  ! call fishpack_2d(nx, rhs, mg%bc, rmin, rmax)
3270  ! #elif 3 == 3
3271  ! call fishpack_3d(nx, rhs, mg%bc, rmin, rmax)
3272  ! #endif
3273 
3274  ! mg%boxes(1)%cc(1:nc, 1:nc, 1:nc, mg_iphi) = rhs
3275  ! else if (my_boxes > 0) then
3276  ! error stop "Boxes at level 1 at different processors"
3277  ! end if
3278 
3279  ! call fill_ghost_cells_lvl(mg, 1)
3280  ! end subroutine solve_coarse_grid
3281 
3282  ! Set rhs on coarse grid, restrict phi, and copy mg_iphi to mg_iold for the
3283  ! correction later
3284  subroutine update_coarse(mg, lvl)
3285  type(mg_t), intent(inout) :: mg !< Tree containing full grid
3286  integer, intent(in) :: lvl !< Update coarse values at lvl-1
3287  integer :: i, id, nc, nc_c
3288 
3289  nc = mg%box_size_lvl(lvl)
3290  nc_c = mg%box_size_lvl(lvl-1)
3291 
3292  ! Compute residual
3293  do i = 1, size(mg%lvls(lvl)%my_ids)
3294  id = mg%lvls(lvl)%my_ids(i)
3295  call residual_box(mg, id, nc)
3296  end do
3297 
3298  ! Restrict phi and the residual
3299  call mg_restrict_lvl(mg, mg_iphi, lvl)
3300  call mg_restrict_lvl(mg, mg_ires, lvl)
3301 
3302  call mg_fill_ghost_cells_lvl(mg, lvl-1, mg_iphi)
3303 
3304  ! Set rhs_c = laplacian(phi_c) + restrict(res) where it is refined, and
3305  ! store current coarse phi in old.
3306  do i = 1, size(mg%lvls(lvl-1)%my_parents)
3307  id = mg%lvls(lvl-1)%my_parents(i)
3308 
3309  ! Set rhs = L phi
3310  call mg%box_op(mg, id, nc_c, mg_irhs)
3311 
3312  ! Add the fine grid residual to rhs
3313  mg%boxes(id)%cc(1:nc_c, 1:nc_c, 1:nc_c, mg_irhs) = &
3314  mg%boxes(id)%cc(1:nc_c, 1:nc_c, 1:nc_c, mg_irhs) + &
3315  mg%boxes(id)%cc(1:nc_c, 1:nc_c, 1:nc_c, mg_ires)
3316 
3317  ! Story a copy of phi
3318  mg%boxes(id)%cc(:, :, :, mg_iold) = &
3319  mg%boxes(id)%cc(:, :, :, mg_iphi)
3320  end do
3321  end subroutine update_coarse
3322 
3323  ! Sets phi = phi + prolong(phi_coarse - phi_old_coarse)
3324  subroutine correct_children(mg, lvl)
3325  type(mg_t), intent(inout) :: mg
3326  integer, intent(in) :: lvl
3327  integer :: i, id
3328 
3329  do i = 1, size(mg%lvls(lvl)%my_parents)
3330  id = mg%lvls(lvl)%my_parents(i)
3331 
3332  ! Store the correction in mg_ires
3333  mg%boxes(id)%cc(:, :, :, mg_ires) = &
3334  mg%boxes(id)%cc(:, :, :, mg_iphi) - &
3335  mg%boxes(id)%cc(:, :, :, mg_iold)
3336  end do
3337 
3338  call mg_prolong(mg, lvl, mg_ires, mg_iphi, mg%box_prolong, add=.true.)
3339  end subroutine correct_children
3340 
3341  subroutine smooth_boxes(mg, lvl, n_cycle)
3342  type(mg_t), intent(inout) :: mg
3343  integer, intent(in) :: lvl
3344  integer, intent(in) :: n_cycle !< Number of cycles to perform
3345  integer :: n, i, id, nc
3346 
3347  nc = mg%box_size_lvl(lvl)
3348 
3349  do n = 1, n_cycle * mg%n_smoother_substeps
3350  call mg_timer_start(mg%timers(timer_smoother))
3351  do i = 1, size(mg%lvls(lvl)%my_ids)
3352  id = mg%lvls(lvl)%my_ids(i)
3353  call mg%box_smoother(mg, id, nc, n)
3354  end do
3355  call mg_timer_end(mg%timers(timer_smoother))
3356 
3357  call mg_timer_start(mg%timers(timer_smoother_gc))
3358  call mg_fill_ghost_cells_lvl(mg, lvl, mg_iphi)
3359  call mg_timer_end(mg%timers(timer_smoother_gc))
3360  end do
3361  end subroutine smooth_boxes
3362 
3363  subroutine residual_box(mg, id, nc)
3364  type(mg_t), intent(inout) :: mg
3365  integer, intent(in) :: id
3366  integer, intent(in) :: nc
3367 
3368  call mg%box_op(mg, id, nc, mg_ires)
3369 
3370  mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, mg_ires) = &
3371  mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, mg_irhs) &
3372  - mg%boxes(id)%cc(1:nc, 1:nc, 1:nc, mg_ires)
3373  end subroutine residual_box
3374 
3375  !> Apply operator to the tree and store in variable i_out
3376  subroutine mg_apply_op(mg, i_out, op)
3377  type(mg_t), intent(inout) :: mg
3378  integer, intent(in) :: i_out
3379  procedure(mg_box_op), optional :: op
3380  integer :: lvl, i, id, nc
3381 
3382  do lvl = mg%lowest_lvl, mg%highest_lvl
3383  nc = mg%box_size_lvl(lvl)
3384  do i = 1, size(mg%lvls(lvl)%my_ids)
3385  id = mg%lvls(lvl)%my_ids(i)
3386  if (present(op)) then
3387  call op(mg, id, nc, i_out)
3388  else
3389  call mg%box_op(mg, id, nc, i_out)
3390  end if
3391  end do
3392  end do
3393  end subroutine mg_apply_op
3394 
3395  !! File ../src/m_restrict.f90
3396 
3397  !> Specify minimum buffer size (per process) for communication
3398  subroutine mg_restrict_buffer_size(mg, n_send, n_recv, dsize)
3399  type(mg_t), intent(inout) :: mg
3400  integer, intent(out) :: n_send(0:mg%n_cpu-1)
3401  integer, intent(out) :: n_recv(0:mg%n_cpu-1)
3402  integer, intent(out) :: dsize
3403  integer :: n_out(0:mg%n_cpu-1, mg%first_normal_lvl:mg%highest_lvl)
3404  integer :: n_in(0:mg%n_cpu-1, mg%first_normal_lvl:mg%highest_lvl)
3405  integer :: lvl, i, id, p_id, p_rank
3406  integer :: i_c, c_id, c_rank, min_lvl
3407 
3408  n_out(:, :) = 0
3409  n_in(:, :) = 0
3410  min_lvl = max(mg%lowest_lvl+1, mg%first_normal_lvl)
3411 
3412  do lvl = min_lvl, mg%highest_lvl
3413  ! Number of messages to receive (at lvl-1)
3414  do i = 1, size(mg%lvls(lvl-1)%my_parents)
3415  id = mg%lvls(lvl-1)%my_parents(i)
3416  do i_c = 1, mg_num_children
3417  c_id = mg%boxes(id)%children(i_c)
3418 
3419  if (c_id > mg_no_box) then
3420  c_rank = mg%boxes(c_id)%rank
3421  if (c_rank /= mg%my_rank) then
3422  n_in(c_rank, lvl) = n_in(c_rank, lvl) + 1
3423  end if
3424  end if
3425  end do
3426  end do
3427 
3428  ! Number of messages to send
3429  do i = 1, size(mg%lvls(lvl)%my_ids)
3430  id = mg%lvls(lvl)%my_ids(i)
3431 
3432  p_id = mg%boxes(id)%parent
3433  p_rank = mg%boxes(p_id)%rank
3434  if (p_rank /= mg%my_rank) then
3435  n_out(p_rank, lvl) = n_out(p_rank, lvl) + 1
3436  end if
3437 
3438  end do
3439  end do
3440 
3441  allocate(mg%comm_restrict%n_send(0:mg%n_cpu-1, &
3442  mg%first_normal_lvl:mg%highest_lvl))
3443  allocate(mg%comm_restrict%n_recv(0:mg%n_cpu-1, &
3444  mg%first_normal_lvl:mg%highest_lvl))
3445  mg%comm_restrict%n_send = n_out
3446  mg%comm_restrict%n_recv = n_in
3447 
3448  dsize = (mg%box_size/2)**3
3449  n_send = maxval(n_out, dim=2)
3450  n_recv = maxval(n_in, dim=2)
3451  end subroutine mg_restrict_buffer_size
3452 
3453  !> Restrict all levels
3454  subroutine mg_restrict(mg, iv)
3455  type(mg_t), intent(inout) :: mg
3456  integer, intent(in) :: iv
3457  integer :: lvl
3458 
3459  do lvl = mg%highest_lvl, mg%lowest_lvl+1, -1
3460  call mg_restrict_lvl(mg, iv, lvl)
3461  end do
3462  end subroutine mg_restrict
3463 
3464  !> Restrict from lvl to lvl-1
3465  subroutine mg_restrict_lvl(mg, iv, lvl)
3466 
3467  type(mg_t), intent(inout) :: mg
3468  integer, intent(in) :: iv
3469  integer, intent(in) :: lvl
3470  integer :: i, id, dsize, nc
3471 
3472  if (lvl <= mg%lowest_lvl) error stop "cannot restrict lvl <= lowest_lvl"
3473 
3474  nc = mg%box_size_lvl(lvl)
3475 
3476  if (lvl >= mg%first_normal_lvl) then
3477  dsize = (nc/2)**3
3478 
3479  mg%buf(:)%i_send = 0
3480  mg%buf(:)%i_ix = 0
3481 
3482  do i = 1, size(mg%lvls(lvl)%my_ids)
3483  id = mg%lvls(lvl)%my_ids(i)
3484  call restrict_set_buffer(mg, id, iv)
3485  end do
3486 
3487  mg%buf(:)%i_recv = mg%comm_restrict%n_recv(:, lvl) * dsize
3488  call sort_and_transfer_buffers(mg, dsize)
3489  mg%buf(:)%i_recv = 0
3490  end if
3491 
3492  do i = 1, size(mg%lvls(lvl-1)%my_parents)
3493  id = mg%lvls(lvl-1)%my_parents(i)
3494  call restrict_onto(mg, id, nc, iv)
3495  end do
3496  end subroutine mg_restrict_lvl
3497 
3498  subroutine restrict_set_buffer(mg, id, iv)
3499  type(mg_t), intent(inout) :: mg
3500  integer, intent(in) :: id
3501  integer, intent(in) :: iv
3502  integer :: i, j, k, n, hnc, p_id, p_rank
3503  real(dp) :: tmp(mg%box_size/2, mg%box_size/2, mg%box_size/2)
3504 
3505  hnc = mg%box_size/2
3506  p_id = mg%boxes(id)%parent
3507  p_rank = mg%boxes(p_id)%rank
3508 
3509  if (p_rank /= mg%my_rank) then
3510  do k = 1, hnc
3511  do j = 1, hnc
3512  do i = 1, hnc
3513  tmp(i, j, k) = 0.125_dp * sum(mg%boxes(id)%cc(2*i-1:2*i, &
3514  2*j-1:2*j, 2*k-1:2*k, iv))
3515  end do
3516  end do
3517  end do
3518 
3519  ! Buffer
3520  n = size(tmp)
3521  i = mg%buf(p_rank)%i_send
3522  mg%buf(p_rank)%send(i+1:i+n) = pack(tmp, .true.)
3523  mg%buf(p_rank)%i_send = mg%buf(p_rank)%i_send + n
3524 
3525  ! To later sort the send buffer according to parent order
3526  i = mg%buf(p_rank)%i_ix
3527  n = mg_ix_to_ichild(mg%boxes(id)%ix)
3528  mg%buf(p_rank)%ix(i+1) = mg_num_children * p_id + n
3529  mg%buf(p_rank)%i_ix = mg%buf(p_rank)%i_ix + 1
3530  end if
3531  end subroutine restrict_set_buffer
3532 
3533  subroutine restrict_onto(mg, id, nc, iv)
3534  type(mg_t), intent(inout) :: mg
3535  integer, intent(in) :: id
3536  integer, intent(in) :: nc
3537  integer, intent(in) :: iv
3538  integer :: i, j, k, hnc, dsize, i_c, c_id
3539  integer :: c_rank, dix(3)
3540 
3541  hnc = nc/2
3542  dsize = hnc**3
3543 
3544  do i_c = 1, mg_num_children
3545  c_id = mg%boxes(id)%children(i_c)
3546  if (c_id == mg_no_box) cycle ! For coarsened grid
3547  c_rank = mg%boxes(c_id)%rank
3548  dix = mg_get_child_offset(mg, c_id)
3549 
3550  if (c_rank == mg%my_rank) then
3551  do j=1, hnc; do i=1, hnc; do k=1, hnc
3552  mg%boxes(id)%cc(dix(1)+i, dix(2)+j, dix(3)+k, iv) = &
3553  0.125_dp * sum(mg%boxes(c_id)%cc(2*i-1:2*i, &
3554  2*j-1:2*j, 2*k-1:2*k, iv))
3555  end do; end do; end do
3556  else
3557  i = mg%buf(c_rank)%i_recv
3558  mg%boxes(id)%cc(dix(1)+1:dix(1)+hnc, &
3559  dix(2)+1:dix(2)+hnc, dix(3)+1:dix(3)+hnc, iv) = &
3560  reshape(mg%buf(c_rank)%recv(i+1:i+dsize), [hnc, hnc, hnc])
3561  mg%buf(c_rank)%i_recv = mg%buf(c_rank)%i_recv + dsize
3562  end if
3563  end do
3564 
3565  end subroutine restrict_onto
3566 
3567  !! File ../src/m_mrgrnk.f90
3568 
3569  Subroutine i_mrgrnk (XDONT, IRNGT)
3570  ! __________________________________________________________
3571  ! MRGRNK = Merge-sort ranking of an array
3572  ! For performance reasons, the first 2 passes are taken
3573  ! out of the standard loop, and use dedicated coding.
3574  ! __________________________________________________________
3575  ! __________________________________________________________
3576  Integer, Dimension (:), Intent (In) :: xdont
3577  Integer, Dimension (:), Intent (Out) :: irngt
3578  ! __________________________________________________________
3579  Integer :: xvala, xvalb
3580  !
3581  Integer, Dimension (SIZE(IRNGT)) :: jwrkt
3582  Integer :: lmtna, lmtnc, irng1, irng2
3583  Integer :: nval, iind, iwrkd, iwrk, iwrkf, jinda, iinda, iindb
3584  !
3585  nval = min(SIZE(xdont), SIZE(irngt))
3586  Select Case (nval)
3587  Case (:0)
3588  Return
3589  Case (1)
3590  irngt(1) = 1
3591  Return
3592  Case Default
3593  Continue
3594  End Select
3595  !
3596  ! Fill-in the index array, creating ordered couples
3597  !
3598  Do iind = 2, nval, 2
3599  If (xdont(iind-1) <= xdont(iind)) Then
3600  irngt(iind-1) = iind - 1
3601  irngt(iind) = iind
3602  Else
3603  irngt(iind-1) = iind
3604  irngt(iind) = iind - 1
3605  End If
3606  End Do
3607  If (modulo(nval, 2) /= 0) Then
3608  irngt(nval) = nval
3609  End If
3610  !
3611  ! We will now have ordered subsets A - B - A - B - ...
3612  ! and merge A and B couples into C - C - ...
3613  !
3614  lmtna = 2
3615  lmtnc = 4
3616  !
3617  ! First iteration. The length of the ordered subsets goes from 2 to 4
3618  !
3619  Do
3620  If (nval <= 2) Exit
3621  !
3622  ! Loop on merges of A and B into C
3623  !
3624  Do iwrkd = 0, nval - 1, 4
3625  If ((iwrkd+4) > nval) Then
3626  If ((iwrkd+2) >= nval) Exit
3627  !
3628  ! 1 2 3
3629  !
3630  If (xdont(irngt(iwrkd+2)) <= xdont(irngt(iwrkd+3))) Exit
3631  !
3632  ! 1 3 2
3633  !
3634  If (xdont(irngt(iwrkd+1)) <= xdont(irngt(iwrkd+3))) Then
3635  irng2 = irngt(iwrkd+2)
3636  irngt(iwrkd+2) = irngt(iwrkd+3)
3637  irngt(iwrkd+3) = irng2
3638  !
3639  ! 3 1 2
3640  !
3641  Else
3642  irng1 = irngt(iwrkd+1)
3643  irngt(iwrkd+1) = irngt(iwrkd+3)
3644  irngt(iwrkd+3) = irngt(iwrkd+2)
3645  irngt(iwrkd+2) = irng1
3646  End If
3647  Exit
3648  End If
3649  !
3650  ! 1 2 3 4
3651  !
3652  If (xdont(irngt(iwrkd+2)) <= xdont(irngt(iwrkd+3))) cycle
3653  !
3654  ! 1 3 x x
3655  !
3656  If (xdont(irngt(iwrkd+1)) <= xdont(irngt(iwrkd+3))) Then
3657  irng2 = irngt(iwrkd+2)
3658  irngt(iwrkd+2) = irngt(iwrkd+3)
3659  If (xdont(irng2) <= xdont(irngt(iwrkd+4))) Then
3660  ! 1 3 2 4
3661  irngt(iwrkd+3) = irng2
3662  Else
3663  ! 1 3 4 2
3664  irngt(iwrkd+3) = irngt(iwrkd+4)
3665  irngt(iwrkd+4) = irng2
3666  End If
3667  !
3668  ! 3 x x x
3669  !
3670  Else
3671  irng1 = irngt(iwrkd+1)
3672  irng2 = irngt(iwrkd+2)
3673  irngt(iwrkd+1) = irngt(iwrkd+3)
3674  If (xdont(irng1) <= xdont(irngt(iwrkd+4))) Then
3675  irngt(iwrkd+2) = irng1
3676  If (xdont(irng2) <= xdont(irngt(iwrkd+4))) Then
3677  ! 3 1 2 4
3678  irngt(iwrkd+3) = irng2
3679  Else
3680  ! 3 1 4 2
3681  irngt(iwrkd+3) = irngt(iwrkd+4)
3682  irngt(iwrkd+4) = irng2
3683  End If
3684  Else
3685  ! 3 4 1 2
3686  irngt(iwrkd+2) = irngt(iwrkd+4)
3687  irngt(iwrkd+3) = irng1
3688  irngt(iwrkd+4) = irng2
3689  End If
3690  End If
3691  End Do
3692  !
3693  ! The Cs become As and Bs
3694  !
3695  lmtna = 4
3696  Exit
3697  End Do
3698  !
3699  ! Iteration loop. Each time, the length of the ordered subsets
3700  ! is doubled.
3701  !
3702  Do
3703  If (lmtna >= nval) Exit
3704  iwrkf = 0
3705  lmtnc = 2 * lmtnc
3706  !
3707  ! Loop on merges of A and B into C
3708  !
3709  Do
3710  iwrk = iwrkf
3711  iwrkd = iwrkf + 1
3712  jinda = iwrkf + lmtna
3713  iwrkf = iwrkf + lmtnc
3714  If (iwrkf >= nval) Then
3715  If (jinda >= nval) Exit
3716  iwrkf = nval
3717  End If
3718  iinda = 1
3719  iindb = jinda + 1
3720  !
3721  ! Shortcut for the case when the max of A is smaller
3722  ! than the min of B. This line may be activated when the
3723  ! initial set is already close to sorted.
3724  !
3725  ! IF (XDONT(IRNGT(JINDA)) <= XDONT(IRNGT(IINDB))) CYCLE
3726  !
3727  ! One steps in the C subset, that we build in the final rank array
3728  !
3729  ! Make a copy of the rank array for the merge iteration
3730  !
3731  jwrkt(1:lmtna) = irngt(iwrkd:jinda)
3732  !
3733  xvala = xdont(jwrkt(iinda))
3734  xvalb = xdont(irngt(iindb))
3735  !
3736  Do
3737  iwrk = iwrk + 1
3738  !
3739  ! We still have unprocessed values in both A and B
3740  !
3741  If (xvala > xvalb) Then
3742  irngt(iwrk) = irngt(iindb)
3743  iindb = iindb + 1
3744  If (iindb > iwrkf) Then
3745  ! Only A still with unprocessed values
3746  irngt(iwrk+1:iwrkf) = jwrkt(iinda:lmtna)
3747  Exit
3748  End If
3749  xvalb = xdont(irngt(iindb))
3750  Else
3751  irngt(iwrk) = jwrkt(iinda)
3752  iinda = iinda + 1
3753  If (iinda > lmtna) exit! Only B still with unprocessed values
3754  xvala = xdont(jwrkt(iinda))
3755  End If
3756  !
3757  End Do
3758  End Do
3759  !
3760  ! The Cs become As and Bs
3761  !
3762  lmtna = 2 * lmtna
3763  End Do
3764  !
3765  Return
3766  !
3767  End Subroutine i_mrgrnk
3768 
3769  !! File ../src/m_ahelmholtz.f90
3770 
3771  subroutine ahelmholtz_set_methods(mg)
3772  type(mg_t), intent(inout) :: mg
3773 
3774  if (mg%n_extra_vars == 0 .and. mg%is_allocated) then
3775  error stop "ahelmholtz_set_methods: mg%n_extra_vars == 0"
3776  else
3777  mg%n_extra_vars = max(3, mg%n_extra_vars)
3778  end if
3779 
3780  ! Use Neumann zero boundary conditions for the variable coefficient, since
3781  ! it is needed in ghost cells.
3782  mg%bc(:, mg_iveps1)%bc_type = mg_bc_neumann
3783  mg%bc(:, mg_iveps1)%bc_value = 0.0_dp
3784 
3785  mg%bc(:, mg_iveps2)%bc_type = mg_bc_neumann
3786  mg%bc(:, mg_iveps2)%bc_value = 0.0_dp
3787 
3788  mg%bc(:, mg_iveps3)%bc_type = mg_bc_neumann
3789  mg%bc(:, mg_iveps3)%bc_value = 0.0_dp
3790 
3791  select case (mg%geometry_type)
3792  case (mg_cartesian)
3793  mg%box_op => box_ahelmh
3794 
3795  select case (mg%smoother_type)
3797  mg%box_smoother => box_gs_ahelmh
3798  case default
3799  error stop "ahelmholtz_set_methods: unsupported smoother type"
3800  end select
3801  case default
3802  error stop "ahelmholtz_set_methods: unsupported geometry"
3803  end select
3804 
3805  end subroutine ahelmholtz_set_methods
3806 
3807  subroutine ahelmholtz_set_lambda(lambda)
3808  real(dp), intent(in) :: lambda
3809 
3810  if (lambda < 0) &
3811  error stop "ahelmholtz_set_lambda: lambda < 0 not allowed"
3812 
3813  ahelmholtz_lambda = lambda
3814  end subroutine ahelmholtz_set_lambda
3815 
3816  !> Perform Gauss-Seidel relaxation on box for a Helmholtz operator
3817  subroutine box_gs_ahelmh(mg, id, nc, redblack_cntr)
3818  type(mg_t), intent(inout) :: mg
3819  integer, intent(in) :: id
3820  integer, intent(in) :: nc
3821  integer, intent(in) :: redblack_cntr !< Iteration counter
3822  integer :: i, j, k, i0, di
3823  logical :: redblack
3824  real(dp) :: idr2(2*3), u(2*3)
3825  real(dp) :: a0(2*3), a(2*3), c(2*3)
3826 
3827  ! Duplicate 1/dr^2 array to multiply neighbor values
3828  idr2(1:2*3:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
3829  idr2(2:2*3:2) = idr2(1:2*3:2)
3830  i0 = 1
3831 
3832  redblack = (mg%smoother_type == mg_smoother_gsrb)
3833  if (redblack) then
3834  di = 2
3835  else
3836  di = 1
3837  end if
3838 
3839  ! The parity of redblack_cntr determines which cells we use. If
3840  ! redblack_cntr is even, we use the even cells and vice versa.
3841  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
3842  i_eps1 => mg_iveps1, &
3843  i_eps2 => mg_iveps2, &
3844  i_eps3 => mg_iveps3)
3845 
3846  do k = 1, nc
3847  do j = 1, nc
3848  if (redblack) &
3849  i0 = 2 - iand(ieor(redblack_cntr, k+j), 1)
3850  do i = i0, nc, di
3851  a0(1:2) = cc(i, j, k, i_eps1)
3852  a0(3:4) = cc(i, j, k, i_eps2)
3853  a0(4:5) = cc(i, j, k, i_eps3)
3854  u(1:2) = cc(i-1:i+1:2, j, k, n)
3855  a(1:2) = cc(i-1:i+1:2, j, k, i_eps1)
3856  u(3:4) = cc(i, j-1:j+1:2, k, n)
3857  a(3:4) = cc(i, j-1:j+1:2, k, i_eps2)
3858  u(5:6) = cc(i, j, k-1:k+1:2, n)
3859  a(5:6) = cc(i, j, k-1:k+1:2, i_eps3)
3860  c(:) = 2 * a0(:) * a(:) / (a0(:) + a(:)) * idr2
3861 
3862  cc(i, j, k, n) = (sum(c(:) * u(:)) - &
3863  cc(i, j, k, mg_irhs)) / &
3864  (sum(c(:)) + ahelmholtz_lambda)
3865  end do
3866  end do
3867  end do
3868  end associate
3869  end subroutine box_gs_ahelmh
3870 
3871  !> Perform Helmholtz operator on a box
3872  subroutine box_ahelmh(mg, id, nc, i_out)
3873  type(mg_t), intent(inout) :: mg
3874  integer, intent(in) :: id
3875  integer, intent(in) :: nc
3876  integer, intent(in) :: i_out !< Index of variable to store Helmholtz in
3877  integer :: i, j, k
3878  real(dp) :: idr2(2*3), a0(2*3), u0, u(2*3), a(2*3)
3879 
3880  ! Duplicate 1/dr^2 array to multiply neighbor values
3881  idr2(1:2*3:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
3882  idr2(2:2*3:2) = idr2(1:2*3:2)
3883 
3884  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
3885  i_eps1 => mg_iveps1, &
3886  i_eps2 => mg_iveps2, &
3887  i_eps3 => mg_iveps3)
3888  do k = 1, nc
3889  do j = 1, nc
3890  do i = 1, nc
3891  u0 = cc(i, j, k, n)
3892  a0(1:2) = cc(i, j, k, i_eps1)
3893  a0(3:4) = cc(i, j, k, i_eps2)
3894  a0(5:6) = cc(i, j, k, i_eps3)
3895  u(1:2) = cc(i-1:i+1:2, j, k, n)
3896  u(3:4) = cc(i, j-1:j+1:2, k, n)
3897  u(5:6) = cc(i, j, k-1:k+1:2, n)
3898  a(1:2) = cc(i-1:i+1:2, j, k, i_eps1)
3899  a(3:4) = cc(i, j-1:j+1:2, k, i_eps2)
3900  a(5:6) = cc(i, j, k-1:k+1:2, i_eps3)
3901 
3902  cc(i, j, k, i_out) = sum(2 * idr2 * &
3903  a0(:)*a(:)/(a0(:) + a(:)) * (u(:) - u0)) - &
3904  ahelmholtz_lambda * u0
3905  end do
3906  end do
3907  end do
3908  end associate
3909  end subroutine box_ahelmh
3910 
3911 end module m_octree_mg_3d
To fill ghost cells near physical boundaries.
subroutine, public helmholtz_set_lambda(lambda)
subroutine, public mg_set_methods(mg)
subroutine, public mg_restrict_buffer_size(mg, n_send, n_recv, dsize)
Specify minimum buffer size (per process) for communication.
integer, parameter, public mg_irhs
Index of right-hand side.
integer, parameter, public mg_vhelmholtz
Indicates a variable-coefficient Helmholtz equation.
integer, dimension(4, 6), parameter, public mg_child_adj_nb
integer, parameter, public mg_ahelmholtz
Indicates a anisotropic-coefficient Helmholtz equation.
integer, dimension(3, 6), parameter, public mg_neighb_dix
integer function, public mg_ix_to_ichild(ix)
Compute the child index for a box with spatial index ix. With child index we mean the index in the ch...
subroutine, public mg_fas_fmg(mg, have_guess, max_res)
Perform FAS-FMG cycle (full approximation scheme, full multigrid).
integer, parameter, public mg_smoother_gsrb
subroutine subtract_mean(mg, iv, include_ghostcells)
subroutine, public mg_load_balance(mg)
Load balance all boxes in the multigrid tree. Compared to mg_load_balance_simple, this method does a ...
integer, parameter, public mg_num_children
subroutine, public mg_timers_show(mg)
integer, parameter, public mg_max_timers
Maximum number of timers to use.
subroutine, public mg_fas_vcycle(mg, highest_lvl, max_res, standalone)
Perform FAS V-cycle (full approximation scheme).
subroutine, public vhelmholtz_set_methods(mg)
integer, parameter, public mg_bc_dirichlet
Value to indicate a Dirichlet boundary condition.
integer, parameter, public mg_iveps
Index of the variable coefficient (at cell centers)
subroutine, public mg_load_balance_parents(mg)
Load balance the parents (non-leafs). Assign them to the rank that has most children.
integer, dimension(3, 8), parameter, public mg_child_dix
subroutine, public mg_phi_bc_store(mg)
Store boundary conditions for the solution variable, this can speed up calculations if the same bound...
integer(i8) function, public mg_number_of_unknowns(mg)
Determine total number of unknowns (on leaves)
integer, parameter, public mg_vlaplacian
Indicates a variable-coefficient Laplacian.
integer, parameter, public mg_cylindrical
Cylindrical coordinate system.
subroutine, public mg_prolong_sparse(mg, p_id, dix, nc, iv, fine)
Prolong from a parent to a child with index offset dix.
subroutine, public mg_build_rectangle(mg, domain_size, box_size, dx, r_min, periodic, n_finer)
subroutine, public vhelmholtz_set_lambda(lambda)
integer, parameter, public mg_ires
Index of residual.
integer, parameter, public mg_smoother_gs
subroutine, public laplacian_set_methods(mg)
subroutine, public mg_load_balance_simple(mg)
Load balance all boxes in the multigrid tree, by simply distributing the load per grid level....
subroutine, public mg_ghost_cell_buffer_size(mg, n_send, n_recv, dsize)
Specify minimum buffer size (per process) for communication.
subroutine, public mg_set_neighbors_lvl(mg, lvl)
integer, parameter, public mg_smoother_jacobi
integer, parameter, public mg_neighb_lowy
integer, parameter, public mg_bc_continuous
Value to indicate a continuous boundary condition.
subroutine, public mg_set_leaves_parents(boxes, level)
Create a list of leaves and a list of parents for a level.
subroutine, public helmholtz_set_methods(mg)
subroutine, public mg_add_children(mg, id)
integer, parameter, public mg_lvl_lo
Minimum allowed grid level.
integer, parameter, public mg_num_neighbors
integer, parameter, public mg_neighb_highy
real(dp), public, protected ahelmholtz_lambda
Module which contains multigrid procedures for a Helmholtz operator of the form: div(D grad(phi)) - l...
integer, parameter, public mg_iold
Index of previous solution (used for correction)
integer, parameter, public mg_bc_neumann
Value to indicate a Neumann boundary condition.
integer, dimension(6), parameter, public mg_neighb_high_pm
integer, parameter, public mg_ndim
Problem dimension.
integer, parameter, public mg_iphi
Index of solution.
integer, parameter, public mg_helmholtz
Indicates a constant-coefficient Helmholtz equation.
real(dp) function get_sum(mg, iv)
subroutine, public mg_allocate_storage(mg)
Allocate communication buffers and local boxes for a tree that has already been created.
subroutine, public mg_timer_end(timer)
integer, parameter, public mg_num_vars
Number of predefined multigrid variables.
subroutine, public ahelmholtz_set_methods(mg)
real(dp), public, protected vhelmholtz_lambda
Module for implicitly solving diffusion equations.
subroutine, public mg_restrict(mg, iv)
Restrict all levels.
integer, parameter, public mg_neighb_lowx
subroutine, public mg_set_next_level_ids(mg, lvl)
integer, parameter, public mg_physical_boundary
Special value that indicates there is a physical boundary.
integer, dimension(6), parameter, public mg_neighb_rev
integer, parameter, public mg_spherical
Spherical coordinate system.
integer, parameter, public mg_laplacian
Indicates a standard Laplacian.
subroutine, public mg_get_face_coords(box, nb, nc, x)
Get coordinates at the face of a box.
subroutine, public mg_timer_start(timer)
subroutine, public mg_prolong_buffer_size(mg, n_send, n_recv, dsize)
Specify minimum buffer size (per process) for communication.
subroutine, public diffusion_solve(mg, dt, diffusion_coeff, order, max_res)
Solve a diffusion equation implicitly, assuming a constant diffusion coefficient. The solution at tim...
integer, parameter, public mg_neighb_highx
integer, parameter, public mg_cartesian
Cartesian coordinate system.
integer, parameter, public mg_iveps2
integer, parameter, public i8
Type for 64-bit integers.
integer, parameter, public mg_iveps1
Indexes of anisotropic variable coefficients.
subroutine, public mg_fill_ghost_cells_lvl(mg, lvl, iv)
Fill ghost cells at a grid level.
integer, parameter, public mg_neighb_lowz
subroutine, public diffusion_solve_vcoeff(mg, dt, order, max_res)
Solve a diffusion equation implicitly, assuming a variable diffusion coefficient which has been store...
subroutine, public mg_comm_init(mg, comm)
Prolong from a parent to a child with index offset dix. This method could sometimes work better than ...
integer, parameter, public mg_lvl_hi
Maximum allowed grid level.
subroutine, public vlaplacian_set_methods(mg)
logical, dimension(6), parameter, public mg_neighb_low
integer, parameter, public mg_iveps3
elemental logical function, public mg_has_children(box)
Return .true. if a box has children.
subroutine, public mg_deallocate_storage(mg)
Deallocate all allocatable arrays.
real(dp), public, protected helmholtz_lambda
Module for load balancing a tree (that already has been constructed). The load balancing determines w...
subroutine sort_sendbuf(gc, dsize)
Sort send buffers according to the idbuf array.
subroutine, public mg_set_refinement_boundaries(boxes, level)
Create a list of refinement boundaries (from the coarse side)
pure integer function, dimension(3), public mg_get_child_offset(mg, id)
Get the offset of a box with respect to its parent (e.g. in 2d, there can be a child at offset 0,...
integer, parameter, public mg_no_box
Special value that indicates there is no box.
subroutine, public mg_restrict_lvl(mg, iv, lvl)
Restrict from lvl to lvl-1.
subroutine, public diffusion_solve_acoeff(mg, dt, order, max_res)
Solve a diffusion equation implicitly, assuming anisotropic diffusion coefficient which has been stor...
pure integer function, public mg_highest_uniform_lvl(mg)
integer, dimension(6), parameter, public mg_neighb_dim
subroutine, public mg_prolong(mg, lvl, iv, iv_to, method, add)
Prolong variable iv from lvl to variable iv_to at lvl+1.
subroutine, public ahelmholtz_set_lambda(lambda)
subroutine, public mg_fill_ghost_cells(mg, iv)
Fill ghost cells at all grid levels.
integer function, public mg_add_timer(mg, name)
subroutine, public mg_apply_op(mg, i_out, op)
Apply operator to the tree and store in variable i_out.
integer, dimension(8, 3), parameter, public mg_child_rev
logical, dimension(3, 8), parameter, public mg_child_low
integer, parameter, public mg_max_num_vars
Maximum number of variables.
integer, parameter, public dp
Type of reals.
subroutine, public sort_and_transfer_buffers(mg, dsize)
integer, parameter, public mg_neighb_highz
Box data structure.
Buffer type (one is used for each pair of communicating processes)
Lists of blocks per refinement level.