| 1 | """Trying to lump parallel stuff into simpler interface |
|---|
| 2 | |
|---|
| 3 | |
|---|
| 4 | """ |
|---|
| 5 | |
|---|
| 6 | |
|---|
| 7 | |
|---|
| 8 | # The abstract Python-MPI interface |
|---|
| 9 | from anuga_parallel.parallel_abstraction import size, rank, get_processor_name |
|---|
| 10 | from anuga_parallel.parallel_abstraction import finalize, send, receive |
|---|
| 11 | from anuga_parallel.parallel_abstraction import pypar_available, barrier |
|---|
| 12 | |
|---|
| 13 | |
|---|
| 14 | # ANUGA parallel engine (only load if pypar can) |
|---|
| 15 | if pypar_available: |
|---|
| 16 | from anuga_parallel.distribute_mesh import send_submesh |
|---|
| 17 | from anuga_parallel.distribute_mesh import rec_submesh |
|---|
| 18 | from anuga_parallel.distribute_mesh import extract_hostmesh |
|---|
| 19 | |
|---|
| 20 | # Mesh partitioning using Metis |
|---|
| 21 | from anuga_parallel.distribute_mesh import build_submesh |
|---|
| 22 | from anuga_parallel.distribute_mesh import pmesh_divide_metis |
|---|
| 23 | |
|---|
| 24 | from anuga_parallel.parallel_shallow_water import Parallel_domain |
|---|
| 25 | |
|---|
| 26 | #------------------------------------------------------------------------------ |
|---|
| 27 | # Read in processor information |
|---|
| 28 | #------------------------------------------------------------------------------ |
|---|
| 29 | |
|---|
| 30 | numprocs = size() |
|---|
| 31 | myid = rank() |
|---|
| 32 | processor_name = get_processor_name() |
|---|
| 33 | #print 'I am processor %d of %d on node %s' %(myid, numprocs, processor_name) |
|---|
| 34 | |
|---|
| 35 | |
|---|
| 36 | |
|---|
| 37 | |
|---|
| 38 | def distribute(domain, verbose=False): |
|---|
| 39 | """ Distribute the domain to all processes |
|---|
| 40 | """ |
|---|
| 41 | |
|---|
| 42 | |
|---|
| 43 | # FIXME: Dummy assignment (until boundaries are refactored to |
|---|
| 44 | # be independent of domains until they are applied) |
|---|
| 45 | if myid == 0: |
|---|
| 46 | bdmap = {} |
|---|
| 47 | for tag in domain.get_boundary_tags(): |
|---|
| 48 | bdmap[tag] = None |
|---|
| 49 | |
|---|
| 50 | |
|---|
| 51 | domain.set_boundary(bdmap) |
|---|
| 52 | |
|---|
| 53 | |
|---|
| 54 | |
|---|
| 55 | |
|---|
| 56 | if not pypar_available: return domain # Bypass |
|---|
| 57 | |
|---|
| 58 | # For some obscure reason this communication must happen prior to |
|---|
| 59 | # the more complex mesh distribution - Oh Well! |
|---|
| 60 | if myid == 0: |
|---|
| 61 | domain_name = domain.get_name() |
|---|
| 62 | domain_dir = domain.get_datadir() |
|---|
| 63 | georef = domain.geo_reference |
|---|
| 64 | |
|---|
| 65 | # FIXME - what other attributes need to be transferred? |
|---|
| 66 | |
|---|
| 67 | for p in range(1, numprocs): |
|---|
| 68 | send((domain_name, domain_dir, georef), p) |
|---|
| 69 | else: |
|---|
| 70 | if verbose: print 'P%d: Receiving domain attributes' %(myid) |
|---|
| 71 | |
|---|
| 72 | domain_name, domain_dir, georef = receive(0) |
|---|
| 73 | |
|---|
| 74 | |
|---|
| 75 | |
|---|
| 76 | # Distribute boundary conditions |
|---|
| 77 | # FIXME: This cannot handle e.g. Time_boundaries due to |
|---|
| 78 | # difficulties pickling functions |
|---|
| 79 | if myid == 0: |
|---|
| 80 | boundary_map = domain.boundary_map |
|---|
| 81 | for p in range(1, numprocs): |
|---|
| 82 | send(boundary_map, p) |
|---|
| 83 | else: |
|---|
| 84 | if verbose: print 'P%d: Receiving boundary map' %(myid) |
|---|
| 85 | |
|---|
| 86 | boundary_map = receive(0) |
|---|
| 87 | |
|---|
| 88 | |
|---|
| 89 | |
|---|
| 90 | |
|---|
| 91 | if myid == 0: |
|---|
| 92 | # Partition and distribute mesh. |
|---|
| 93 | # Structures returned is in the |
|---|
| 94 | # correct form for the ANUGA data structure |
|---|
| 95 | |
|---|
| 96 | |
|---|
| 97 | points, vertices, boundary, quantities,\ |
|---|
| 98 | ghost_recv_dict, full_send_dict,\ |
|---|
| 99 | number_of_full_nodes, number_of_full_triangles =\ |
|---|
| 100 | distribute_mesh(domain, verbose=verbose) |
|---|
| 101 | |
|---|
| 102 | |
|---|
| 103 | if verbose: print 'Communication done' |
|---|
| 104 | |
|---|
| 105 | else: |
|---|
| 106 | # Read in the mesh partition that belongs to this |
|---|
| 107 | # processor |
|---|
| 108 | if verbose: print 'P%d: Receiving submeshes' %(myid) |
|---|
| 109 | points, vertices, boundary, quantities,\ |
|---|
| 110 | ghost_recv_dict, full_send_dict,\ |
|---|
| 111 | number_of_full_nodes, number_of_full_triangles =\ |
|---|
| 112 | rec_submesh(0, verbose) |
|---|
| 113 | |
|---|
| 114 | |
|---|
| 115 | #------------------------------------------------------------------------ |
|---|
| 116 | # Build the domain for this processor using partion structures |
|---|
| 117 | #------------------------------------------------------------------------ |
|---|
| 118 | |
|---|
| 119 | if verbose: print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % (myid, number_of_full_nodes, number_of_full_triangles) |
|---|
| 120 | |
|---|
| 121 | |
|---|
| 122 | domain = Parallel_domain(points, vertices, boundary, |
|---|
| 123 | full_send_dict=full_send_dict, |
|---|
| 124 | ghost_recv_dict=ghost_recv_dict, |
|---|
| 125 | number_of_full_nodes=number_of_full_nodes, |
|---|
| 126 | number_of_full_triangles=number_of_full_triangles) |
|---|
| 127 | |
|---|
| 128 | #------------------------------------------------------------------------ |
|---|
| 129 | # Transfer initial conditions to each subdomain |
|---|
| 130 | #------------------------------------------------------------------------ |
|---|
| 131 | for q in quantities: |
|---|
| 132 | domain.set_quantity(q, quantities[q]) |
|---|
| 133 | |
|---|
| 134 | |
|---|
| 135 | #------------------------------------------------------------------------ |
|---|
| 136 | # Transfer boundary conditions to each subdomain |
|---|
| 137 | #------------------------------------------------------------------------ |
|---|
| 138 | boundary_map['ghost'] = None # Add binding to ghost boundary |
|---|
| 139 | domain.set_boundary(boundary_map) |
|---|
| 140 | |
|---|
| 141 | |
|---|
| 142 | #------------------------------------------------------------------------ |
|---|
| 143 | # Transfer other attributes to each subdomain |
|---|
| 144 | #------------------------------------------------------------------------ |
|---|
| 145 | domain.set_name(domain_name) |
|---|
| 146 | domain.set_datadir(domain_dir) |
|---|
| 147 | domain.geo_reference = georef |
|---|
| 148 | |
|---|
| 149 | #------------------------------------------------------------------------ |
|---|
| 150 | # Return parallel domain to all nodes |
|---|
| 151 | #------------------------------------------------------------------------ |
|---|
| 152 | return domain |
|---|
| 153 | |
|---|
| 154 | |
|---|
| 155 | |
|---|
| 156 | |
|---|
| 157 | |
|---|
| 158 | |
|---|
| 159 | def distribute_mesh(domain, verbose=False): |
|---|
| 160 | |
|---|
| 161 | numprocs = size() |
|---|
| 162 | |
|---|
| 163 | |
|---|
| 164 | # Subdivide the mesh |
|---|
| 165 | if verbose: print 'Subdivide mesh' |
|---|
| 166 | nodes, triangles, boundary, triangles_per_proc, quantities = \ |
|---|
| 167 | pmesh_divide_metis(domain, numprocs) |
|---|
| 168 | |
|---|
| 169 | |
|---|
| 170 | |
|---|
| 171 | |
|---|
| 172 | # Build the mesh that should be assigned to each processor, |
|---|
| 173 | # this includes ghost nodes and the communication pattern |
|---|
| 174 | if verbose: print 'Build submeshes' |
|---|
| 175 | submesh = build_submesh(nodes, triangles, boundary,\ |
|---|
| 176 | quantities, triangles_per_proc) |
|---|
| 177 | |
|---|
| 178 | if verbose: |
|---|
| 179 | for p in range(numprocs): |
|---|
| 180 | N = len(submesh['ghost_nodes'][p]) |
|---|
| 181 | M = len(submesh['ghost_triangles'][p]) |
|---|
| 182 | print 'There are %d ghost nodes and %d ghost triangles on proc %d'\ |
|---|
| 183 | %(N, M, p) |
|---|
| 184 | |
|---|
| 185 | |
|---|
| 186 | # Send the mesh partition to the appropriate processor |
|---|
| 187 | if verbose: print 'Distribute submeshes' |
|---|
| 188 | for p in range(1, numprocs): |
|---|
| 189 | send_submesh(submesh, triangles_per_proc, p, verbose) |
|---|
| 190 | |
|---|
| 191 | # Build the local mesh for processor 0 |
|---|
| 192 | points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict =\ |
|---|
| 193 | extract_hostmesh(submesh, triangles_per_proc) |
|---|
| 194 | |
|---|
| 195 | # Keep track of the number full nodes and triangles. |
|---|
| 196 | # This is useful later if one needs access to a ghost-free domain |
|---|
| 197 | # Here, we do it for process 0. The others are done in rec_submesh. |
|---|
| 198 | number_of_full_nodes = len(submesh['full_nodes'][0]) |
|---|
| 199 | number_of_full_triangles = len(submesh['full_triangles'][0]) |
|---|
| 200 | |
|---|
| 201 | #print |
|---|
| 202 | #for p in range(numprocs): |
|---|
| 203 | # print 'Process %d:' %(p) |
|---|
| 204 | # |
|---|
| 205 | # print 'full_triangles:' |
|---|
| 206 | # print submesh['full_triangles'][p] |
|---|
| 207 | # |
|---|
| 208 | # print 'full_nodes:' |
|---|
| 209 | # print submesh['full_nodes'][p] |
|---|
| 210 | # |
|---|
| 211 | # print 'ghost_triangles:' |
|---|
| 212 | # print submesh['ghost_triangles'][p]# |
|---|
| 213 | # |
|---|
| 214 | # print 'ghost_nodes:' |
|---|
| 215 | # print submesh['ghost_nodes'][p] |
|---|
| 216 | # print |
|---|
| 217 | # |
|---|
| 218 | #print 'Receive dict' |
|---|
| 219 | #print ghost_recv_dict |
|---|
| 220 | # |
|---|
| 221 | #print 'Send dict' |
|---|
| 222 | #print full_send_dict |
|---|
| 223 | |
|---|
| 224 | |
|---|
| 225 | # Return structures necessary for building the parallel domain |
|---|
| 226 | return points, vertices, boundary, quantities,\ |
|---|
| 227 | ghost_recv_dict, full_send_dict,\ |
|---|
| 228 | number_of_full_nodes, number_of_full_triangles |
|---|
| 229 | |
|---|
| 230 | |
|---|
| 231 | |
|---|
| 232 | |
|---|
