source: anuga_core/source/anuga_parallel/parallel_api.py @ 7448

Last change on this file since 7448 was 7448, checked in by steve, 15 years ago

Reorganising parallel files

File size: 7.2 KB
RevLine 
[3584]1"""Trying to lump parallel stuff into simpler interface
2
[3585]3
[3584]4"""
[3585]5
[3776]6
[7400]7
[3776]8# The abstract Python-MPI interface
9from anuga_parallel.parallel_abstraction import size, rank, get_processor_name
10from anuga_parallel.parallel_abstraction import finalize, send, receive
[3904]11from anuga_parallel.parallel_abstraction import pypar_available, barrier
[3776]12
[3585]13
[3776]14# ANUGA parallel engine (only load if pypar can)
15if pypar_available:
[7448]16    from anuga_parallel.distribute_mesh  import send_submesh
17    from anuga_parallel.distribute_mesh  import rec_submesh
18    from anuga_parallel.distribute_mesh  import extract_hostmesh
[3585]19
[3884]20    # Mesh partitioning using Metis
[7448]21    from anuga_parallel.distribute_mesh import build_submesh
22    from anuga_parallel.distribute_mesh import pmesh_divide_metis
[3776]23
[7448]24    from anuga_parallel.parallel_shallow_water import Parallel_Domain
[3884]25
[3585]26#------------------------------------------------------------------------------
27# Read in processor information
28#------------------------------------------------------------------------------
29
[3628]30numprocs = size()
31myid = rank()
32processor_name = get_processor_name()
[3585]33print 'I am processor %d of %d on node %s' %(myid, numprocs, processor_name)
34
35
36
37
[3595]38def distribute(domain, verbose=False):
39    """ Distribute the domain to all processes
40    """
[3593]41
[3818]42
43    # FIXME: Dummy assignment (until boundaries are refactored to
44    # be independent of domains until they are applied)
[5847]45    if myid == 0:
46        bdmap = {}
47        for tag in domain.get_boundary_tags():
48            bdmap[tag] = None
[3818]49   
50   
[5847]51        domain.set_boundary(bdmap)
[3818]52
53
54
55
[3776]56    if not pypar_available: return domain # Bypass
57
[3595]58    # For some obscure reason this communication must happen prior to
59    # the more complex mesh distribution - Oh Well!
[3593]60    if myid == 0:
[3595]61        domain_name = domain.get_name()
[3612]62        domain_dir = domain.get_datadir()
[3904]63        georef = domain.geo_reference
64       
[3612]65        # FIXME - what other attributes need to be transferred?
66
[3595]67        for p in range(1, numprocs):
[3904]68            send((domain_name, domain_dir, georef), p)
[3595]69    else:
[3612]70        if verbose: print 'P%d: Receiving domain attributes' %(myid)
[3593]71
[3904]72        domain_name, domain_dir, georef = receive(0)
[3595]73
74
[3612]75
[3624]76    # Distribute boundary conditions
77    # FIXME: This cannot handle e.g. Time_boundaries due to
78    # difficulties pickling functions
[3595]79    if myid == 0:
[3612]80        boundary_map = domain.boundary_map
81        for p in range(1, numprocs):
[3628]82            send(boundary_map, p)
[3612]83    else:
84        if verbose: print 'P%d: Receiving boundary map' %(myid)       
85
[3628]86        boundary_map = receive(0)
[3612]87       
88
89
90
91    if myid == 0:
[3595]92        # Partition and distribute mesh.
93        # Structures returned is in the
94        # correct form for the ANUGA data structure
95
96
[3593]97        points, vertices, boundary, quantities,\
98                ghost_recv_dict, full_send_dict,\
[3926]99                number_of_full_nodes, number_of_full_triangles =\
100                distribute_mesh(domain)
[3595]101
[3926]102
[3595]103        if verbose: print 'Communication done'
[3593]104       
105    else:
106        # Read in the mesh partition that belongs to this
[3595]107        # processor
[3612]108        if verbose: print 'P%d: Receiving submeshes' %(myid)               
[3593]109        points, vertices, boundary, quantities,\
110                ghost_recv_dict, full_send_dict,\
[3926]111                number_of_full_nodes, number_of_full_triangles =\
112                rec_submesh(0)
[3593]113
[3595]114
[3593]115    #------------------------------------------------------------------------
[3595]116    # Build the domain for this processor using partion structures
[3593]117    #------------------------------------------------------------------------
[3926]118
[6721]119    if verbose: print 'myid ',myid, number_of_full_nodes, number_of_full_triangles
120
121   
[3593]122    domain = Parallel_Domain(points, vertices, boundary,
[3926]123                             full_send_dict=full_send_dict,
124                             ghost_recv_dict=ghost_recv_dict,
125                             number_of_full_nodes=number_of_full_nodes,
126                             number_of_full_triangles=number_of_full_triangles)
[3593]127
128    #------------------------------------------------------------------------
[3595]129    # Transfer initial conditions to each subdomain
[3593]130    #------------------------------------------------------------------------
131    for q in quantities:
[3595]132        domain.set_quantity(q, quantities[q]) 
[3593]133
134
135    #------------------------------------------------------------------------
[3612]136    # Transfer boundary conditions to each subdomain
[3595]137    #------------------------------------------------------------------------
[3612]138    boundary_map['ghost'] = None  # Add binding to ghost boundary
139    domain.set_boundary(boundary_map)
[3595]140
141
142    #------------------------------------------------------------------------
[3612]143    # Transfer other attributes to each subdomain
144    #------------------------------------------------------------------------
145    domain.set_name(domain_name)
[3904]146    domain.set_datadir(domain_dir)     
147    domain.geo_reference = georef   
[3612]148
149    #------------------------------------------------------------------------
[3593]150    # Return parallel domain to all nodes
151    #------------------------------------------------------------------------
152    return domain   
153
154
155
156
157
158
[3585]159def distribute_mesh(domain):
160
[3635]161    numprocs = size()
[3585]162
163   
164    # Subdivide the mesh
165    print 'Subdivide mesh'
166    nodes, triangles, boundary, triangles_per_proc, quantities = \
167           pmesh_divide_metis(domain, numprocs)
168
[3818]169
[7400]170
171
[3585]172    # Build the mesh that should be assigned to each processor,
[3818]173    # this includes ghost nodes and the communication pattern
[3585]174    print 'Build submeshes'   
175    submesh = build_submesh(nodes, triangles, boundary,\
176                            quantities, triangles_per_proc)
177
[3926]178    for p in range(numprocs):
[3928]179        N = len(submesh['ghost_nodes'][p])               
[3926]180        M = len(submesh['ghost_triangles'][p])
[3928]181        print 'There are %d ghost nodes and %d ghost triangles on proc %d'\
182              %(N, M, p)
[3926]183
184
[3585]185    # Send the mesh partition to the appropriate processor
186    print 'Distribute submeshes'       
187    for p in range(1, numprocs):
188      send_submesh(submesh, triangles_per_proc, p)
189
190    # Build the local mesh for processor 0
[3926]191    points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict =\
[3585]192              extract_hostmesh(submesh, triangles_per_proc)
193
[3926]194    # Keep track of the number full nodes and triangles.
195    # This is useful later if one needs access to a ghost-free domain
196    # Here, we do it for process 0. The others are done in rec_submesh.
197    number_of_full_nodes = len(submesh['full_nodes'][0])
198    number_of_full_triangles = len(submesh['full_triangles'][0])
199       
200    #print
201    #for p in range(numprocs):
202    #    print 'Process %d:' %(p)
203    #
204    #    print 'full_triangles:'
205    #    print submesh['full_triangles'][p]
206    #
207    #    print 'full_nodes:'
208    #    print submesh['full_nodes'][p]
209    #
210    #    print 'ghost_triangles:'
211    #    print submesh['ghost_triangles'][p]#
212    #
213    #    print 'ghost_nodes:'
214    #   print submesh['ghost_nodes'][p]                               
215    #    print
216    #
217    #print 'Receive dict'
218    #print ghost_recv_dict
219    #
220    #print 'Send dict'
221    #print full_send_dict       
222
223
[3588]224    # Return structures necessary for building the parallel domain
[3926]225    return points, vertices, boundary, quantities,\
226           ghost_recv_dict, full_send_dict,\
227           number_of_full_nodes, number_of_full_triangles
[3585]228   
229
230
231
Note: See TracBrowser for help on using the repository browser.