source: anuga_core/source/anuga_parallel/build_commun.py @ 3593

Last change on this file since 3593 was 3460, checked in by jack, 18 years ago

Moved parallel to development from inundation

File size: 10.5 KB
Line 
1#########################################################
2#
3# Handle the communication between the host machine
4# (processor 0) and the processors. The host machine is
5# responsible for the doing the initial grid partitioning.
6#
7# The routines given below should be moved to the
8# build_submesh.py and build_local.py file to allow
9# overlapping of  communication and computation.
10# This should be done after more debugging.
11#
12#
13#  Author: Linda Stals, June 2005
14#  Modified: Linda Stals, Nov 2005 (optimise python code)
15#
16#
17#########################################################
18
19from Numeric import array, Int, Float, zeros
20import logging, logging.config
21logger = logging.getLogger('parallel')
22logger.setLevel(logging.WARNING)
23
24try:
25    logging.config.fileConfig('log.ini')
26except:
27    pass
28
29import sys
30
31import pypar
32
33from build_local import build_local_mesh
34
35#########################################################
36#
37# Send the submesh to processor p.
38#
39# *) The order and form is strongly coupled with
40# rec_submesh.
41#
42# -------------------------------------------------------
43#
44# *) All of the information has been sent to processor p.
45#
46#########################################################
47
48def send_submesh(submesh, triangles_per_proc, p):
49
50    myid = pypar.rank()
51    print 'process %d sending submesh to process %d' %(myid, p)
52   
53    # build and send the tagmap for the boundary conditions
54   
55    tagmap = {}
56    counter = 1
57    for b in submesh["full_boundary"][p]:
58         bkey = submesh["full_boundary"][p][b]
59         if not tagmap.has_key(bkey):
60             tagmap[bkey] = counter
61             counter = counter+1
62    for b in submesh["ghost_boundary"][p]:
63         bkey = submesh["ghost_boundary"][p][b]
64         if not tagmap.has_key(bkey):
65             tagmap[bkey] = counter
66             counter = counter+1
67    pypar.send(tagmap, p)
68
69    # send the quantities key information
70   
71    pypar.send(submesh["full_quan"].keys(), p)
72   
73    # send the number of triangles per processor
74
75    pypar.send(triangles_per_proc, p, use_buffer=True)
76
77    # compress full_commun
78
79    flat_full_commun = []
80
81    for c in submesh["full_commun"][p]:
82        for i in range(len(submesh["full_commun"][p][c])):
83            flat_full_commun.append([c,submesh["full_commun"][p][c][i]])
84
85    # send the array sizes so memory can be allocated
86
87    setup_array = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
88    setup_array[0] = len(submesh["full_nodes"][p])
89    setup_array[1] = len(submesh["ghost_nodes"][p])
90    setup_array[2] = len(submesh["full_triangles"][p])
91    setup_array[3] = len(submesh["ghost_triangles"][p])
92    setup_array[4] = len(submesh["full_boundary"][p])
93    setup_array[5] = len(submesh["ghost_boundary"][p])
94    setup_array[6] = len(submesh["ghost_commun"][p])
95    setup_array[7] = len(flat_full_commun)
96
97    pypar.send(setup_array, p)
98   
99    # send the nodes
100   
101    pypar.send(submesh["full_nodes"][p], p, use_buffer=True)
102    pypar.send(submesh["ghost_nodes"][p], p, use_buffer=True)
103
104    # send the triangles
105
106    pypar.send(array(submesh["full_triangles"][p], Int), p, use_buffer=True)
107    pypar.send(submesh["ghost_triangles"][p], p, use_buffer=True)
108
109    # send the boundary
110
111    bc = []
112    for b in submesh["full_boundary"][p]:
113        bc.append([b[0], b[1], tagmap[submesh["full_boundary"][p][b]]])
114    pypar.send(bc, p, use_buffer=True)
115    bc = []
116    for b in submesh["ghost_boundary"][p]:
117        bc.append([b[0], b[1], tagmap[submesh["ghost_boundary"][p][b]]])
118    pypar.send(bc, p, use_buffer=True)
119
120    # send the communication pattern
121
122    pypar.send(submesh["ghost_commun"][p], p, use_buffer=True)
123    pypar.send(flat_full_commun, p, use_buffer=True)
124
125    # send the quantities
126   
127    for k in submesh["full_quan"]:
128        pypar.send(submesh["full_quan"][k][p], p, use_buffer=True)
129       
130    for k in submesh["ghost_quan"]:
131        pypar.send(submesh["ghost_quan"][k][p], p, use_buffer=True)
132       
133
134#########################################################
135#
136# Receive the submesh from processor p.
137#
138# *) The order and form is strongly coupled with
139# send_submesh.
140#
141# -------------------------------------------------------
142#
143# *) All of the information has been received by the
144# processor p and passed into build_local.
145#
146# *) The information is returned in a form needed by the
147# GA datastructure.
148#
149#########################################################
150
151def rec_submesh_flat(p):
152
153    numproc = pypar.size()
154    myid = pypar.rank()
155
156    submesh_cell = {}
157   
158    print 'process %d receiving submesh from process %d' %(myid, p)
159
160    # receive the tagmap for the boundary conditions
161   
162    tagmap = pypar.receive(p)
163    itagmap = {}
164    for t in tagmap:
165        itagmap[tagmap[t]]=t
166
167    # receive the quantities key information
168   
169    qkeys = pypar.receive(p)
170
171    # receive the number of triangles per processor
172
173    triangles_per_proc = []
174    for i in range(numproc):
175        triangles_per_proc.append([0])
176
177    triangles_per_proc = pypar.receive(p, triangles_per_proc)
178
179    # recieve information about the array sizes
180
181    setup_array = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
182    setup_array = pypar.receive(p, setup_array)
183
184    # receive the full nodes
185
186    no_full_nodes = setup_array[0]
187    full_nodes = zeros((no_full_nodes, 3), Float)
188    submesh_cell["full_nodes"] = pypar.receive(p, full_nodes)
189   
190    # receive the ghost nodes
191
192    no_ghost_nodes = setup_array[1]
193    ghost_nodes = zeros((no_ghost_nodes, 3), Float)
194    submesh_cell["ghost_nodes"] = pypar.receive(p, ghost_nodes)
195
196   
197    # receive the full triangles
198
199    no_full_triangles = setup_array[2]
200    full_triangles = zeros((no_full_triangles, 3), Int)
201    submesh_cell["full_triangles"] = pypar.receive(p, full_triangles)
202   
203    # receive the ghost triangles
204
205    no_ghost_triangles = setup_array[3]
206    ghost_triangles = zeros((no_ghost_triangles, 4), Int)
207    submesh_cell["ghost_triangles"] = pypar.receive(p, ghost_triangles)
208   
209    # receive the full boundary
210
211    no_full_boundary = setup_array[4]
212    bc = []
213    for i in range(no_full_boundary):
214        bc.append([0.0, 0.0, 0.0])
215    bnd_c = pypar.receive(p, bc)
216
217    submesh_cell["full_boundary"] = {}
218    for b in bnd_c:
219        submesh_cell["full_boundary"][b[0],b[1]]=itagmap[b[2]]
220
221    # receive the ghost boundary
222
223    no_ghost_boundary = setup_array[5]
224    bc = []
225    for i in range(no_ghost_boundary):
226        bc.append([0.0, 0.0, 0.0])
227    bnd_c = pypar.receive(p, bc)
228
229    submesh_cell["ghost_boundary"] = {}
230    for b in bnd_c:
231        submesh_cell["ghost_boundary"][b[0],b[1]]=itagmap[b[2]]
232
233    # receive the ghost communication pattern
234
235    no_ghost_commun = setup_array[6]
236    ghost_commun = zeros((no_ghost_commun, 2), Int)
237    submesh_cell["ghost_commun"] = pypar.receive(p, ghost_commun)
238   
239    # receive the full communication pattern
240
241    no_full_commun = setup_array[7]
242    full_commun = []
243    for i in range(no_full_commun):
244        full_commun.append([0.0, 0.0])
245
246    full_commun = pypar.receive(p, full_commun)
247
248    submesh_cell["full_commun"] = {}
249    for c in full_commun:
250        submesh_cell["full_commun"][c[0]] = []
251    for c in full_commun:
252        submesh_cell["full_commun"][c[0]].append(c[1])
253
254    # receive the quantities
255
256    no_quantities = len(qkeys)
257    new_quan = zeros((no_full_triangles, 3), Float)
258    submesh_cell["full_quan"]={}
259   
260    for i in range(no_quantities):
261        tmp = pypar.receive(p, new_quan)
262        submesh_cell["full_quan"][qkeys[i]]=zeros((no_full_triangles,3), Float)
263        submesh_cell["full_quan"][qkeys[i]][:] = tmp[:]
264
265    new_quan = zeros((no_ghost_triangles, 3), Float)
266    submesh_cell["ghost_quan"]={}
267    for i in range(no_quantities):
268        tmp = pypar.receive(p, new_quan)
269        submesh_cell["ghost_quan"][qkeys[i]]= zeros((no_ghost_triangles,3), Float)
270        submesh_cell["ghost_quan"][qkeys[i]][:] = tmp[:]
271   
272    return submesh_cell, triangles_per_proc
273
274
275
276#########################################################
277#
278# Receive the submesh from processor p.
279#
280# *) The order and form is strongly coupled with
281# send_submesh.
282#
283# -------------------------------------------------------
284#
285# *) All of the information has been received by the
286# processor p and passed into build_local.
287#
288# *) The information is returned in a form needed by the
289# GA datastructure.
290#
291#########################################################
292
293def rec_submesh(p):
294
295    numproc = pypar.size()
296    myid = pypar.rank()
297
298    [submesh_cell, triangles_per_proc] = rec_submesh_flat(p)
299   
300    # find the full triangles assigned to this processor
301
302    lower_t = 0
303    for i in range(myid):
304        lower_t = lower_t+triangles_per_proc[i]
305    upper_t = lower_t+triangles_per_proc[myid]
306
307    # convert the information into a form needed by the GA
308    # datastructure
309
310    [GAnodes, GAtriangles, boundary, quantities, ghost_rec, full_send] = \
311              build_local_mesh(submesh_cell, lower_t, upper_t, \
312                               numproc)
313   
314    return GAnodes, GAtriangles, boundary, quantities, ghost_rec, full_send
315
316#########################################################
317#
318# Extract the submesh that will belong to the
319# "host processor" (i.e. processor zero)
320#
321#  *) See the documentation for build_submesh
322#
323# -------------------------------------------------------
324#
325#  *) A dictionary containing the full_triangles,
326# full_nodes, full_boundary, ghost_triangles, ghost_nodes,
327# ghost_boundary, ghost_commun and full_commun belonging
328# to processor zero are returned.
329#
330#########################################################
331def extract_hostmesh(submesh, triangles_per_proc):
332
333    submesh_cell = {}
334    submesh_cell["full_nodes"] = submesh["full_nodes"][0]
335    submesh_cell["ghost_nodes"] = submesh["ghost_nodes"][0]
336    submesh_cell["full_triangles"] = submesh["full_triangles"][0]
337    submesh_cell["ghost_triangles"] = submesh["ghost_triangles"][0]
338    submesh_cell["full_boundary"] = submesh["full_boundary"][0]
339    submesh_cell["ghost_boundary"] = submesh["ghost_boundary"][0]
340    submesh_cell["ghost_commun"] = submesh["ghost_commun"][0]
341    submesh_cell["full_commun"] = submesh["full_commun"][0]
342    submesh_cell["full_quan"] ={}
343    submesh_cell["ghost_quan"]={}
344    for k in submesh["full_quan"]:
345        submesh_cell["full_quan"][k] = submesh["full_quan"][k][0]
346        submesh_cell["ghost_quan"][k] = submesh["ghost_quan"][k][0]
347
348    numprocs = pypar.size()
349    points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict = \
350            build_local_mesh(submesh_cell, 0, triangles_per_proc[0], numprocs)
351    return  points, vertices, boundary, quantities, ghost_recv_dict, \
352           full_send_dict
353
354
355
Note: See TracBrowser for help on using the repository browser.