source: anuga_core/source/anuga_parallel/build_commun.py @ 5529

Last change on this file since 5529 was 3926, checked in by ole, 17 years ago

First step towards keeping track of full nodes and triangles in
parallel domains.

File size: 10.6 KB
Line 
1#########################################################
2#
3# Handle the communication between the host machine
4# (processor 0) and the processors. The host machine is
5# responsible for the doing the initial grid partitioning.
6#
7# The routines given below should be moved to the
8# build_submesh.py and build_local.py file to allow
9# overlapping of  communication and computation.
10# This should be done after more debugging.
11#
12#
13#  Author: Linda Stals, June 2005
14#  Modified: Linda Stals, Nov 2005 (optimise python code)
15#
16#
17#########################################################
18
19from Numeric import array, Int, Float, zeros
20import logging, logging.config
21logger = logging.getLogger('parallel')
22logger.setLevel(logging.WARNING)
23
24try:
25    logging.config.fileConfig('log.ini')
26except:
27    pass
28
29import sys
30
31import pypar
32
33from build_local import build_local_mesh
34
35#########################################################
36#
37# Send the submesh to processor p.
38#
39# *) The order and form is strongly coupled with
40# rec_submesh.
41#
42# -------------------------------------------------------
43#
44# *) All of the information has been sent to processor p.
45#
46#########################################################
47
48def send_submesh(submesh, triangles_per_proc, p):
49
50    myid = pypar.rank()
51    print 'process %d sending submesh to process %d' %(myid, p)
52   
53    # build and send the tagmap for the boundary conditions
54   
55    tagmap = {}
56    counter = 1
57    for b in submesh["full_boundary"][p]:
58         bkey = submesh["full_boundary"][p][b]
59         if not tagmap.has_key(bkey):
60             tagmap[bkey] = counter
61             counter = counter+1
62    for b in submesh["ghost_boundary"][p]:
63         bkey = submesh["ghost_boundary"][p][b]
64         if not tagmap.has_key(bkey):
65             tagmap[bkey] = counter
66             counter = counter+1
67    pypar.send(tagmap, p)
68
69    # send the quantities key information
70   
71    pypar.send(submesh["full_quan"].keys(), p)
72   
73    # send the number of triangles per processor
74
75    pypar.send(triangles_per_proc, p, use_buffer=True)
76
77    # compress full_commun
78
79    flat_full_commun = []
80
81    for c in submesh["full_commun"][p]:
82        for i in range(len(submesh["full_commun"][p][c])):
83            flat_full_commun.append([c,submesh["full_commun"][p][c][i]])
84
85    # send the array sizes so memory can be allocated
86
87    setup_array = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
88    setup_array[0] = len(submesh["full_nodes"][p])
89    setup_array[1] = len(submesh["ghost_nodes"][p])
90    setup_array[2] = len(submesh["full_triangles"][p])
91    setup_array[3] = len(submesh["ghost_triangles"][p])
92    setup_array[4] = len(submesh["full_boundary"][p])
93    setup_array[5] = len(submesh["ghost_boundary"][p])
94    setup_array[6] = len(submesh["ghost_commun"][p])
95    setup_array[7] = len(flat_full_commun)
96
97    pypar.send(setup_array, p)
98   
99    # send the nodes
100   
101    pypar.send(submesh["full_nodes"][p], p, use_buffer=True)
102    pypar.send(submesh["ghost_nodes"][p], p, use_buffer=True)
103
104    # send the triangles
105
106    pypar.send(array(submesh["full_triangles"][p], Int), p, use_buffer=True)
107    pypar.send(submesh["ghost_triangles"][p], p, use_buffer=True)
108
109    # send the boundary
110
111    bc = []
112    for b in submesh["full_boundary"][p]:
113        bc.append([b[0], b[1], tagmap[submesh["full_boundary"][p][b]]])
114    pypar.send(bc, p, use_buffer=True)
115    bc = []
116    for b in submesh["ghost_boundary"][p]:
117        bc.append([b[0], b[1], tagmap[submesh["ghost_boundary"][p][b]]])
118    pypar.send(bc, p, use_buffer=True)
119
120    # send the communication pattern
121
122    pypar.send(submesh["ghost_commun"][p], p, use_buffer=True)
123    pypar.send(flat_full_commun, p, use_buffer=True)
124
125    # send the quantities
126   
127    for k in submesh["full_quan"]:
128        pypar.send(submesh["full_quan"][k][p], p, use_buffer=True)
129       
130    for k in submesh["ghost_quan"]:
131        pypar.send(submesh["ghost_quan"][k][p], p, use_buffer=True)
132       
133
134#########################################################
135#
136# Receive the submesh from processor p.
137#
138# *) The order and form is strongly coupled with
139# send_submesh.
140#
141# -------------------------------------------------------
142#
143# *) All of the information has been received by the
144# processor p and passed into build_local.
145#
146# *) The information is returned in a form needed by the
147# GA datastructure.
148#
149#########################################################
150
151def rec_submesh_flat(p):
152
153    numproc = pypar.size()
154    myid = pypar.rank()
155
156    submesh_cell = {}
157   
158    print 'process %d receiving submesh from process %d' %(myid, p)
159
160    # receive the tagmap for the boundary conditions
161   
162    tagmap = pypar.receive(p)
163    itagmap = {}
164    for t in tagmap:
165        itagmap[tagmap[t]]=t
166
167    # receive the quantities key information
168   
169    qkeys = pypar.receive(p)
170
171    # receive the number of triangles per processor
172
173    triangles_per_proc = []
174    for i in range(numproc):
175        triangles_per_proc.append([0])
176
177    triangles_per_proc = pypar.receive(p, triangles_per_proc)
178
179    # recieve information about the array sizes
180
181    setup_array = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
182    setup_array = pypar.receive(p, setup_array)
183
184    # receive the full nodes
185
186    no_full_nodes = setup_array[0]
187    full_nodes = zeros((no_full_nodes, 3), Float)
188    submesh_cell["full_nodes"] = pypar.receive(p, full_nodes)
189   
190    # receive the ghost nodes
191
192    no_ghost_nodes = setup_array[1]
193    ghost_nodes = zeros((no_ghost_nodes, 3), Float)
194    submesh_cell["ghost_nodes"] = pypar.receive(p, ghost_nodes)
195
196   
197    # receive the full triangles
198
199    no_full_triangles = setup_array[2]
200    full_triangles = zeros((no_full_triangles, 3), Int)
201    submesh_cell["full_triangles"] = pypar.receive(p, full_triangles)
202   
203    # receive the ghost triangles
204
205    no_ghost_triangles = setup_array[3]
206    ghost_triangles = zeros((no_ghost_triangles, 4), Int)
207    submesh_cell["ghost_triangles"] = pypar.receive(p, ghost_triangles)
208   
209    # receive the full boundary
210
211    no_full_boundary = setup_array[4]
212    bc = []
213    for i in range(no_full_boundary):
214        bc.append([0.0, 0.0, 0.0])
215    bnd_c = pypar.receive(p, bc)
216
217    submesh_cell["full_boundary"] = {}
218    for b in bnd_c:
219        submesh_cell["full_boundary"][b[0],b[1]]=itagmap[b[2]]
220
221    # receive the ghost boundary
222
223    no_ghost_boundary = setup_array[5]
224    bc = []
225    for i in range(no_ghost_boundary):
226        bc.append([0.0, 0.0, 0.0])
227    bnd_c = pypar.receive(p, bc)
228
229    submesh_cell["ghost_boundary"] = {}
230    for b in bnd_c:
231        submesh_cell["ghost_boundary"][b[0],b[1]]=itagmap[b[2]]
232
233    # receive the ghost communication pattern
234
235    no_ghost_commun = setup_array[6]
236    ghost_commun = zeros((no_ghost_commun, 2), Int)
237    submesh_cell["ghost_commun"] = pypar.receive(p, ghost_commun)
238   
239    # receive the full communication pattern
240
241    no_full_commun = setup_array[7]
242    full_commun = []
243    for i in range(no_full_commun):
244        full_commun.append([0.0, 0.0])
245
246    full_commun = pypar.receive(p, full_commun)
247
248    submesh_cell["full_commun"] = {}
249    for c in full_commun:
250        submesh_cell["full_commun"][c[0]] = []
251    for c in full_commun:
252        submesh_cell["full_commun"][c[0]].append(c[1])
253
254    # receive the quantities
255
256    no_quantities = len(qkeys)
257    new_quan = zeros((no_full_triangles, 3), Float)
258    submesh_cell["full_quan"]={}
259   
260    for i in range(no_quantities):
261        tmp = pypar.receive(p, new_quan)
262        submesh_cell["full_quan"][qkeys[i]]=zeros((no_full_triangles,3), Float)
263        submesh_cell["full_quan"][qkeys[i]][:] = tmp[:]
264
265    new_quan = zeros((no_ghost_triangles, 3), Float)
266    submesh_cell["ghost_quan"]={}
267    for i in range(no_quantities):
268        tmp = pypar.receive(p, new_quan)
269        submesh_cell["ghost_quan"][qkeys[i]]= zeros((no_ghost_triangles,3), Float)
270        submesh_cell["ghost_quan"][qkeys[i]][:] = tmp[:]
271   
272    return submesh_cell, triangles_per_proc,\
273           no_full_nodes, no_full_triangles
274
275
276
277#########################################################
278#
279# Receive the submesh from processor p.
280#
281# *) The order and form is strongly coupled with
282# send_submesh.
283#
284# -------------------------------------------------------
285#
286# *) All of the information has been received by the
287# processor p and passed into build_local.
288#
289# *) The information is returned in a form needed by the
290# GA datastructure.
291#
292#########################################################
293
294def rec_submesh(p):
295
296    numproc = pypar.size()
297    myid = pypar.rank()
298
299    [submesh_cell, triangles_per_proc,\
300     number_of_full_nodes, number_of_full_triangles] = rec_submesh_flat(p)
301   
302    # find the full triangles assigned to this processor
303
304    lower_t = 0
305    for i in range(myid):
306        lower_t = lower_t+triangles_per_proc[i]
307    upper_t = lower_t+triangles_per_proc[myid]
308
309    # convert the information into a form needed by the GA
310    # datastructure
311
312    [GAnodes, GAtriangles, boundary, quantities, ghost_rec, full_send] = \
313              build_local_mesh(submesh_cell, lower_t, upper_t, \
314                               numproc)
315   
316    return GAnodes, GAtriangles, boundary, quantities,\
317           ghost_rec, full_send,\
318           number_of_full_nodes, number_of_full_triangles
319
320
321#########################################################
322#
323# Extract the submesh that will belong to the
324# "host processor" (i.e. processor zero)
325#
326#  *) See the documentation for build_submesh
327#
328# -------------------------------------------------------
329#
330#  *) A dictionary containing the full_triangles,
331# full_nodes, full_boundary, ghost_triangles, ghost_nodes,
332# ghost_boundary, ghost_commun and full_commun belonging
333# to processor zero are returned.
334#
335#########################################################
336def extract_hostmesh(submesh, triangles_per_proc):
337
338    submesh_cell = {}
339    submesh_cell["full_nodes"] = submesh["full_nodes"][0]
340    submesh_cell["ghost_nodes"] = submesh["ghost_nodes"][0]
341    submesh_cell["full_triangles"] = submesh["full_triangles"][0]
342    submesh_cell["ghost_triangles"] = submesh["ghost_triangles"][0]
343    submesh_cell["full_boundary"] = submesh["full_boundary"][0]
344    submesh_cell["ghost_boundary"] = submesh["ghost_boundary"][0]
345    submesh_cell["ghost_commun"] = submesh["ghost_commun"][0]
346    submesh_cell["full_commun"] = submesh["full_commun"][0]
347    submesh_cell["full_quan"] ={}
348    submesh_cell["ghost_quan"]={}
349    for k in submesh["full_quan"]:
350        submesh_cell["full_quan"][k] = submesh["full_quan"][k][0]
351        submesh_cell["ghost_quan"][k] = submesh["ghost_quan"][k][0]
352
353    numprocs = pypar.size()
354    points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict = \
355            build_local_mesh(submesh_cell, 0, triangles_per_proc[0], numprocs)
356    return  points, vertices, boundary, quantities, ghost_recv_dict, \
357           full_send_dict
358
359
360
Note: See TracBrowser for help on using the repository browser.