Changeset 8543

Timestamp:

Aug 30, 2012, 2:48:35 PM (13 years ago)

Author:

steve

Message:

small changes to parallel code

Location:

trunk/anuga_core/source

Files:

• anuga/abstract_2d_finite_volumes/generic_domain.py (modified) (2 diffs)
• anuga_parallel/distribute_mesh.py (modified) (12 diffs)
• anuga_parallel/parallel_api.py (modified) (4 diffs)
• anuga_parallel/run_parallel_sw_rectangular_cross.py (modified) (3 diffs)

trunk/anuga_core/source/anuga/abstract_2d_finite_volumes/generic_domain.py

@@ -206 +206 @@
         self.numproc = numproc
         self.ghost_layer_width = ghost_layer_width
+        self.communication_time = 0.0
+        self.communication_reduce_time = 0.0
+        self.communication_broadcast_time = 0.0
 
         # Setup Communication Buffers
@@ -1287 +1290 @@
         self.starttime = float(time)
         self.set_time(self.starttime)
-        
+
+
+
+    '''
+    Outputs domain triangulation, full triangles are shown in green while ghost triangles are shown in blue.
+    The default filename is "domain.png"
+    '''
+    def dump_triangulation(self, filename="domain.png"):
+        # Get vertex coordinates, partition full and ghost triangles based on self.tri_full_flag
+
+        try:
+            import matplotlib
+            matplotlib.use('Agg')
+            import matplotlib.pyplot as plt
+            import matplotlib.tri as tri
+        except:
+            print "Couldn't import module from matplotlib, probably you need to update matplotlib"
+            raise
+
+        vertices = self.get_vertex_coordinates()
+        full_mask = num.repeat(self.tri_full_flag == 1, 3)
+        ghost_mask = num.repeat(self.tri_full_flag == 0, 3)
+
+
+        # Proc 0 gathers full and ghost nodes from self and other processors
+        fx = vertices[full_mask,0]
+        fy = vertices[full_mask,1]
+        gx = vertices[ghost_mask,0]
+        gy = vertices[ghost_mask,1]
+
+
+        # Plot full triangles
+        n = int(len(fx)/3)
+
+        triang = num.array(range(0,3*n))
+        triang.shape = (n, 3)
+        plt.triplot(fx, fy, triang, 'g-')
+
+        # Plot ghost triangles
+        n = int(len(gx)/3)
+        if n > 0:
+            triang = num.array(range(0,3*n))
+            triang.shape = (n, 3)
+            plt.triplot(gx, g, triang, 'b--')
+
+        # Save triangulation to location pointed by filename
+        plt.savefig(filename)
+
+
 
 ################################################################################

trunk/anuga_core/source/anuga_parallel/distribute_mesh.py

@@ -1076 +1076 @@
     
     myid = pypar.rank()
+    nprocs = pypar.size()
     
     if verbose: print 'P%d: Sending submesh to P%d' %(myid, p)
@@ -1094 +1095 @@
              counter = counter+1
 
+
+    def protocol(x):
+        vanilla=False
+        control_info, x = pypar.create_control_info(x, vanilla, return_object=True)
+        print 'protocol', control_info[0]
+
+    # FIXME SR: Creates cPickle dump
+    print 'tagmap', tagmap
+    protocol(tagmap)
     pypar.send(tagmap, p)
 
     # send the quantities key information
 
+    # FIXME SR: Creates cPickle dump
+    print 'full_quant keys', submesh["full_quan"].keys()
+    protocol(submesh["full_quan"].keys())
     pypar.send(submesh["full_quan"].keys(), p)
 
-    # send the number of triangles per processor
-
-    pypar.send(triangles_per_proc, p)
-
-    # ghost layer width
-
-    pypar.send(submesh["ghost_layer_width"][p], p)
-
     # compress full_commun
-
     flat_full_commun = []
 
@@ -1128 +1132 @@
     setup_array[7] = len(flat_full_commun)
     setup_array[8] = len(submesh["full_quan"])
-    
-    pypar.send(num.array(setup_array, num.int), p)
-    
+
+    i=0
+    i +=1 ; print 'send', i
+    protocol(num.array(setup_array, num.int))
+    pypar.send(setup_array, p, bypass=True)
+
+
+    # ghost layer width
+    i +=1 ; print 'send', i
+    protocol(num.array(submesh["ghost_layer_width"][p], num.int))
+    pypar.send(num.array(submesh["ghost_layer_width"][p], num.int), p, bypass=True)
+
+
+    # send the number of triangles per processor
+    i +=1 ; print 'send', i
+    protocol(num.array(triangles_per_proc))
+    pypar.send(num.array(triangles_per_proc), p, bypass=True)
+
     # send the nodes
-
+    i +=1 ; print 'send', i
+    protocol(num.array(submesh["full_nodes"][p], num.float))
+    #pypar.send(num.array(submesh["full_nodes"][p], num.float), p, bypass=False)
     pypar.send(num.array(submesh["full_nodes"][p], num.float), p)
-    pypar.send(num.array(submesh["ghost_nodes"][p], num.float),p)
+
+
+    i +=1 ; print 'send', i
+    protocol(num.array(submesh["ghost_nodes"][p], num.float))
+    pypar.send(num.array(submesh["ghost_nodes"][p], num.float),p, bypass=False)
 
     # send the triangles
 
-    pypar.send(num.array(submesh["full_triangles"][p],  num.int), p)
-    pypar.send(num.array(submesh["ghost_triangles"][p], num.int), p)
+    i +=1 ; print 'send', i
+    protocol(num.array(submesh["full_triangles"][p],  num.int))
+    pypar.send(num.array(submesh["full_triangles"][p],  num.int), p, bypass=False)
+    i +=1 ; print 'send', i
+    protocol(num.array(submesh["ghost_triangles"][p], num.int))
+    pypar.send(num.array(submesh["ghost_triangles"][p], num.int), p, bypass=False)
 
     # send the boundary
@@ -1147 +1176 @@
         bc.append([b[0], b[1], tagmap[submesh["full_boundary"][p][b]]])
 
-
-    pypar.send(num.array(bc, num.int), p)
+    i +=1 ; print 'send', i
+    protocol(num.array(bc, num.int))
+    pypar.send(num.array(bc, num.int), p, bypass=False)
 
     bc = []
@@ -1154 +1184 @@
         bc.append([b[0], b[1], tagmap[submesh["ghost_boundary"][p][b]]])
 
-    pypar.send(num.array(bc, num.int), p)
+    i +=1 ; print 'send', i
+    protocol(num.array(bc, num.int))
+    pypar.send(num.array(bc, num.int), p, bypass=False)
 
     # send the communication pattern
 
-    pypar.send(submesh["ghost_commun"][p], p)
-
-    pypar.send(num.array(flat_full_commun, num.int), p)
+    # submesh["ghost_commun"][p] is numpy array
+    i +=1 ; print 'send', i
+    protocol(submesh["ghost_commun"][p])
+    pypar.send(submesh["ghost_commun"][p], p, bypass=False)
+
+    i +=1 ; print 'send', i
+    protocol(num.array(flat_full_commun, num.int))
+    pypar.send(num.array(flat_full_commun, num.int), p, bypass=False)
 
     # send the quantities
+
     
     for k in submesh["full_quan"]:
-        pypar.send(num.array(submesh["full_quan"][k][p], num.float), p)
+        print 'send full', ++i, k
+        protocol(num.array(submesh["full_quan"][k][p], num.float))
+        pypar.send(num.array(submesh["full_quan"][k][p], num.float), p, bypass=False)
         
     for k in submesh["ghost_quan"]:
-        pypar.send(num.array(submesh["ghost_quan"][k][p], num.float),p)
+        print 'send ghost', ++i, k
+        protocol(num.array(submesh["ghost_quan"][k][p], num.float))
+        pypar.send(num.array(submesh["ghost_quan"][k][p], num.float),p, bypass=False)
         
 
@@ -1192 +1234 @@
     import pypar
     
-    numproc = pypar.size()
+    numprocs = pypar.size()
     myid = pypar.rank()
 
@@ -1211 +1253 @@
     qkeys = pypar.receive(p)
 
-    # receive the number of triangles per processor
-
-    triangles_per_proc = pypar.receive(p)
-
-    # ghost layer width
-
-    submesh_cell["ghost_layer_width"] = pypar.receive(p)
 
     # recieve information about the array sizes
 
-    setup_array = pypar.receive(p)
+    x = num.zeros((9,),num.int)
+    pypar.receive(p, buffer=x,  bypass=True)
+    setup_array = x
 
     no_full_nodes      = setup_array[0]
@@ -1232 +1269 @@
     no_full_commun     = setup_array[7]
     no_quantities      = setup_array[8]
-    
+
+
+    # ghost layer width
+    x = num.zeros((1,),num.int)
+    pypar.receive(p, buffer=x,  bypass=True)
+    submesh_cell["ghost_layer_width"] = x[0]
+
+
+    # receive the number of triangles per processor
+    x = num.zeros((numprocs,),num.int)
+    pypar.receive(p, buffer=x,  bypass=True)
+    triangles_per_proc = x
+
     # receive the full nodes
+#    x = num.zeros((no_full_nodes,),num.int)
+#    pypar.receive(p, buffer=x,  bypass=True)
+#    submesh_cell["full_nodes"] = x
+
 
     submesh_cell["full_nodes"] = pypar.receive(p)
-
     # receive the ghost nodes
 
-    submesh_cell["ghost_nodes"] = pypar.receive(p)
+    submesh_cell["ghost_nodes"] = pypar.receive(p, bypass=False)
     
     # receive the full triangles
 
-    submesh_cell["full_triangles"] = pypar.receive(p)
+    submesh_cell["full_triangles"] = pypar.receive(p, bypass=False)
     
     # receive the ghost triangles
 
-    submesh_cell["ghost_triangles"] = pypar.receive(p)
+    submesh_cell["ghost_triangles"] = pypar.receive(p, bypass=False)
 
     # receive the full boundary
 
-    bnd_c = pypar.receive(p)
+    bnd_c = pypar.receive(p, bypass=False)
 
     submesh_cell["full_boundary"] = {}
@@ -1259 +1311 @@
     # receive the ghost boundary
 
-    bnd_c = pypar.receive(p)
+    bnd_c = pypar.receive(p, bypass=False)
 
     submesh_cell["ghost_boundary"] = {}
@@ -1267 +1319 @@
     # receive the ghost communication pattern
 
-    submesh_cell["ghost_commun"] = pypar.receive(p)
+    submesh_cell["ghost_commun"] = pypar.receive(p, bypass=False)
     
     # receive the full communication pattern
 
-    full_commun = pypar.receive(p)
+    full_commun = pypar.receive(p, bypass=False)
 
     submesh_cell["full_commun"] = {}
@@ -1284 +1336 @@
     
     for i in range(no_quantities):
-        tmp = pypar.receive(p)
+        tmp = pypar.receive(p, bypass=False)
         submesh_cell["full_quan"][qkeys[i]]=num.zeros((no_full_triangles,3), num.float)
         submesh_cell["full_quan"][qkeys[i]][:] = tmp[:]
@@ -1290 +1342 @@
     submesh_cell["ghost_quan"]={}
     for i in range(no_quantities):
-        tmp = pypar.receive(p)
+        tmp = pypar.receive(p, bypass=False)
         submesh_cell["ghost_quan"][qkeys[i]]= num.zeros((no_ghost_triangles,3), num.float)
         submesh_cell["ghost_quan"][qkeys[i]][:] = tmp[:]

trunk/anuga_core/source/anuga_parallel/parallel_api.py

@@ -57 +57 @@
 
 
-
-
-    if not pypar_available: return domain # Bypass
+    if not pypar_available or numprocs == 1 : return domain # Bypass
 
     # For some obscure reason this communication must happen prior to
@@ -75 +73 @@
 
         for p in range(1, numprocs):
+            # FIXME SR: Creates cPickle dump
             send((domain_name, domain_dir, domain_store, \
                   domain_minimum_storable_height, georef, \
@@ -130 +129 @@
         # Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs
         for p in range(1, numprocs):
+            # FIXME SR: Creates cPickle dump
             send(s2p_map, p)
+            # FIXME SR: Creates cPickle dump
             send(p2s_map, p)
 
@@ -250 +251 @@
     if verbose: print 'Distribute submeshes'        
     for p in range(1, numprocs):
-      send_submesh(submesh, triangles_per_proc, p, verbose)
+        send_submesh(submesh, triangles_per_proc, p, verbose)
 
     # Build the local mesh for processor 0

trunk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangular_cross.py

@@ -33 +33 @@
 t0 = time.time()
 
-verbose = False
+verbose = True
 
 #--------------------------------------------------------------------------
@@ -41 +41 @@
     length = 2.0
     width = 2.0
-    dx = dy = 0.005
+    #dx = dy = 0.005
+    dx = dy = 0.01
     domain = rectangular_cross_domain(int(length/dx), int(width/dy),
                                               len1=length, len2=width)
@@ -159 +160 @@
 
 
-#domain.dump_triangulation(filename="rectangular_cross_%g.png"% numprocs)
+if domain.number_of_global_triangles < 50000:
+    print 'Create dump of triangulation for {0} triangles'.format(domain.number_of_global_triangles)
+    domain.dump_triangulation(filename="rectangular_cross_%g.png"% numprocs)
 
 finalize()