Changeset 3105

Timestamp:

Jun 6, 2006, 4:41:39 PM (18 years ago)

Author:

ole

Message:

Attempt at parallelising flagstaff

Location:

production/wollongong_2006

Files:

• project.py (modified) (2 diffs)
• run_flagstaff.py (modified) (5 diffs)

production/wollongong_2006/project.py

@@ -4 +4 @@
 import sys
 from os import sep as s, environ
+from os.path import expanduser
 
 # Making assumptions about the location of scenario data
@@ -18 +19 @@
     
 # Derive subdirectories and filenames
-home = environ['INUNDATIONHOME']     #Sandpit's parent dir
-#if sys.platform == 'win32':
-#    home = environ['INUNDATIONHOME']     #Sandpit's parent dir
-#else:    
-#    home = expanduser('~')
+try:
+    home = environ['INUNDATIONHOME']
+except Exception, e:
+    print 'Environment variable INUNDATIONHOME was not found: %s' %e
+    home = expanduser('~')
+
 
 meshdir = home+s+scenario_dirname+s+'meshes'+s

production/wollongong_2006/run_flagstaff.py

@@ -21 +21 @@
 from os import sep
 from os.path import dirname, basename
+from Numeric import zeros, Float
 
 # Related major packages
@@ -31 +32 @@
 from pmesh.mesh import importUngenerateFile, Segment
 
+# Parallelism
+from pypar_dist import pypar   # The Python-MPI interface
+from parallel.pmesh_divide import pmesh_divide_metis
+from parallel.build_submesh import build_submesh
+from parallel.build_local   import build_local_mesh
+from parallel.build_commun  import send_submesh, rec_submesh, extract_hostmesh
+from parallel.parallel_shallow_water import Parallel_Domain
+
+
 # Application specific imports
 import project
+
+
+#------------------------------------------------------------------------------
+# Read in processor information
+#------------------------------------------------------------------------------
+
+numprocs = pypar.size()
+myid = pypar.rank()
+processor_name = pypar.Get_processor_name()
+print 'I am processor %d of %d on node %s' %(myid, numprocs, processor_name)
+
+bounding_rectangle = zeros(4, Float) # Buffer for results
 
 
@@ -41 +63 @@
 #------------------------------------------------------------------------------
 
-# Create DEM from asc data 
-convert_dem_from_ascii2netcdf(project.demname, use_cache=True, verbose=True)
-
-# Create pts file from DEM
-dem2pts(project.demname,
-        easting_min=project.xllcorner,
-        easting_max=project.xurcorner,
-        northing_min=project.yllcorner,
-        northing_max= project.yurcorner,
-        use_cache=True, 
-        verbose=True)
-
-
-#------------------------------------------------------------------------------
-# Create the triangular mesh based on overall clipping polygon with a tagged
-# boundary and interior regions defined in project.py along with
-# resolutions (maximal area of per triangle) for each polygon
-#------------------------------------------------------------------------------
-
-# Generate basic mesh
-mesh = create_mesh_from_regions(project.bounding_polygon,
-                                boundary_tags=project.boundary_tags,
-                                maximum_triangle_area=project.base_resolution,
-                                interior_regions=project.interior_regions)
-
-# Add buildings
-# This should bind to a Reflective boundary
-mesh.import_ungenerate_file(project.buildings_filename,tag='wall')
-
-# Generate and write mesh to file
-mesh.generate_mesh(maximum_triangle_area=project.base_resolution,
-                   verbose=True)
-
-mesh.export_mesh_file(project.mesh_filename)
-
-
-#------------------------------------------------------------------------------
-# Setup computational domain
-#------------------------------------------------------------------------------
-
-domain = Domain(project.mesh_filename, use_cache = False, verbose = True)
-print domain.statistics()
-
-domain.set_name(project.basename)
-domain.set_datadir(project.outputdir)
-domain.set_quantities_to_be_stored(['stage', 'xmomentum', 'ymomentum'])
+
+max_area = project.base_resolution
+if myid == 0:
+    # Create DEM from asc data 
+    convert_dem_from_ascii2netcdf(project.demname,
+                                  use_cache=True,
+                                  verbose=True)
+
+    # Create pts file from DEM
+    dem2pts(project.demname,
+            easting_min=project.xllcorner,
+            easting_max=project.xurcorner,
+            northing_min=project.yllcorner,
+            northing_max= project.yurcorner,
+            use_cache=True, 
+            verbose=True)
+
+
+    #--------------------------------------------------------------------------
+    # Create the triangular mesh based on overall clipping polygon with a
+    # tagged boundary and interior regions defined in project.py along with
+    # resolutions (maximal area of per triangle) for each polygon
+    #--------------------------------------------------------------------------
+
+
+    ## Generate basic mesh
+    mesh = create_mesh_from_regions(project.bounding_polygon,
+                                    boundary_tags=project.boundary_tags,
+                                    maximum_triangle_area=max_area,
+                                    interior_regions=project.interior_regions)
+    
+    ## Add buildings
+    #dict = importUngenerateFile(project.buildings_filename)
+    #Segment.set_default_tag('wall') # This should bind to a Reflective boundary
+    #mesh.addVertsSegs(dict)
+
+  
+    # Add buildings
+    # This should bind to a Reflective boundary
+    mesh.import_ungenerate_file(project.buildings_filename, tag='wall')
+
+    # Generate and write mesh to file
+    mesh.generate_mesh(maximum_triangle_area=max_area,
+                       verbose=True)
+    mesh.export_mesh_file(project.mesh_filename)
+
+
+    #--------------------------------------------------------------------------
+    # Setup computational domain
+    #--------------------------------------------------------------------------
+
+    domain = Domain(project.mesh_filename, use_cache = False, verbose = True)
+    print domain.statistics()
+
+    domain.set_name(project.basename)
+    domain.set_datadir(project.outputdir)
+    domain.set_quantities_to_be_stored(['stage', 'xmomentum', 'ymomentum'])
+
+    bounding_rectangle[:] = domain.get_extent(absolute=True)
+
+    # Subdivide the mesh
+    print 'Subdivide mesh'
+    nodes, triangles, boundary, triangles_per_proc, quantities = \
+           pmesh_divide_metis(domain, numprocs)
+
+    # Build the mesh that should be assigned to each processor,
+    # this includes ghost nodes and the communicaiton pattern
+    print 'Build submeshes'    
+    submesh = build_submesh(nodes, triangles, boundary,\
+                            quantities, triangles_per_proc)
+
+    # Send the mesh partition to the appropriate processor
+    print 'Distribute submeshes'        
+    for p in range(1, numprocs):
+      send_submesh(submesh, triangles_per_proc, p)
+
+    # Build the local mesh for processor 0
+    points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict = \
+              extract_hostmesh(submesh, triangles_per_proc)
+
+    print 'Communication done'        
+    
+else:
+    # Read in the mesh partition that belongs to this
+    # processor (note that the information is in the
+    # correct form for the GA data structure)
+
+    points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict \
+            = rec_submesh(0)
+
+
+
+
+#------------------------------------------------------------------------------
+# Start the computations on each subpartion
+#------------------------------------------------------------------------------
+
+
+pypar.broadcast(bounding_rectangle, 0)
+
+# Build the domain for this processor
+domain = Parallel_Domain(points, vertices, boundary,
+                         full_send_dict  = full_send_dict,
+                         ghost_recv_dict = ghost_recv_dict)
 
 
@@ -113 +193 @@
                      'side': D,
                      'wall': R,
-                     'ocean': W})
-
-
-                             
+                     'ocean': W,
+                     'ghost': None})
+
+                            
 #------------------------------------------------------------------------------
 # Evolve system through time
@@ -122 +202 @@
 
 t0 = time.time()
-for t in domain.evolve(yieldstep = 1, finaltime = 1200): 
-    domain.write_time()
-    domain.write_boundary_statistics(tags = 'ocean')     
+for t in domain.evolve(yieldstep = 1, finaltime = 1200):
+    if myid == 0:
+        domain.write_time()
+        #domain.write_boundary_statistics(tags = 'ocean')     
   
-print 'That took %.2f seconds' %(time.time()-t0)
+
+if myid == 0:
+    print 'That took %.2f seconds' %(time.time()-t0)
+    print 'Communication time %.2f seconds'%domain.communication_time
+    print 'Reduction Communication time %.2f seconds'\
+          %domain.communication_reduce_time
+    print 'Broadcast time %.2f seconds'\
+          %domain.communication_broadcast_time
+
+pypar.finalize()