Index: /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangular_tsunami.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangular_tsunami.py	(revision 8609)
+++ /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangular_tsunami.py	(revision 8610)
@@ -55,6 +55,6 @@
 #--------------------------------------------------------------------------
 if myid == 0:
-    length = 1.0
-    width = 1.0
+    length = 2.0
+    width = 2.0
     dx = dy = 0.005
     #dx = dy = 0.00125
Index: /trunk/anuga_core/source/anuga_parallel/run_sequential_dist_distribute_rectangular.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_sequential_dist_distribute_rectangular.py	(revision 8609)
+++ /trunk/anuga_core/source/anuga_parallel/run_sequential_dist_distribute_rectangular.py	(revision 8610)
@@ -16,4 +16,5 @@
 import time
 import sys
+import numpy
 
 
@@ -27,5 +28,5 @@
 # Parallel interface
 #---------------------------
-from anuga_parallel.sequential_distribute import sequential_distribute
+from anuga_parallel.sequential_distribute import sequential_distribute_dump
 
 
@@ -35,28 +36,47 @@
 
 #--------------------------------------------------------------------------
+# Setup functions for topograpy etc
+#--------------------------------------------------------------------------
+scale_me=1.0
+
+def topography(x,y):
+	return (-x/2.0 +0.05*numpy.sin((x+y)*200.0))*scale_me
+
+def stagefun(x,y):
+    stge=-0.2*scale_me #+0.01*(x>0.9)
+    #topo=topography(x,y)
+    return stge#*(stge>topo) + (topo)*(stge<=topo)
+
+
+#--------------------------------------------------------------------------
 # Setup Domain only on processor 0
 #--------------------------------------------------------------------------
 myid = 0
-numprocs = 100
-if myid == 0:
-    length = 2.0
-    width = 2.0
-    dx = dy = 0.005  # 640,000
-    dx = dy = 0.00125
-    #dx = dy  = 0.5
-    domain = rectangular_cross_domain(int(length/dx), int(width/dy),
-                                              len1=length, len2=width, verbose=verbose)
+numprocs = 4
 
 
-    print domain.number_of_global_triangles
-    domain.set_store(True)
-    domain.set_quantity('elevation', lambda x,y : -1.0-x )
-    domain.set_quantity('stage', 1.0)
-    domain.set_flow_algorithm('tsunami')
-    domain.set_name('sw_rectangle')
-    #domain.print_statistics()
 
-else:
-    domain = None
+length = 2.0
+width = 2.0
+dx = dy = 0.005  # 640,000
+dx = dy  = 0.05
+domain = rectangular_cross_domain(int(length/dx), int(width/dy),
+                                          len1=length, len2=width, verbose=verbose)
+
+
+print domain.number_of_global_triangles
+
+domain.set_store(True)
+domain.set_flow_algorithm('tsunami')
+domain.set_minimum_allowed_height(0.01)
+domain.set_quantity('elevation',topography)     # Use function for elevation
+domain.get_quantity('elevation').smooth_vertex_values()
+domain.set_quantity('friction',0.03)            # Constant friction
+domain.set_quantity('stage', stagefun)          # Constant negative initial stage
+domain.get_quantity('stage').smooth_vertex_values()
+
+domain.set_name('sw_rectangle')
+#domain.print_statistics()
+
 
 t1 = time.time()
@@ -76,5 +96,5 @@
 #domain = distribute(domain,verbose=verbose)
 
-sequential_distribute(domain,numprocs, verbose = True)
+sequential_distribute_dump(domain, numprocs, verbose = True)
 
 
@@ -84,5 +104,5 @@
     print 'Distribute domain ',t2-t1
     
-if myid == 0 : print 'after parallel domain'
 
 
+
Index: /trunk/anuga_core/source/anuga_parallel/run_sequential_dist_parallel_evolve_rectangular.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_sequential_dist_parallel_evolve_rectangular.py	(revision 8610)
+++ /trunk/anuga_core/source/anuga_parallel/run_sequential_dist_parallel_evolve_rectangular.py	(revision 8610)
@@ -0,0 +1,134 @@
+#########################################################
+#
+#  Main file for parallel mesh testing.
+#
+#  This is a modification of the run_parallel_advection.py
+# file.
+#
+#
+#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
+# June 2005
+#
+#
+#
+#########################################################
+
+import time
+import sys
+import numpy
+import anuga
+
+
+#----------------------------
+# Parallel interface
+#---------------------------
+from anuga_parallel import myid, numprocs, finalize, barrier
+
+from anuga_parallel.sequential_distribute import sequential_distribute_load
+
+
+#--------------------------------------------------------------------------
+# Setup functions for topograpy etc
+#--------------------------------------------------------------------------
+scale_me=1.0
+
+def topography(x,y):
+	return (-x/2.0 +0.05*numpy.sin((x+y)*200.0))*scale_me
+
+def stagefun(x,y):
+    stge=-0.2*scale_me #+0.01*(x>0.9)
+    #topo=topography(x,y)
+    return stge#*(stge>topo) + (topo)*(stge<=topo)
+
+
+
+
+verbose = True
+basename = 'sw_rectangle'
+#basename = 'test'
+
+#--------------------------------------------------------------------------
+# Create domains
+#--------------------------------------------------------------------------
+#import cPickle
+#pickle_name = pickle_base+'_P%g_%g.pickle'% (numprocs,myid)
+#f = file(pickle_name, 'rb')
+# = cPickle.load(f)
+#f.close()
+
+
+t1 = time.time()
+
+if myid == 0 and verbose:
+    print 'Loading DISTRIBUTING DOMAIN'
+    sys.stdout.flush()
+
+#-------------------------------------------------------------------------
+# Distribute domain
+#-------------------------------------------------------------------------
+domain = sequential_distribute_load(filename = basename, verbose=verbose)
+
+
+t2 = time.time()
+
+if myid == 0 :
+    print 'Load Distribute domain ',t2-t1
+    
+if myid == 0 : print 'after parallel domain'
+
+
+#Boundaries
+T = anuga.Transmissive_boundary(domain)
+R = anuga.Reflective_boundary(domain)
+D = anuga.Dirichlet_boundary([-0.1*scale_me,0.,0.])
+
+
+domain.set_boundary( {'left': R, 'right': D, 'bottom': R, 'top': R, 'ghost': None} )
+
+
+if myid == 0 : print 'after set_boundary'
+
+
+yieldstep = 0.2
+finaltime = 20.0
+
+barrier()
+
+t0 = time.time()
+
+#===========================================================================
+# Main Evolve Loop
+#===========================================================================
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    if myid == 0:
+        domain.write_time()
+        sys.stdout.flush()
+						
+
+
+
+for p in range(numprocs):
+    barrier()
+    if myid == p:
+        print 50*'='
+        print 'P%g' %(myid)
+        print 'That took %.2f seconds' %(time.time()-t0)
+        print 'Communication time %.2f seconds'%domain.communication_time
+        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+        sys.stdout.flush()
+
+
+if domain.number_of_global_triangles < 50000:
+    if myid == 0 :
+        print 'Create dump of triangulation for %g triangles' % domain.number_of_global_triangles
+    domain.dump_triangulation(filename="rectangular_cross_%g.png"% numprocs)
+
+
+#--------------------------------------------------
+# Merge the individual sww files into one file
+#--------------------------------------------------
+domain.sww_merge(delete_old=False)
+
+
+finalize()
Index: /trunk/anuga_core/source/anuga_parallel/sequential_distribute.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/sequential_distribute.py	(revision 8609)
+++ /trunk/anuga_core/source/anuga_parallel/sequential_distribute.py	(revision 8610)
@@ -18,5 +18,5 @@
 
 
-def sequential_distribute(domain, numprocs=1, verbose=False, debug=False, parameters = None):
+def sequential_distribute_dump(domain, numprocs=1, verbose=False, debug=False, parameters = None):
     """ Distribute the domain, create parallel domain and pickle result
     """
@@ -122,5 +122,5 @@
 
 
-        args = [points, vertices, boundary]
+        #args = [points, vertices, boundary]
 
         kwargs = {'full_send_dict': full_send_dict,
@@ -139,5 +139,5 @@
                 'ghost_layer_width':  ghost_layer_width}
 
-        parallel_domain = Parallel_domain(*args, **kwargs)
+#        parallel_domain = Parallel_domain(points, vertices, boundary, **kwargs)
 
 
@@ -146,6 +146,6 @@
         # Transfer initial conditions to each subdomain
         #------------------------------------------------------------------------
-        for q in quantities:
-            parallel_domain.set_quantity(q, quantities[q])
+#        for q in quantities:
+#            parallel_domain.set_quantity(q, quantities[q])
 
 
@@ -153,6 +153,6 @@
         # Transfer boundary conditions to each subdomain
         #------------------------------------------------------------------------
-        boundary_map['ghost'] = None  # Add binding to ghost boundary
-        parallel_domain.set_boundary(boundary_map)
+#        boundary_map['ghost'] = None  # Add binding to ghost boundary
+#        parallel_domain.set_boundary(boundary_map)
 
 
@@ -160,11 +160,11 @@
         # Transfer other attributes to each subdomain
         #------------------------------------------------------------------------
-        parallel_domain.set_name(domain_name)
-        parallel_domain.set_datadir(domain_dir)
-        parallel_domain.set_store(domain_store)
-        parallel_domain.set_minimum_storable_height(domain_minimum_storable_height)
-        parallel_domain.set_minimum_allowed_height(domain_minimum_allowed_height)
-        parallel_domain.set_flow_algorithm(domain_flow_algorithm)
-        parallel_domain.geo_reference = georef
+#        parallel_domain.set_name(domain_name)
+#        parallel_domain.set_datadir(domain_dir)
+#        parallel_domain.set_store(domain_store)
+#        parallel_domain.set_minimum_storable_height(domain_minimum_storable_height)
+#        parallel_domain.set_minimum_allowed_height(domain_minimum_allowed_height)
+#        parallel_domain.set_flow_algorithm(domain_flow_algorithm)
+#        parallel_domain.geo_reference = georef
 
 
@@ -173,21 +173,73 @@
         # Now let's store the parallel_domain via cPickle
         #-----------------------------------------------------------------------
+#        import cPickle
+#        pickle_name = domain_name + '_P%g_%g.pickle'% (numprocs,p)
+#        f = file(pickle_name, 'wb')
+#        cPickle.dump(parallel_domain, f, protocol=cPickle.HIGHEST_PROTOCOL)
+#        f.close()
+
+
+        #FIXME SR: Looks like we could reduce storage by a factor of 4 by just
+        # storing the data to create the parallel_domain instead of pickling
+        # a created domain
         import cPickle
         pickle_name = domain_name + '_P%g_%g.pickle'% (numprocs,p)
         f = file(pickle_name, 'wb')
-        cPickle.dump(parallel_domain, f, protocol=cPickle.HIGHEST_PROTOCOL)
-        f.close()
-
-        #FIXME SR: Looks like we could reduce storage by a factor of 4 by just
-        # storing the data to create the parallel_domain instead of pickling
-        # a created domain
-        #pickle_name = 'test_P%g_%g.pickle'% (numprocs,p)
-        #f = file(pickle_name, 'wb')
-        #cPickle.dump( (args, kwargs, quantities), f, protocol=cPickle.HIGHEST_PROTOCOL)
-        #f.close()
+        tostore = (kwargs, points, vertices, boundary, quantities, boundary_map, domain_name, domain_dir, domain_store, domain_minimum_storable_height, \
+                   domain_minimum_allowed_height, domain_flow_algorithm, georef)
+        cPickle.dump( tostore, f, protocol=cPickle.HIGHEST_PROTOCOL)
 
     return
 
 
+def sequential_distribute_load(filename = 'domain', verbose = False):
+
+
+    from anuga_parallel import myid, numprocs
+
+
+    #---------------------------------------------------------------------------
+    # Open pickle files
+    #---------------------------------------------------------------------------
+    import cPickle
+    pickle_name = filename+'_P%g_%g.pickle'% (numprocs,myid)
+    f = file(pickle_name, 'rb')
+    kwargs, points, vertices, boundary, quantities, boundary_map, domain_name, domain_dir, domain_store, domain_minimum_storable_height, \
+                   domain_minimum_allowed_height, domain_flow_algorithm, georef = cPickle.load(f)
+    f.close()
+
+    #---------------------------------------------------------------------------
+    # Create parallel domain
+    #---------------------------------------------------------------------------
+    parallel_domain = Parallel_domain(points, vertices, boundary, **kwargs)
+
+
+    #------------------------------------------------------------------------
+    # Copy in quantity data
+    #------------------------------------------------------------------------
+    for q in quantities:
+        parallel_domain.set_quantity(q, quantities[q])
+
+
+    #------------------------------------------------------------------------
+    # Transfer boundary conditions to each subdomain
+    #------------------------------------------------------------------------
+    boundary_map['ghost'] = None  # Add binding to ghost boundary
+    parallel_domain.set_boundary(boundary_map)
+
+
+    #------------------------------------------------------------------------
+    # Transfer other attributes to each subdomain
+    #------------------------------------------------------------------------
+    parallel_domain.set_name(domain_name)
+    parallel_domain.set_datadir(domain_dir)
+    parallel_domain.set_store(domain_store)
+    parallel_domain.set_minimum_storable_height(domain_minimum_storable_height)
+    parallel_domain.set_minimum_allowed_height(domain_minimum_allowed_height)
+    parallel_domain.set_flow_algorithm(domain_flow_algorithm)
+    parallel_domain.geo_reference = georef
+
+
+    return parallel_domain
 
 def extract_l2g_map(map):
