Index: /trunk/anuga_core/source/anuga_parallel/examples/run_advection.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_advection.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_advection.py	(revision 9424)
@@ -0,0 +1,50 @@
+#import pdb
+#pdb.set_trace()
+
+#========================================================================
+from anuga.config import g, epsilon
+
+#from Numeric import allclose, array, zeros, ones, Float
+
+from anuga.advection import Domain, Transmissive_boundary, Dirichlet_boundary
+
+#from Numeric import array
+import numpy as num
+
+
+from anuga.interface import rectangular_cross
+
+#points, vertices, boundary = rectangular(60, 60)
+points, vertices, boundary = rectangular_cross(10, 10)
+
+#Create advection domain with direction (1,-1)
+domain = Domain(points, vertices, boundary, velocity=[1.0, 0.0])
+
+# Initial condition is zero by default
+
+#domain.initialise_visualiser(scale_z=1.0)
+#domain.visualise_range_z = 0.5
+#domain.visualise_color_stage = True
+
+
+#Boundaries
+T = Transmissive_boundary(domain)
+D = Dirichlet_boundary(num.array([1.0]))
+
+#turn on the visualisation
+rect = [0.0, 0.0, 1.0, 1.0]
+#domain.initialise_visualiser(rect=rect)
+domain.visualise = False
+
+
+domain.default_order = 2
+
+print domain.quantities.keys()
+
+domain.set_boundary( {'left': D, 'right': T, 'bottom': T, 'top': T} )
+domain.check_integrity()
+
+#Check that the boundary value gets propagated to all elements
+for t in domain.evolve(yieldstep = 0.1, finaltime = 1.5):
+    domain.write_time()
+    #pdb.set_trace()
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_advection.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_advection.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_advection.py	(revision 9424)
@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+#########################################################
+#
+#  Main file for parallel mesh testing. Runs an advection
+# flow simulation using a rectangular mesh
+#
+#
+#  Authors: Steve Roberts June 2005
+#  Modified by Linda Stals April 2006
+#
+#
+#########################################################
+
+
+
+# Parallel communication routines
+
+import pypar
+
+#  Mesh partition routines
+
+from parallel_meshes import parallel_rectangle
+
+# Parallel Domain
+ 
+from parallel_advection import Parallel_Domain
+from parallel_advection import Transmissive_boundary
+
+############################
+# Set the initial conditions
+############################
+class Set_Stage:
+    """Set an initial condition with constant water height, for x<x0
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))
+
+###############################
+# Read in processor information
+###############################
+
+numprocs = pypar.size()
+myid = pypar.rank()
+processor_name = pypar.get_processor_name()
+
+N = 5
+M = 2
+
+N = 10
+M = 5
+
+#######################
+# Partition the mesh
+#######################
+
+# Build a unit mesh, subdivide it over numproces processors with each
+# submesh containing M*N nodes
+
+points, vertices, boundary, full_send_dict, ghost_recv_dict =  \
+    parallel_rectangle(N, M, len1_g=1.0)
+
+#print 'ghost ',ghost_recv_dict
+#print 'full  ',full_send_dict
+
+
+
+print "Myid = ", myid, "no points = ", len(points), \
+      "no vertices = ", len(vertices), "no boundaries = ", len(boundary)
+
+###########################################
+# Start the computations on each subpartion
+###########################################
+
+# Create advection domain with direction (1,-1)
+# Initial condition is zero by default
+
+domain = Parallel_Domain(points, vertices, boundary,
+                         full_send_dict, ghost_recv_dict, velocity=[1.0, 0.0])
+
+#print 'ghost ',ghost_recv_dict
+#print 'full  ',full_send_dict
+
+
+# Boundaries
+
+T = Transmissive_boundary(domain)
+
+
+
+
+domain.set_boundary( {'left': T, 'right': T, 'bottom': T, 'top': T, \
+                      'ghost': None} )
+
+
+# Set Evolve parameters
+domain.set_default_order(2)
+domain.set_timestepping_method('rk2')
+
+print domain.get_timestepping_method()
+
+#domain.use_edge_limiter = True
+#domain.tight_slope_limiters = True
+#domain.use_centroid_velocities = False
+
+domain.CFL = 1.0
+
+domain.set_beta(0.8)
+
+
+# Ensure that the domain definitions make sense
+
+domain.check_integrity()
+
+# Set the inititial conditions
+
+domain.set_quantity('stage', Set_Stage(0.2,0.4,1.0))
+
+# Let processor 0 output some timing information
+
+visualise = False
+if visualise:
+    from anuga.visualiser import RealtimeVisualiser
+    vis = RealtimeVisualiser(domain)
+    vis.render_quantity_height("stage", zScale = 5.0, dynamic=True)
+    vis.colour_height_quantity('stage', (0.2, 0.2, 0.8))
+    vis.start()
+    import time
+    time.sleep(2.0)
+    
+
+
+if myid == 0:
+    import time
+    t0 = time.time()
+
+for t in domain.evolve(yieldstep = 5.0, finaltime = 30.0):
+    if myid == 0:
+        domain.write_time()
+        
+    if visualise:
+        vis.update()						
+
+        
+
+if visualise: vis.evolveFinished()
+    
+
+# Output some computation statistics
+
+if myid == 0:
+    print 'That took %.2f seconds' %(time.time()-t0)
+    print 'Communication time %.2f seconds'\
+          %domain.communication_time
+    print 'Reduction Communication time %.2f seconds'\
+          %domain.communication_reduce_time
+        
+  
+if visualise: vis.join()
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_advection_prof.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_advection_prof.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_advection_prof.py	(revision 9424)
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+#########################################################
+#
+#  Main file for parallel mesh testing. Runs an advection
+# flow simulation using a rectangular mesh
+#
+#
+#  Authors: Steve Roberts June 2005
+#  Modified by Linda Stals April 2006
+#
+#
+#########################################################
+
+
+
+# Parallel communication routines
+
+import pypar
+
+#  Mesh partition routines
+
+from parallel_meshes import parallel_rectangle
+
+# Parallel Domain
+ 
+from parallel_advection import Parallel_Domain
+from parallel_advection import Transmissive_boundary
+
+############################
+# Set the initial conditions
+############################
+class Set_Stage:
+    """Set an initial condition with constant water height, for x<x0
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))
+
+###############################
+# Read in processor information
+###############################
+
+numprocs = pypar.size()
+myid = pypar.rank()
+processor_name = pypar.Get_processor_name()
+
+N = 80
+M = 80
+
+#######################
+# Partition the mesh
+#######################
+
+# Build a unit mesh, subdivide it over numproces processors with each
+# submesh containing M*N nodes
+
+points, vertices, boundary, full_send_dict, ghost_recv_dict =  \
+    parallel_rectangle(N, M, len1_g=1.0)
+
+print "Myid = ", myid, "no points = ", len(points), \
+      "no vertices = ", len(vertices), "no boundaries = ", len(boundary)
+
+###########################################
+# Start the computations on each subpartion
+###########################################
+
+# Create advection domain with direction (1,-1)
+# Initial condition is zero by default
+
+domain = Parallel_Domain(points, vertices, boundary,
+                         full_send_dict, ghost_recv_dict, velocity=[1.0, 0.0])
+
+# Boundaries
+
+T = Transmissive_boundary(domain)
+domain.default_order = 2
+domain.set_boundary( {'left': T, 'right': T, 'bottom': T, 'top': T, \
+                      'ghost': None} )
+
+# Ensure that the domain definitions make sense
+
+domain.check_integrity()
+
+# Set the inititial conditions
+
+domain.set_quantity('stage', Set_Stage(0.2,0.4,1.0))
+
+# Let processor 0 output some timing information
+
+if myid == 0:
+    import time
+    t0 = time.time()
+
+# Start the evolve computions
+import hotshot
+profiler = hotshot.Profile("hotshot." + str(numprocs) + "." + str(myid) + ".prof")
+s = '''for t in domain.evolve(yieldstep = 0.1, finaltime = 2.0):
+  if myid == 0:
+    domain.write_time()
+'''
+    
+
+result = profiler.runctx(s, globals(), locals())
+profiler.close()
+
+# for t in domain.evolve(yieldstep = 0.1, finaltime = 3.0):
+#     if myid == 0:
+#         domain.write_time()
+
+# Output some computation statistics
+
+if myid == 0:
+    print 'That took %.2f seconds' %(time.time()-t0)
+    print 'Communication time %.2f seconds'\
+          %domain.communication_time
+    print 'Reduction Communication time %.2f seconds'\
+          %domain.communication_reduce_time
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_boyd_box_op.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_boyd_box_op.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_boyd_box_op.py	(revision 9424)
@@ -0,0 +1,167 @@
+import os.path
+import sys
+
+
+import anuga
+
+from anuga import myid, numprocs, finalize
+from anuga import Inlet_operator, Boyd_box_operator
+
+
+
+
+"""
+This test exercises the parallel culvert
+"""
+verbose = True
+
+    
+
+length = 40.
+width = 16.
+
+dx = dy = 2           # Resolution: Length of subdivisions on both axes
+
+#----------------------------------------------------------------------
+# Setup initial conditions
+#----------------------------------------------------------------------
+
+def topography(x, y):
+    """Set up a weir
+    
+    A culvert will connect either side
+    """
+    # General Slope of Topography
+    z=-x/1000
+    
+    N = len(x)
+    for i in range(N):
+
+       # Sloping Embankment Across Channel
+        if 5.0 < x[i] < 10.1:
+            # Cut Out Segment for Culvert face                
+            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
+               z[i]=z[i]
+            else:
+               z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
+        if 10.0 < x[i] < 12.1:
+           z[i] +=  2.5                    # Flat Crest of Embankment
+        if 12.0 < x[i] < 14.5:
+            # Cut Out Segment for Culvert face                
+            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
+               z[i]=z[i]
+            else:
+               z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
+                   
+        
+    return z
+
+##-----------------------------------------------------------------------
+## Setup domain
+##-----------------------------------------------------------------------
+if myid == 0:
+    points, vertices, boundary = anuga.rectangular_cross(int(length/dx),
+                                                   int(width/dy),
+                                                   len1=length, 
+                                                   len2=width)
+
+    domain = anuga.Domain(points, vertices, boundary)   
+    domain.set_name()                 # Output name output_script_name.sww
+    domain.set_flow_algorithm('1_5')
+else:
+    domain = None
+    
+##-----------------------------------------------------------------------
+## Distribute domain
+##-----------------------------------------------------------------------
+
+
+domain = anuga.distribute(domain)
+domain.dump_triangulation("run_parallel_boyd_box_op_domain.png")
+    
+
+##-----------------------------------------------------------------------
+## Setup boundary conditions
+##-----------------------------------------------------------------------
+    
+domain.set_quantity('elevation', topography) 
+domain.set_quantity('friction', 0.01)         # Constant friction 
+domain.set_quantity('stage',
+                        expression='elevation')   # Dry initial condition
+
+    
+Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
+Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
+domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
+
+
+
+    ################ Define Fractional Operators ##########################
+line0 = [[10.0, 10.0], [30.0, 10.0]]
+#line0 = [[29.0, 10.0], [30.0, 10.0]]
+poly1 = [[0.0, 10.0], [0.0, 15.0], [5.0, 15.0], [5.0, 10.0]]
+Q0 = anuga.file_function('test_hydrograph.tms', quantities=['hydrograph'])
+Q1 = 5.0
+
+samples = 50
+
+
+inlet0 = None
+inlet1 = None
+boyd_box0 = None
+
+inlet0 = Inlet_operator(domain, line0, Q0, logging=True, description='inlet0', verbose = False)
+inlet1 = Inlet_operator(domain, poly1, Q1, logging=True, description='inlet1', verbose = False)
+
+# Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
+
+boyd_box0 = Boyd_box_operator(domain,
+                              end_points=[[9.0, 2.5],[19.0, 2.5]],
+                              losses=1.5,
+                              width=5.0,
+                              apron=5.0,
+                              use_momentum_jet=True,
+                              use_velocity_head=False,
+                              manning=0.013,
+                              logging=True,
+                              description='boyd_box_0',
+                              verbose=False)
+
+if inlet0 is not None and verbose: inlet0.print_statistics()
+if inlet1 is not None and verbose: inlet1.print_statistics()
+if boyd_box0 is not None and verbose: boyd_box0.print_statistics()
+
+
+##-----------------------------------------------------------------------
+## Evolve system through time
+##-----------------------------------------------------------------------
+
+for t in domain.evolve(yieldstep = 2.0, finaltime = 20.0):
+    if myid == 0 and verbose:
+        domain.write_time()
+
+    #print domain.volumetric_balance_statistics()
+
+    stage = domain.get_quantity('stage')
+
+
+    if boyd_box0 is not None and verbose : boyd_box0.print_timestepping_statistics()
+
+    #for i in range(samples):
+    #    if tri_ids[i] >= 0:                
+    #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
+
+    sys.stdout.flush()
+
+    pass
+
+##-----------------------------------------------------------------------
+## Assign/Test Control data
+##-----------------------------------------------------------------------
+
+domain.sww_merge()
+
+finalize()
+
+    
+
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_gate_operator.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_gate_operator.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_gate_operator.py	(revision 9424)
@@ -0,0 +1,192 @@
+import os.path
+import sys
+
+#from anuga.utilities.system_tools import get_pathname_from_package
+#from anuga.geometry.polygon_function import Polygon_function
+        
+#from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
+#from anuga.abstract_2d_finite_volumes.quantity import Quantity
+
+import anuga
+import time
+
+                            
+from math import pi, pow, sqrt
+
+import numpy as num
+
+from anuga import distribute, myid, numprocs, finalize, barrier
+
+from anuga import Inlet_operator, Boyd_box_operator
+
+
+
+#----------------------------------------------------------------------
+# Setup initial conditions
+#----------------------------------------------------------------------
+
+def topography(x, y):
+    """Set up a weir
+
+    A culvert will connect either side
+    """
+    # General Slope of Topography
+    z=-x/1000
+
+    N = len(x)
+    for i in range(N):
+
+        # Sloping Embankment Across Channel
+        if 5.0 < x[i] < 10.1:
+            # Cut Out Segment for Culvert face
+            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0:
+                z[i]=z[i]
+            else:
+                z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
+        if 10.0 < x[i] < 12.1:
+            z[i] +=  2.5                    # Flat Crest of Embankment
+        if 12.0 < x[i] < 14.5:
+            # Cut Out Segment for Culvert face
+            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
+                z[i]=z[i]
+            else:
+                z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
+
+    return z
+
+  
+
+length = 40.
+width = 15.
+
+dx = dy = 0.5          # Resolution: Length of subdivisions on both axes
+
+
+if myid == 0:
+
+    points, vertices, boundary = anuga.rectangular_cross(int(length/dx),
+                                                         int(width/dy),
+                                                         len1=length, 
+                                                         len2=width)
+    domain = anuga.Domain(points, vertices, boundary)
+    domain.set_name()                 # Output name
+    domain.set_flow_algorithm('DE0')
+    domain.set_store_vertices_uniquely(True)
+    domain.set_quantity('elevation', topography)
+    domain.set_quantity('friction', 0.01)         # Constant friction
+    domain.set_quantity('stage',
+                        expression='elevation')   # Dry initial condition
+
+
+
+else:
+
+    domain = None
+
+
+domain = distribute(domain, verbose = True)
+
+#domain.set_store_vertices_uniquely(False)
+
+
+gate = Boyd_box_operator(domain,
+                            end_points=[[9.0, 2.5],[13.0, 2.5]],
+                            losses=1.5,
+                            width=1.5,
+                            height = 0.0001,
+                            apron=5.0,
+                            use_momentum_jet=True,
+                            use_velocity_head=False,
+                            manning=0.013,
+                            verbose=False)
+
+# Close the gate
+if gate is not None:
+    gate.set_culvert_height(0.0)
+
+line = [[0.0, 5.0], [0.0, 10.0]]
+Q = 1.0
+in0 = Inlet_operator(domain, line, Q)
+
+
+
+
+##-----------------------------------------------------------------------
+## Setup boundary conditions
+##-----------------------------------------------------------------------
+
+## Inflow based on Flow Depth and Approaching Momentum
+Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
+
+domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
+
+
+##-----------------------------------------------------------------------
+## Evolve system through time
+##-----------------------------------------------------------------------
+barrier()
+
+t0 = time.time()
+
+for t in domain.evolve(yieldstep = 1.0, finaltime = 50):
+
+    if myid == 0:
+        print 80*'='
+        domain.write_time()
+
+    #================================================
+    # gate operator lives on a number of processors
+    #
+    # Need to run parallel gate operations on all
+    # processors where gate lives.
+    # But true result only on master processor.
+    #================================================
+    if gate is not None:
+        [s0, s1] = gate.get_enquiry_stages()
+        [d0, d1] = gate.get_enquiry_depths()
+        [e0, e1] = gate.get_enquiry_elevations()
+        [i0, i1] = gate.get_enquiry_invert_elevations()
+        [w0, w1] = gate.get_enquiry_water_depths()
+
+        output = gate.discharge_routine()
+
+        if myid == gate.get_master_proc():
+            print 'myid ', myid, s0,s1
+            print 'myid ', myid, d0,d1
+            print 'myid ', myid, e0,e1
+            print 'myid ', myid, i0,i1
+            print 'myid ', myid, w0,w1
+
+            print 'myid ',myid, output
+
+
+            if d0 > 0.2: gate.set_culvert_height(10.0)
+
+            
+    
+
+barrier()
+
+
+domain.sww_merge(delete_old=True)
+
+
+
+
+barrier()
+
+for p in range(numprocs):
+    if myid == p:
+        print 'Processor %g ' %myid
+        print 'That took %.2f seconds' %(time.time()-t0)
+        print 'Communication time %.2f seconds'%domain.communication_time
+        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+    else:
+        pass
+
+    barrier()
+
+
+finalize()
+
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_job.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_job.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_job.py	(revision 9424)
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+"""Run job in parallel using PBS batch system with
+   specified number of processors
+
+   Syntax
+     run_parallel_job.py 4 <prog>
+"""
+
+queue = 'normal'
+walltime = '00:10:00'
+vmem = '1Mb'
+
+import sys, string, time, os
+arg = sys.argv
+command = arg[0]
+ext = '.py'
+
+if len(arg)<3:
+    print __doc__
+    print 'Usage:'
+    print command + ' p <filename>' + ext
+    sys.exit()
+
+numproc = int(arg[1])    
+progname = arg[2]
+
+
+#Create batchfile
+
+s = """
+#!/bin/csh
+
+# This file is automatically generated - don't edit
+#
+
+#PBS -wd  
+#PBS -q %s
+#PBS -l ncpus=%d
+#PBS -l walltime=%s,vmem=%s
+#PBS -l software=python
+
+prun %s""" %(queue, numproc, walltime, vmem, progname)
+
+
+scriptname = progname + '.run'
+fid = open(scriptname, 'w')
+fid.write(s)
+fid.close()
+
+#Cleanup
+os.system('/bin/rm *.o* *.e*')
+
+# Run
+os.system('qsub %s' %scriptname)
+os.system('/bin/rm %s' %scriptname)
+
+#Create check scripts
+fid = open('q', 'w')
+fid.write("""#!/bin/csh
+#This file is generated automatically.
+#Any modifications will be lost.
+#Edit runjob.py instead.
+echo '--------------------------------------------'
+nqstat %s\n""" %queue)
+fid.close()
+
+#fid = open('o', 'w')
+#fid.write("""#!/bin/csh
+##This file is generated automatically.
+##Any modifications will be lost.
+##Edit runjob.py instead.
+#cat %s.e* %s.o*\n""" %((progname,)*2))
+#fid.close()
+
+
+fid = open('o', 'w')
+fid.write("""#!/bin/csh
+#This file is generated automatically.
+#Any modifications will be lost.
+#Edit runjob.py instead.
+cat *.e* *.o*\n""")
+
+fid.close()
+
+os.system('chmod +x o q')
+
+
+
+
+
+
+
+
+
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_merimbula_test.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_merimbula_test.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_merimbula_test.py	(revision 9424)
@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+#########################################################
+#
+#  Main file for parallel mesh testing. Runs an advection
+# flow simulation using a rectangular mesh
+#
+#  This is a modification of the run_parallel_advection.py
+# file
+#
+#
+# *) The (new) files that have been added to manage the
+# grid partitioning are
+#    +) pmesh_divide_metis.py: subdivide a pmesh
+#    +) build_submesh.py: build the submeshes on the host
+# processor.
+#    +) build_local.py: build the GA mesh datastructure
+# on each processor.
+#    +) build_commun.py: handle the communication between
+# the host and processors
+#
+#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
+# June 2005
+#
+#
+#
+#########################################################
+
+import pypar    # The Python-MPI interface
+import time
+
+from sys import path
+
+print path
+
+# Numeric arrays
+from Numeric import array, zeros, Float
+
+from print_stats import print_test_stats, build_full_flag
+
+from anuga.abstract_2d_finite_volumes.pmesh2domain\
+     import pmesh_to_domain_instance
+from anuga.advection.advection import Domain as Advection_Domain
+from parallel_advection import Parallel_Domain
+
+from anuga.abstract_2d_finite_volumes.generic_boundary_conditions\
+     import Transmissive_boundary
+
+# mesh partition routines
+
+from pmesh_divide  import pmesh_divide_metis
+from build_submesh import build_submesh
+from build_local   import build_local_mesh
+from build_commun  import send_submesh, rec_submesh, extract_submesh
+
+
+
+class Set_Stage:
+    """Set an initial condition with constant water height, for x<x0
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))
+
+# read in the processor information
+
+numprocs = pypar.size()
+myid = pypar.rank()
+processor_name = pypar.Get_processor_name()
+
+
+#-------
+# Domain
+rect = zeros( 4, Float) # Buffer for results
+
+if myid == 0:
+
+    # read in the test files
+
+    filename = 'test-100.tsh'
+#    filename = 'merimbula_10785.tsh'
+    nx = numprocs
+    ny = 1
+    if nx*ny != numprocs:
+        print "WARNING: number of subboxes is not equal to the number of proc"
+
+    domain_full = pmesh_to_domain_instance(filename, Advection_Domain)
+    domain_full.set_quantity('stage', Set_Stage(200.0,300.0,1.0))
+#    domain_full.set_quantity('stage', Set_Stage(756000.0,756500.0,4.0))
+
+    nodes, triangles, boundary, triangles_per_proc, quantities  =\
+            pmesh_divide_metis(domain_full, numprocs)
+    
+    # subdivide the mesh
+    rect = array(domain_full.xy_extent, Float)
+#    rect = array(rect, Float)
+
+    submesh = build_submesh(nodes, triangles, boundary, quantities, \
+                            triangles_per_proc)
+
+    # send the mesh partition to the appropriate processor
+
+    for p in range(1, numprocs):
+      send_submesh(submesh, triangles_per_proc, p)
+
+    # Build the local mesh for processor 0
+    
+    points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict = \
+              extract_submesh(submesh, triangles_per_proc)
+
+# read in the mesh partition that belongs to this
+# processor (note that the information is in the
+# correct form for the GA data structure
+
+else:
+    [points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict] = \
+             rec_submesh(0)
+
+pypar.broadcast(rect,0)
+
+domain = Parallel_Domain(points, vertices, boundary,
+                                   full_send_dict  = full_send_dict,
+                                   ghost_recv_dict = ghost_recv_dict,
+                                   velocity = [0.1,0.0])
+
+# Make a notes of which triangles are full and which are ghost
+
+tri_full_flag = build_full_flag(domain, ghost_recv_dict)
+
+#domain.initialise_visualiser(rect=rect)
+
+#Boundaries
+
+T = Transmissive_boundary(domain)
+#R = Reflective_boundary(domain)
+domain.set_boundary( {'outflow': T, 'inflow': T, 'inner':T, 'exterior': T, 'open':T, 'ghost':None} )
+
+
+
+domain.set_quantity('stage', quantities['stage'])
+
+#---------
+# Evolution
+t0 = time.time()
+
+
+try:
+    domain.initialise_visualiser(rect=rect)
+    #domain.visualiser.coloring['stage'] = True
+    #domain.visualiser.scale_z['stage'] = 0.2
+except:
+    print 'No visualiser'
+
+
+from norms import linf_norm
+
+yieldstep = 1
+finaltime = 200
+
+#yieldstep = 1000
+#finaltime = 50000
+
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    if myid == 0:
+        domain.write_time()
+    print_test_stats(domain, tri_full_flag)
+
+if myid == 0:
+    print 'That took %.2f seconds' %(time.time()-t0)
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_merimbula.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_merimbula.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_merimbula.py	(revision 9424)
@@ -0,0 +1,200 @@
+
+"""Run parallel shallow water domain.
+
+   run using command like:
+
+   mpirun -np m python run_parallel_sw_merimbula.py
+
+   where m is the number of processors to be used.
+   
+   Will produce sww files with names domain_Pn_m.sww where m is number of processors and
+   n in [0, m-1] refers to specific processor that owned this part of the partitioned mesh.
+"""
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+
+import os
+import sys
+import time
+import numpy as num
+
+#------------------------
+# ANUGA Modules
+#------------------------
+	
+from anuga import Domain
+from anuga import Reflective_boundary
+from anuga import Dirichlet_boundary
+from anuga import Time_boundary
+from anuga import Transmissive_boundary
+from anuga import Transmissive_n_momentum_zero_t_momentum_set_stage_boundary
+
+from anuga import rectangular_cross
+from anuga import create_domain_from_file
+
+
+from anuga_parallel import distribute, myid, numprocs, finalize, barrier
+
+
+#--------------------------------------------------------------------------
+# Setup parameters
+#--------------------------------------------------------------------------
+
+#mesh_filename = "merimbula_10785_1.tsh" ; x0 = 756000.0 ; x1 = 756500.0; yieldstep = 10; finaltime = 100
+mesh_filename = "merimbula_17156.tsh"   ; x0 = 756000.0 ; x1 = 756500.0; yieldstep = 50; finaltime = 500
+#mesh_filename = "merimbula_43200_1.tsh"   ; x0 = 756000.0 ; x1 = 756500.0; yieldstep = 50; finaltime = 500
+#mesh_filename = "test-100.tsh" ; x0 = 200.0 ; x1 = 300.0; yieldstep = 1; finaltime = 10
+#mesh_filename = "test-20.tsh" ; x0 = 250.0 ; x1 = 350.0; yieldstep = 1; finaltime = 50
+
+verbose = False
+
+#--------------------------------------------------------------------------
+# Setup procedures
+#--------------------------------------------------------------------------
+class Set_Stage:
+    """Set an initial condition with constant water height, for x0<x<x1
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))+1.0
+
+
+class Set_Elevation:
+    """Set an elevation
+    """
+
+    def __init__(self, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return x/self.h
+    
+
+#--------------------------------------------------------------------------
+# Setup Domain only on processor 0
+#--------------------------------------------------------------------------
+if myid == 0:
+    domain = create_domain_from_file(mesh_filename)
+    domain.set_quantity('stage', Set_Stage(x0, x1, 1.0))
+    #domain.set_quantity('stage', 0.0)
+    #domain.set_datadir('.')
+    domain.set_name('merimbula_new')
+    domain.set_store(True)
+    #domain.set_store_vertices_smoothly(True)
+    domain.set_store_vertices_smoothly(False)
+    #domain.set_quantity('elevation', Set_Elevation(500.0))
+
+    #print domain.statistics()
+    #print domain.get_extent()
+    #print domain.get_extent(absolute=True)
+    #print domain.geo_reference
+else:
+    domain = None
+
+#--------------------------------------------------------------------------
+# Distribute sequential domain on processor 0 to other processors
+#--------------------------------------------------------------------------
+
+if myid == 0 and verbose: print 'DISTRIBUTING DOMAIN'
+domain = distribute(domain, verbose=verbose)
+
+#--------------------------------------------------------------------------
+# On all processors, setup evolve parameters for domains on all processors
+# (all called "domain"
+#--------------------------------------------------------------------------
+
+domain.set_flow_algorithm('2_0')
+domain.set_store_centroids()
+
+#for p in range(numprocs):
+#    if myid == p:
+#        print 'P%d'%p
+#        print domain.get_extent()
+#        print domain.get_extent(absolute=True)
+#        print domain.geo_reference
+#        print domain.s2p_map
+#        print domain.p2s_map
+#        print domain.tri_l2g
+#        print domain.node_l2g
+#    else:
+#        pass
+#
+#    barrier()
+
+
+domain.set_quantities_to_be_stored({'elevation':1,
+                                    'friction':1,
+                                    'stage':2,
+                                    'xmomentum':2,
+                                    'ymomentum':2})
+                                 
+#------------------------------------------------------------------------------
+# Setup boundary conditions
+# This must currently happen *after* domain has been distributed
+#------------------------------------------------------------------------------
+Br = Reflective_boundary(domain)      # Solid reflective wall
+
+from math import sin
+Bts = Transmissive_n_momentum_zero_t_momentum_set_stage_boundary(domain, lambda t: 10*sin(t/60))
+
+domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Bts})
+
+
+#bdy_ids = domain.tag_boundary_cells['open']
+#vol_ids = domain.boundary_cells[bdy_ids]
+#edge_ids = domain.boundary_edges[bdy_ids]
+
+#print domain.mesh.edge_midpoint_coordinates[3*vol_ids+edge_ids]
+
+
+
+#------------------------------------------------------------------------------
+# Evolution
+#------------------------------------------------------------------------------
+if myid == 0 and verbose: print 'EVOLVE'
+
+barrier()
+t0 = time.time()
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    #print 'P:'+ str(myid) + ' '+ domain.timestepping_statistics()
+    if myid == 0:
+        domain.write_time()
+
+
+barrier()
+
+for p in range(numprocs):
+    if myid == p:
+        print 'Processor %g ' %myid
+        print 'That took %.2f seconds' %(time.time()-t0)
+        print 'Communication time %.2f seconds'%domain.communication_time
+        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+    else:
+        pass
+
+    barrier()
+
+
+#--------------------------------------------------
+# Merge the individual sww files into one file
+#--------------------------------------------------
+domain.sww_merge(delete_old=True)
+
+
+domain.dump_triangulation()
+
+finalize()
+
+
+
+
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_merimbula_profile.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_merimbula_profile.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_merimbula_profile.py	(revision 9424)
@@ -0,0 +1,186 @@
+
+"""Run parallel shallow water domain.
+
+   run using command like:
+
+   mpirun -np m python run_parallel_sw_merimbula.py
+
+   where m is the number of processors to be used.
+   
+   Will produce sww files with names domain_Pn_m.sww where n is number of processors and
+   m in [0, n-1] refers to specific processor that owned this part of the partitioned mesh.
+"""
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+
+import os
+import sys
+import time
+import numpy as num
+
+#------------------------
+# ANUGA Modules
+#------------------------
+	
+from anuga import Domain
+from anuga import Reflective_boundary
+from anuga import Dirichlet_boundary
+from anuga import Time_boundary
+from anuga import Transmissive_boundary
+
+from anuga import rectangular_cross
+from anuga import create_domain_from_file
+
+
+from anuga_parallel import distribute, myid, numprocs, finalize, barrier
+
+
+#--------------------------------------------------------------------------
+# Setup parameters
+#--------------------------------------------------------------------------
+
+#mesh_filename = "merimbula_10785_1.tsh" ; x0 = 756000.0 ; x1 = 756500.0
+mesh_filename = "merimbula_43200.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
+#mesh_filename = "test-100.tsh" ; x0 = 0.25 ; x1 = 0.5
+
+finaltime = 500
+yieldstep = 50
+verbose = True
+
+#--------------------------------------------------------------------------
+# Setup procedures
+#--------------------------------------------------------------------------
+class Set_Stage:
+    """Set an initial condition with constant water height, for x0<x<x1
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1)) + 1.0
+
+#--------------------------------------------------------------------------
+# Setup Domain only on processor 0
+#--------------------------------------------------------------------------
+if myid == 0:
+    domain = create_domain_from_file(mesh_filename)
+    domain.set_quantity('stage', Set_Stage(x0, x1, 2.0))
+else:
+    domain = None
+
+#--------------------------------------------------------------------------
+# Distribute sequential domain on processor 0 to other processors
+#--------------------------------------------------------------------------
+
+if myid == 0 and verbose: print 'DISTRIBUTING DOMAIN'
+domain = distribute(domain)
+
+#domain.smooth = False
+barrier()
+for p in range(numprocs):
+    if myid == p:
+        print 'Process ID %g' %myid
+        print 'Number of triangles %g ' %domain.get_number_of_triangles()
+
+    barrier()
+
+
+domain.set_flow_algorithm(2.0)
+
+if myid == 0:
+    domain.print_algorithm_parameters()
+
+barrier()
+
+domain.set_name('meribula')
+
+#------------------------------------------------------------------------------
+# Setup boundary conditions
+# This must currently happen *after* domain has been distributed
+#------------------------------------------------------------------------------
+Br = Reflective_boundary(domain)      # Solid reflective wall
+
+domain.set_boundary({'exterior' :Br, 'open' :Br})
+
+
+barrier()
+for p in range(numprocs):
+    if myid == p:
+        print domain.boundary_statistics()
+
+    barrier()
+
+#------------------------------------------------------------------------------
+# Evolution
+#------------------------------------------------------------------------------
+if myid == 0 and verbose: print 'EVOLVE'
+
+t0 = time.time()
+
+
+s = """
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    if myid == 0:
+        domain.write_time()
+"""
+
+
+# Profiling
+import cProfile
+prof_file = 'evolve-prof'+ str(numprocs) + '_' + str(myid) + '.dat'
+cProfile.run(s,prof_file)
+
+
+barrier()
+
+if myid == 0:
+    import pstats
+    p = pstats.Stats(prof_file)
+    #p.sort_stats('cumulative').print_stats(25)
+
+
+    p.sort_stats('time').print_stats(25)
+
+barrier()
+
+
+if myid == 1:
+    import pstats
+    p = pstats.Stats(prof_file)
+    #p.sort_stats('cumulative').print_stats(25)
+
+
+    p.sort_stats('time').print_stats(25)
+
+    #p.print_stats()
+
+# Run evolve loop
+#result = profiler.runctx(s, globals(), locals())
+
+#result.dump_stats("profile." + str(numprocs) + "." + str(myid) + ".dat")
+#profiler.close()
+
+barrier()
+
+for p in range(numprocs):
+    if myid == p:
+        print 'Process ID %g' %myid
+        print 'Number of processors %g ' %numprocs
+        print 'That took %.2f seconds' %(time.time()-t0)
+        print 'Communication time %.2f seconds'%domain.communication_time
+        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+
+    barrier()
+
+
+finalize()
+
+
+
+
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_merimbula_test.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_merimbula_test.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_merimbula_test.py	(revision 9424)
@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+#########################################################
+#
+#  Main file for parallel mesh testing.
+#
+#  This is a modification of the run_parallel_advection.py
+# file.
+#
+#
+# *) The (new) files that have been added to manage the
+# grid partitioning are
+#    +) pmesh_divide_metis.py: subdivide a pmesh
+#    +) build_submesh.py: build the submeshes on the host
+# processor.
+#    +) build_local.py: build the GA mesh datastructure
+# on each processor.
+#    +) build_commun.py: handle the communication between
+# the host and processors
+#
+# *) Things still to do:
+#    +) Overlap the communication and computation: The
+# communication routines in build_commun.py should be
+# interdispersed in the build_submesh.py and build_local.py
+# files. This will overlap the communication and
+# computation and will be far more efficient. This should
+# be done after more testing and there more confidence in
+# the subpartioning.
+#    +) Much more testing especially with large numbers of
+# processors.
+#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
+# June 2005
+#
+#
+#
+#########################################################
+import sys
+import pypar    # The Python-MPI interface
+import time
+
+# Numeric arrays
+import numpy as num
+#from numpy import array, zeros, float
+
+# Print debugging information
+from print_stats import print_test_stats, build_full_flag
+
+# pmesh
+from anuga.shallow_water import Domain
+from parallel_shallow_water import Parallel_domain
+from anuga.abstract_2d_finite_volumes.pmesh2domain\
+     import pmesh_to_domain_instance
+
+# Reuse previous mesh import
+from anuga.caching import cache
+
+# Mesh partition routines
+from distribute_mesh  import pmesh_divide_metis
+from distribute_mesh  import build_submesh
+from distribute_mesh  import build_local_mesh
+from distribute_mesh  import send_submesh, rec_submesh, extract_submesh
+
+
+###############################
+# Read in processor information
+###############################
+
+numprocs = pypar.size()
+myid = pypar.rank()
+processor_name = pypar.get_processor_name()
+
+############################
+# Set the initial conditions
+############################
+
+rect = num.zeros( 4, num.float) # Buffer for results
+
+class Set_Stage:
+    """Set an initial condition with constant water height, for x<x0
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))
+
+#######################
+# Partition the domain
+#######################
+
+if myid == 0:
+
+    # Read in the test files
+
+#    filename = 'test-100.tsh'
+#    filename = 'merimbula_10785_1.tsh'
+    filename = 'merimbula_43200.tsh'
+
+    # Build the whole domain
+    
+    domain_full = pmesh_to_domain_instance(filename, Domain)
+
+#    domain_full = cache(pmesh_to_domain_instance,
+#               (filename, Domain),
+#              dependencies = [filename])
+
+    rect = num.array(domain_full.get_extent(), num.float)
+    print rect
+
+    # Initialise the wave
+
+    #domain_full.set_quantity('stage', Set_Stage(200.0,300.0,1.0))
+    domain_full.set_quantity('stage', Set_Stage(756000.0,756500.0,2.0))
+#    domain_full.set_quantity('stage', Set_Stage(756000.0,756500.0,0.0))
+
+    # Subdivide the domain
+
+    # Note the different arguments compared with pmesh_divide,
+    # pmesh_divide_steve etc.
+    
+    nodes, triangles, boundary, triangles_per_proc, quantities = \
+         pmesh_divide_metis(domain_full, numprocs)
+
+    print triangles_per_proc
+    
+    rect = num.array(domain_full.get_extent(), num.float)
+
+    submesh = build_submesh(nodes, triangles, boundary,\
+                            quantities, triangles_per_proc)
+
+    # Send the mesh partition to the appropriate processor
+
+    for p in range(1, numprocs):
+      send_submesh(submesh, triangles_per_proc, p)
+
+    # Build the local mesh for processor 0
+
+    points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict = \
+             extract_submesh(submesh, triangles_per_proc)
+
+# Read in the mesh partition that belongs to this
+# processor (note that the information is in the
+# correct form for the GA data structure
+
+else:
+    points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict , \
+            no_full_nodes, no_full_trigs = rec_submesh(0)
+
+
+###########################################
+# Start the computations on each subpartion
+###########################################
+
+#if myid == 0:
+#    print 'ghost'
+#    print ghost_recv_dict
+#processor_name
+#if myid == 0:
+#    print 'full'
+#    print full_send_dict
+
+# The visualiser needs to know the size of the whole domain
+
+pypar.broadcast(rect,0)
+
+domain = Parallel_domain(points, vertices, boundary,
+                         full_send_dict  = full_send_dict,
+                         ghost_recv_dict = ghost_recv_dict)
+
+# Make a note of which triangles are full and which are ghost
+
+tri_full_flag = build_full_flag(domain, ghost_recv_dict)
+
+try:
+    #domain.initialise_visualiser(rect=rect)
+    #domain.visualiser.coloring['stage'] = True
+    #domain.visualiser.scale_z['stage'] = 0.2
+    #domain.visualiser.scale_z['elevation'] = 0.05
+    pass
+except:
+    print 'No visualiser'
+
+
+
+domain.default_order = 1
+
+#Boundaries
+from anuga.interface import Transmissive_boundary, Reflective_boundary
+
+T = Transmissive_boundary(domain)
+R = Reflective_boundary(domain)
+domain.set_boundary( {'outflow': R, 'inflow': R, 'inner':R, 'exterior': R, 'open':R, 'ghost':None} )
+
+
+domain.set_quantity('stage', quantities['stage'])
+domain.set_quantity('elevation', quantities['elevation'])
+
+domain.store = False
+
+#---------
+# Evolution
+t0 = time.time()
+
+print 'Processor %d on %s: No of elements %d'%(domain.processor,processor_name,domain.number_of_elements)
+yieldstep = 50.0
+finaltime = 500.0
+
+#yieldstep = 1000
+#finaltime = 40000
+
+#yieldstep = 1
+#finaltime = 1
+#processor_name
+#for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+#    if myid == 0:
+#        domain.write_time()
+        #print 'Processor %d, Integral of stage %d'%\
+        #       (domain.processor,domain.quantities['stage'].get_integral())
+        #    print_test_stats(domain, tri_full_flag)
+
+
+# Profiling
+#import profile
+#profiler = profile.Profile()
+#result.dump_stats("profile." + str(numprocs) + "." + str(myid) + ".dat")
+
+## #New hotshot profiling
+## import hotshot
+## profiler = hotshot.Profile("hotshot." + str(numprocs) + "." + str(myid) + ".prof")
+## s = '''for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+##   if myid == 0:
+##     domain.write_time()
+##   print_test_stats(domain, tri_full_flag)
+
+## '''
+## result = profiler.runctx(s, globals(), locals())
+## profiler.close()
+
+#from vtk_realtime_visualiser import Visualiser
+#V = Visualiser(domain,default_scale_z=100.0)
+#V.coloring['stage'] = True
+#V.coloring['elevation'] = False
+#V.setup['elevation']=True
+#V.updating['stage']=True
+#V.qcolor['stage'] = (0.1,0.4,0.99)
+
+
+#V.start()
+#V.idle.wait()
+#V.idle.clear()
+
+
+
+
+
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    if myid == 0:
+        domain.write_time()
+        #print_test_stats(domain, tri_full_flag)
+
+#    V.redraw_ready.set()
+#    V.idle.wait()
+#    V.idle.clear()
+#    V.unpaused.wait()
+
+
+#print 'P%d: That took %.2f seconds' %(myid, time.time()-t0)
+#print 'P%d: Communication time %.2f seconds' %(myid, domain.communication_time)
+#print 'P%d: Reduction Communication time %.2f seconds' %(myid, domain.communication_reduce_time)
+#print 'P%d: Broadcast time %.2f seconds' %(myid, domain.communication_broadcast_time)
+
+
+
+if myid == 0:
+    print 'That took %.2f seconds' %(time.time()-t0)
+    print 'Communication time %.2f seconds'%domain.communication_time
+    print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+    print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+
+
+pypar.finalize()
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangle.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangle.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangle.py	(revision 9424)
@@ -0,0 +1,162 @@
+#########################################################
+#
+#  Main file for parallel mesh testing.
+#
+#  This is a modification of the run_parallel_advection.py
+# file.
+#
+#
+#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
+# June 2005
+#
+#
+#
+#########################################################
+
+import time
+
+import numpy as num
+
+from print_stats import print_test_stats, build_full_flag
+
+#----------------------------
+# Sequential interface
+#---------------------------
+from anuga import Domain
+from anuga import Transmissive_boundary, Reflective_boundary
+
+#----------------------------
+# Parallel interface
+#---------------------------
+from anuga_parallel import Parallel_shallow_water_domain
+from anuga_parallel import parallel_rectangle
+from anuga_parallel import myid, numprocs, finalize, get_processor_name
+
+
+processor_name = get_processor_name()
+
+M = 50
+N = M*numprocs
+
+
+if myid == 0:
+    print 'N == %d' %N
+
+points, vertices, boundary, full_send_dict, ghost_recv_dict =\
+        parallel_rectangle(N, M, len1_g=1.0*numprocs, len2_g = 1.0)
+
+print "Myid = ", myid, "no points = ", len(points), \
+      "no vertices = ", len(vertices), "no boundaries = ", len(boundary)
+
+###########################################
+# Start the computations on each subpartion
+###########################################
+
+domain = Parallel_shallow_water_domain(points, vertices, boundary,
+                                       full_send_dict  = full_send_dict,
+                                       ghost_recv_dict = ghost_recv_dict)
+
+
+print 'after parallel domain'
+
+
+
+domain.set_name('sw_rectangle')
+
+#Boundaries
+T = Transmissive_boundary(domain)
+R = Reflective_boundary(domain)
+
+
+domain.set_boundary( {'left': R, 'right': R, 'bottom': R, 'top': R, 'ghost': None} )
+
+
+print 'after set_boundary'
+
+
+
+domain.check_integrity()
+
+class Set_Stage:
+    """Set an initial condition with constant water height, for x<x0
+    """
+
+    def __init__(self, x0=0.25, x1=0.75, y0=0.0, y1=1.0, h=5.0, h0=0.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.y0 = y0
+        self.y1 = y1
+        self.h  = h
+        self.h0 = h0
+
+    def __call__(self, x, y):
+        return self.h0 + self.h*((x>self.x0)&(x<self.x1)&(y>self.y0)&(y<self.y1))
+
+print 'after check_integrity'
+
+domain.set_quantity('stage', Set_Stage(0.2, 0.4, 0.25, 0.75, 1.0, 0.00))
+
+
+print 'after set quantity'
+
+# Set Evolve parameters
+domain.set_default_order(2)
+domain.set_timestepping_method('rk2')
+domain.set_CFL(0.7)
+domain.set_beta(1.5)
+
+#print domain.get_timestepping_method()
+#domain.use_edge_limiter = True
+#domain.tight_slope_limiters = True
+#domain.use_centroid_velocities = False
+
+
+print 'after evolve parameters'
+
+
+import pdb; pdb.set_trace()
+
+if myid == 0:
+    import time
+    t0 = time.time()
+
+
+# Turn on the visualisation
+visualise = False
+if visualise:
+    from anuga.visualiser import RealtimeVisualiser
+    vis = RealtimeVisualiser(domain)
+    vis.render_quantity_height("elevation", offset=0.001, dynamic=False)
+    vis.render_quantity_height("stage", dynamic=True)
+    vis.colour_height_quantity('stage', (0.2, 0.2, 0.8))
+    vis.start()
+    import time
+    time.sleep(2.0)
+
+
+
+yieldstep = 0.05
+finaltime = 2.0
+
+#Check that the boundary value gets propagated to all elements
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    if myid == 0:
+        domain.write_time()
+    #print_test_stats(domain, tri_full_flag)
+    if visualise:
+        vis.update()						
+
+
+
+if visualise: vis.evolveFinished()
+
+if myid == 0:
+    print 'That took %.2f seconds' %(time.time()-t0)
+    print 'Communication time %.2f seconds'%domain.communication_time
+    print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+    print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+
+
+if visualise: vis.join()
+
+finalize()
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangular_cross.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangular_cross.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangular_cross.py	(revision 9424)
@@ -0,0 +1,162 @@
+#########################################################
+#
+#  Main file for parallel mesh testing.
+#
+#  This is a modification of the run_parallel_advection.py
+# file.
+#
+#
+#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
+# June 2005
+#
+#
+#
+#########################################################
+
+import time
+import sys
+
+
+#----------------------------
+# Sequential interface
+#---------------------------
+from anuga import Transmissive_boundary, Reflective_boundary
+from anuga import rectangular_cross_domain
+from anuga import Set_quantity
+
+#----------------------------
+# Parallel interface
+#---------------------------
+from anuga_parallel import distribute, myid, numprocs, finalize, barrier
+
+
+t0 = time.time()
+
+verbose = True
+
+#--------------------------------------------------------------------------
+# Setup Domain only on processor 0
+#--------------------------------------------------------------------------
+if myid == 0:
+    length = 2.0
+    width = 2.0
+    #dx = dy = 0.005
+    #dx = dy = 0.00125
+    dx = dy  = 0.5
+    domain = rectangular_cross_domain(int(length/dx), int(width/dy),
+                                              len1=length, len2=width, origin=(-length/2, -width/2), verbose=verbose)
+
+
+    domain.set_store(True)
+    domain.set_quantity('elevation', lambda x,y : -1.0-x )
+    domain.set_quantity('stage', 1.0)
+    domain.set_flow_algorithm('DE0')
+    domain.print_statistics()
+else:
+    domain = None
+
+t1 = time.time()
+
+if myid == 0 :
+    print 'Create sequential domain: Time',t1-t0
+
+if myid == 0 and verbose: 
+    print 'DISTRIBUTING DOMAIN'
+    sys.stdout.flush()
+    
+barrier()
+
+#-------------------------------------------------------------------------
+# Distribute domain
+#-------------------------------------------------------------------------
+domain = distribute(domain,verbose=verbose)
+
+
+t2 = time.time()
+
+if myid == 0 :
+    print 'Distribute domain: Time ',t2-t1
+    
+if myid == 0 : print 'after parallel domain'
+
+
+
+domain.set_name('sw_rectangle')
+
+#Boundaries
+T = Transmissive_boundary(domain)
+R = Reflective_boundary(domain)
+
+
+domain.set_boundary( {'left': R, 'right': R, 'bottom': R, 'top': R, 'ghost': None} )
+
+
+if myid == 0 : print 'after set_boundary'
+
+
+
+#domain.check_integrity()
+
+if myid == 0 : print 'after check_integrity'
+
+class Set_Stage:
+    """Set an initial condition with constant water height, for x<x0
+    """
+
+    def __init__(self, x0=0.25, x1=0.75, y0=0.0, y1=1.0, h=5.0, h0=0.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.y0 = y0
+        self.y1 = y1
+        self.h  = h
+        self.h0 = h0
+
+    def __call__(self, x, y):
+        return self.h0 + self.h*((x>self.x0)&(x<self.x1)&(y>self.y0)&(y<self.y1))
+
+
+
+#domain.set_quantity('stage', Set_Stage(0.2, 0.4, 0.25, 0.75, 1.0, 0.00))
+
+Set_quantity(domain,'stage',center=(0.0,0.0), radius=0.5, value = 2.0)
+
+if myid == 0 : print 'after set quantity'
+
+
+
+yieldstep = 0.005
+finaltime = 0.05
+
+barrier()
+
+t0 = time.time()
+
+#===========================================================================
+# Main Evolve Loop
+#===========================================================================
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    if myid == 0:
+        domain.write_time()
+        sys.stdout.flush()
+        
+        
+
+
+for p in range(numprocs):
+    barrier()
+    if myid == p:
+        print 50*'='
+        print 'P%g' %(myid)
+        print 'That took %.2f seconds' %(time.time()-t0)
+        print 'Communication time %.2f seconds'%domain.communication_time
+        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+        sys.stdout.flush()
+
+
+if domain.number_of_global_triangles < 50000:
+    if myid == 0 :
+        print 'Create dump of triangulation for %g triangles' % domain.number_of_global_triangles
+    domain.dump_triangulation(filename="rectangular_cross_%g.png"% numprocs)
+
+finalize()
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangular_cross_profile.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangular_cross_profile.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangular_cross_profile.py	(revision 9424)
@@ -0,0 +1,182 @@
+
+"""Run parallel shallow water domain.
+
+   run using command like:
+
+   mpirun -np m python run_parallel_sw_rectangular_cross.py
+
+   where m is the number of processors to be used.
+   
+   Will produce sww files with names domain_Pn_m.sww where n is number of processors and
+   m in [0, n-1] refers to specific processor that owned this part of the partitioned mesh.
+"""
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+
+import os
+import sys
+import time
+import numpy as num
+
+#------------------------
+# ANUGA Modules
+#------------------------
+	
+from anuga import Domain
+from anuga import Reflective_boundary
+from anuga import Dirichlet_boundary
+from anuga import Time_boundary
+from anuga import Transmissive_boundary
+
+from anuga import rectangular_cross_domain
+from anuga import create_domain_from_file
+
+
+from anuga_parallel import distribute, myid, numprocs, finalize, barrier
+
+
+#--------------------------------------------------------------------------
+# Setup parameters
+#--------------------------------------------------------------------------
+
+#mesh_filename = "merimbula_10785_1.tsh" ; x0 = 756000.0 ; x1 = 756500.0
+#mesh_filename = "merimbula_43200.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
+#mesh_filename = "test-100.tsh" ; x0 = 0.25 ; x1 = 0.5
+
+
+verbose = True
+
+#--------------------------------------------------------------------------
+# Setup procedures
+#--------------------------------------------------------------------------
+class Set_Stage:
+    """Set an initial condition with constant water height, for x0<x<x1
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))
+
+#--------------------------------------------------------------------------
+# Setup Domain only on processor 0
+#--------------------------------------------------------------------------
+if myid == 0:
+    length = 2.0
+    width = 2.0
+    dx = dy = 0.01
+    domain = rectangular_cross_domain(int(length/dx), int(width/dy),
+                                              len1=length, len2=width)
+
+    domain.set_quantity('elevation', -1.0)
+    domain.set_quantity('stage', Set_Stage(h=2.0))
+else:
+    domain = None
+
+#--------------------------------------------------------------------------
+# Distribute sequential domain on processor 0 to other processors
+#--------------------------------------------------------------------------
+
+if myid == 0 and verbose: print 'DISTRIBUTING DOMAIN'
+domain = distribute(domain)
+
+#domain.smooth = False
+barrier()
+for p in range(numprocs):
+    if myid == p:
+        print 'Process ID %g' %myid
+        print 'Number of triangles %g ' %domain.get_number_of_triangles()
+        sys.stdout.flush()
+        
+    barrier()
+
+
+domain.set_flow_algorithm(2.0)
+
+if myid == 0:
+    domain.print_algorithm_parameters()
+    sys.stdout.flush()
+    
+barrier()
+
+domain.set_name('rectangular_cross')
+domain.set_store(False)
+
+#------------------------------------------------------------------------------
+# Setup boundary conditions
+# This must currently happen *after* domain has been distributed
+#------------------------------------------------------------------------------
+Br = Reflective_boundary(domain)      # Solid reflective wall
+
+domain.set_boundary({'top' :Br, 'bottom' :Br, 'left' :Br, 'right' :Br, 'ghost' : None })
+
+"""
+barrier()
+for p in range(numprocs):
+    if myid == p:
+        print domain.boundary_statistics()
+        sys.stdout.flush()
+        
+    barrier()
+"""
+
+
+domain.dump_triangulation()
+
+#------------------------------------------------------------------------------
+# Evolution
+#------------------------------------------------------------------------------
+if myid == 0 and verbose: print 'EVOLVE'
+
+t0 = time.time()
+finaltime = 0.25
+yieldstep = 0.05
+
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    if myid == 0:
+        domain.write_time()
+
+
+
+## Profiling
+#import cProfile
+#prof_file = 'evolve-prof'+ str(numprocs) + '_' + str(myid) + '.dat'
+#cProfile.run(s,prof_file)
+
+
+barrier()
+
+#for id in range(numprocs):
+#    if myid == id:
+#        import pstats
+#        p = pstats.Stats(prof_file)
+#        #p.sort_stats('cumulative').print_stats(25)
+#        p.sort_stats('time').print_stats(25)
+#        sys.stdout.flush
+#
+#    barrier()
+#
+#barrier()
+
+for p in range(numprocs):
+    if myid == p:
+        print 'Process ID %g' %myid
+        print 'Number of processors %g ' %numprocs
+        print 'That took %.2f seconds' %(time.time()-t0)
+        print 'Communication time %.2f seconds'%domain.communication_time
+        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+
+    barrier()
+
+
+finalize()
+
+
+
+
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangular_tsunami.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangular_tsunami.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_parallel_sw_rectangular_tsunami.py	(revision 9424)
@@ -0,0 +1,159 @@
+#########################################################
+#
+#  Main file for parallel mesh testing.
+#
+#  This is a modification of the run_parallel_advection.py
+# file.
+#
+#
+#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
+# June 2005
+#
+#
+#
+#########################################################
+
+import time
+import sys
+import numpy
+import anuga
+
+
+#----------------------------
+# Parallel interface
+#---------------------------
+from anuga_parallel import distribute, myid, numprocs, finalize, barrier
+
+
+#--------------------------------------------------------------------------
+# Setup functions for topograpy etc
+#--------------------------------------------------------------------------
+scale_me=1.0
+
+def topography(x,y):
+	return (-x/2.0 +0.05*numpy.sin((x+y)*200.0))*scale_me
+
+def stagefun(x,y):
+    stge=-0.2*scale_me #+0.01*(x>0.9)
+    #topo=topography(x,y)
+    return stge#*(stge>topo) + (topo)*(stge<=topo)
+
+
+
+#--------------------------------------------------------------------------
+# Create domains
+#--------------------------------------------------------------------------
+
+
+
+t0 = time.time()
+
+verbose = True
+
+#--------------------------------------------------------------------------
+# Setup Domain only on processor 0
+#--------------------------------------------------------------------------
+if myid == 0:
+    length = 2.0
+    width = 2.0
+    dx = dy = 0.005
+    #dx = dy = 0.00125
+    dx = dy  = 0.05
+    domain = anuga.rectangular_cross_domain(int(length/dx), int(width/dy),
+                                              len1=length, len2=width, verbose=verbose)
+
+    #---------------------------------------
+    # Add these two commands to use Gareth's
+    # tsunami algorithm. Play with the 
+    # minimum allowed height to remove possible 
+    # unrealistic large velocities
+    #---------------------------------------
+    domain.set_flow_algorithm('tsunami')
+    domain.set_minimum_allowed_height(0.01)
+
+    domain.set_store(True)
+    domain.set_quantity('elevation',topography)     # Use function for elevation
+    domain.get_quantity('elevation').smooth_vertex_values()
+    domain.set_quantity('friction',0.03)            # Constant friction
+    domain.set_quantity('stage', stagefun)          # Constant negative initial stage
+    domain.get_quantity('stage').smooth_vertex_values()
+
+    domain.set_name('rectangular_tsunami')
+
+    domain.print_statistics()
+else:
+    domain = None
+
+t1 = time.time()
+
+if myid == 0 :
+    print 'Create sequential domain ',t1-t0
+
+if myid == 0 and verbose: 
+    print 'DISTRIBUTING DOMAIN'
+    sys.stdout.flush()
+    
+barrier()
+
+#-------------------------------------------------------------------------
+# Distribute domain
+#-------------------------------------------------------------------------
+domain = distribute(domain,verbose=verbose)
+
+
+t2 = time.time()
+
+if myid == 0 :
+    print 'Distribute domain ',t2-t1
+    
+if myid == 0 : print 'after parallel domain'
+
+
+#Boundaries
+T = anuga.Transmissive_boundary(domain)
+R = anuga.Reflective_boundary(domain)
+D = anuga.Dirichlet_boundary([-0.1*scale_me,0.,0.])
+
+
+domain.set_boundary( {'left': R, 'right': D, 'bottom': R, 'top': R, 'ghost': None} )
+
+
+if myid == 0 : print 'after set_boundary'
+
+
+yieldstep = 0.2
+finaltime = 20.0
+
+barrier()
+
+t0 = time.time()
+
+#===========================================================================
+# Main Evolve Loop
+#===========================================================================
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    if myid == 0:
+        domain.write_time()
+        sys.stdout.flush()
+						
+
+
+
+for p in range(numprocs):
+    barrier()
+    if myid == p:
+        print 50*'='
+        print 'P%g' %(myid)
+        print 'That took %.2f seconds' %(time.time()-t0)
+        print 'Communication time %.2f seconds'%domain.communication_time
+        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+        sys.stdout.flush()
+
+
+if domain.number_of_global_triangles < 50000:
+    if myid == 0 :
+        print 'Create dump of triangulation for %g triangles' % domain.number_of_global_triangles
+    domain.dump_triangulation(filename="rectangular_cross_%g.png"% numprocs)
+
+finalize()
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_all_merimbula.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_all_merimbula.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_all_merimbula.py	(revision 9424)
@@ -0,0 +1,159 @@
+"""Run parallel shallow water domain.
+
+   run using command like:
+
+   mpirun -np m python run_sequential_dist_all.py
+
+   where m is the number of processors to be used.
+   
+"""
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+
+import os
+import sys
+import time
+import numpy as num
+
+#------------------------
+# ANUGA Modules
+#------------------------
+	
+from anuga import Domain
+from anuga import Reflective_boundary
+from anuga import Dirichlet_boundary
+from anuga import Time_boundary
+from anuga import Transmissive_boundary
+
+from anuga import rectangular_cross
+from anuga import create_domain_from_file
+
+
+#from anuga_parallel.sequential_distribute import sequential_distribute
+
+from anuga_parallel import distribute, myid, numprocs, finalize, barrier
+
+
+#--------------------------------------------------------------------------
+# Setup parameters
+#--------------------------------------------------------------------------
+
+#mesh_filename = "merimbula_10785_1.tsh" ; x0 = 756000.0 ; x1 = 756500.0
+mesh_filename = "merimbula_17156.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
+#mesh_filename = "merimbula_43200_1.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
+#mesh_filename = "test-100.tsh" ; x0 = 0.25 ; x1 = 0.5
+#mesh_filename = "test-20.tsh" ; x0 = 250.0 ; x1 = 350.0
+yieldstep = 50
+finaltime = 1500
+verbose = True
+
+#--------------------------------------------------------------------------
+# Setup procedures
+#--------------------------------------------------------------------------
+class Set_Stage:
+    """Set an initial condition with constant water height, for x0<x<x1
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))+1.0
+
+
+class Set_Elevation:
+    """Set an elevation
+    """
+
+    def __init__(self, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return x/self.h
+    
+
+#--------------------------------------------------------------------------
+# Setup Sequential Domain
+#--------------------------------------------------------------------------
+if myid == 0 :
+    domain = create_domain_from_file(mesh_filename)
+    domain.set_quantity('stage', Set_Stage(x0, x1, 2.0))
+    #domain.set_datadir('.')
+    domain.set_name('merimbula_new')
+    domain.set_store(True)
+else:
+    domain = None
+
+
+#--------------------------------------------------------------------------
+# Distribute sequential domain on processor 0 to other processors
+#--------------------------------------------------------------------------
+if verbose: print 'DISTRIBUTING DOMAIN'
+domain = distribute(domain)
+
+
+#--------------------------------------------------------------------------
+# On all processors, setup evolve parameters for domains on all processors
+# (all called "domain"
+#--------------------------------------------------------------------------
+
+domain.set_flow_algorithm('DE0')
+
+#------------------------------------------------------------------------------
+# Setup boundary conditions
+# This must currently happen *after* domain has been distributed
+#------------------------------------------------------------------------------
+Br = Reflective_boundary(domain)      # Solid reflective wall
+
+domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Br})
+
+#------------------------------------------------------------------------------
+# Evolution
+#------------------------------------------------------------------------------
+if myid == 0 and verbose: print 'EVOLVE'
+
+barrier()
+t0 = time.time()
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    if myid == 0:
+        domain.write_time()
+
+
+barrier()
+
+for p in range(numprocs):
+    if myid == p:
+        print 'Processor %g ' %myid
+        print 'That took %.2f seconds' %(time.time()-t0)
+        print 'Communication time %.2f seconds'%domain.communication_time
+        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+    else:
+        pass
+
+    barrier()
+
+
+#--------------------------------------------------
+# Merge the individual sww files into one file
+#--------------------------------------------------
+domain.sww_merge(delete_old=False)
+
+finalize()
+
+
+
+
+
+
+
+
+
+
+
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_distribute_merimbula.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_distribute_merimbula.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_distribute_merimbula.py	(revision 9424)
@@ -0,0 +1,100 @@
+
+"""Run parallel shallow water domain.
+
+   run using command like:
+
+   python run_sequential_dist_distribute_merimbula.py
+
+   will produce 4 pickled files corresponding partitioned domains
+   
+"""
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+
+import os
+import sys
+import time
+import numpy as num
+
+#------------------------
+# ANUGA Modules
+#------------------------
+	
+from anuga import Domain
+from anuga import Reflective_boundary
+from anuga import Dirichlet_boundary
+from anuga import Time_boundary
+from anuga import Transmissive_boundary
+
+from anuga import rectangular_cross
+from anuga import create_domain_from_file
+
+
+from anuga_parallel.sequential_distribute import sequential_distribute_dump
+
+
+#--------------------------------------------------------------------------
+# Setup parameters
+#--------------------------------------------------------------------------
+
+#mesh_filename = "merimbula_10785_1.tsh" ; x0 = 756000.0 ; x1 = 756500.0
+mesh_filename = "merimbula_17156.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
+#mesh_filename = "merimbula_43200_1.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
+#mesh_filename = "test-100.tsh" ; x0 = 0.25 ; x1 = 0.5
+#mesh_filename = "test-20.tsh" ; x0 = 250.0 ; x1 = 350.0
+yieldstep = 50
+finaltime = 1500
+verbose = True
+
+#--------------------------------------------------------------------------
+# Setup procedures
+#--------------------------------------------------------------------------
+class Set_Stage:
+    """Set an initial condition with constant water height, for x0<x<x1
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))+1.0
+
+
+class Set_Elevation:
+    """Set an elevation
+    """
+
+    def __init__(self, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return x/self.h
+    
+
+#--------------------------------------------------------------------------
+# Setup Sequential Domain
+#--------------------------------------------------------------------------
+domain = create_domain_from_file(mesh_filename)
+domain.set_quantity('stage', Set_Stage(x0, x1, 2.0))
+#domain.set_datadir('.')
+domain.set_name('merimbula_new')
+domain.set_store(True)
+
+
+#--------------------------------------------------------------------------
+# Distribute sequential domain on processor 0 to other processors
+#--------------------------------------------------------------------------
+
+if verbose: print 'DISTRIBUTING DOMAIN'
+sequential_distribute_dump(domain, 4, verbose=True)
+
+
+
+
+
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_distribute_rectangular.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_distribute_rectangular.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_distribute_rectangular.py	(revision 9424)
@@ -0,0 +1,96 @@
+#########################################################
+#
+#  Main file for parallel mesh testing.
+#
+#  This is a modification of the run_parallel_advection.py
+# file.
+#
+#
+#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
+# June 2005
+#
+#
+#
+#########################################################
+
+import time
+import sys
+import numpy
+
+
+#----------------------------
+# Sequential interface
+#---------------------------
+from anuga import Transmissive_boundary, Reflective_boundary
+from anuga import rectangular_cross_domain
+
+#----------------------------
+# Parallel interface
+#---------------------------
+from anuga_parallel.sequential_distribute import sequential_distribute_dump
+
+
+t0 = time.time()
+
+verbose = True
+
+#--------------------------------------------------------------------------
+# Setup functions for topograpy etc
+#--------------------------------------------------------------------------
+scale_me=1.0
+
+def topography(x,y):
+	return (-x/2.0 +0.05*numpy.sin((x+y)*200.0))*scale_me
+
+def stagefun(x,y):
+    stge=-0.2*scale_me #+0.01*(x>0.9)
+    #topo=topography(x,y)
+    return stge#*(stge>topo) + (topo)*(stge<=topo)
+
+
+#--------------------------------------------------------------------------
+# Setup Domain only on processor 0
+#--------------------------------------------------------------------------
+myid = 0
+numprocs = 3
+
+
+
+length = 2.0
+width = 2.0
+dx = dy = 0.005  # 640,000
+dx = dy  = 0.05
+domain = rectangular_cross_domain(int(length/dx), int(width/dy),
+                                          len1=length, len2=width, verbose=verbose)
+
+
+print domain.number_of_global_triangles
+
+domain.set_store(True)
+domain.set_flow_algorithm('DE0')
+domain.set_minimum_allowed_height(0.01)
+domain.set_quantity('elevation',topography)     # Use function for elevation
+domain.get_quantity('elevation').smooth_vertex_values()
+domain.set_quantity('friction',0.03)            # Constant friction
+domain.set_quantity('stage', stagefun)          # Constant negative initial stage
+domain.get_quantity('stage').smooth_vertex_values()
+
+domain.set_name('sw_rectangle')
+
+
+t1 = time.time()
+print 'Create sequential domain: time ',t1-t0
+    
+
+#-------------------------------------------------------------------------
+# Distribute domain
+#----------------------------------------------------------------------
+sequential_distribute_dump(domain, numprocs, verbose = True)
+
+
+t2 = time.time()
+print 'Sequential Distribute domain: time ',t2-t1
+    
+
+
+
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_evolve_merimbula.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_evolve_merimbula.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_evolve_merimbula.py	(revision 9424)
@@ -0,0 +1,109 @@
+
+"""Run parallel shallow water domain.
+
+   run using command like:
+
+   mpirun -np 4  python run_sequential_dist_evolve_merimbula.py
+
+   Need to have run run_sequential_dist_distribute_merimbula.py
+
+   first to produce the required pickle files encoding the partitioning
+   """
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+
+import os
+import sys
+import time
+import numpy as num
+
+#------------------------
+# ANUGA Modules
+#------------------------
+	
+from anuga import Domain
+from anuga import Reflective_boundary
+from anuga import Dirichlet_boundary
+from anuga import Time_boundary
+from anuga import Transmissive_boundary
+
+from anuga import rectangular_cross
+from anuga import create_domain_from_file
+
+
+from anuga_parallel import distribute, myid, numprocs, finalize, barrier
+
+from anuga_parallel.sequential_distribute import sequential_distribute_load
+
+#--------------------------------------------------------------------------
+# Setup parameters
+#--------------------------------------------------------------------------
+yieldstep = 50
+finaltime = 1500
+verbose = True
+
+
+
+#-------------------------------------------------------------------------------
+# Read in cPickled distribute mesh data
+#-------------------------------------------------------------------------------
+domain = sequential_distribute_load(filename='merimbula_new', verbose = True)
+
+#--------------------------------------------------------------------------
+# On all processors, setup evolve parameters for domains on all processors
+# (all called "domain"
+#--------------------------------------------------------------------------
+
+domain.set_flow_algorithm('DE0')
+
+
+
+#------------------------------------------------------------------------------
+# Setup boundary conditions
+# This must currently happen *after* domain has been distributed
+#------------------------------------------------------------------------------
+Br = Reflective_boundary(domain)      # Solid reflective wall
+
+domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Br})
+
+#------------------------------------------------------------------------------
+# Evolution
+#------------------------------------------------------------------------------
+if myid == 0 and verbose: print 'EVOLVE'
+
+barrier()
+t0 = time.time()
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    if myid == 0:
+        domain.write_time()
+
+
+barrier()
+
+for p in range(numprocs):
+    if myid == p:
+        print 'Processor %g ' %myid
+        print 'That took %.2f seconds' %(time.time()-t0)
+        print 'Communication time %.2f seconds'%domain.communication_time
+        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+    else:
+        pass
+
+    barrier()
+
+
+#--------------------------------------------------
+# Merge the individual sww files into one file
+#--------------------------------------------------
+
+
+
+
+finalize()
+
+
+
+
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_parallel_evolve_rectangular.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_parallel_evolve_rectangular.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_sequential_dist_parallel_evolve_rectangular.py	(revision 9424)
@@ -0,0 +1,134 @@
+#########################################################
+#
+#  Main file for parallel mesh testing.
+#
+#  This is a modification of the run_parallel_advection.py
+# file.
+#
+#
+#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
+# June 2005
+#
+#
+#
+#########################################################
+
+import time
+import sys
+import numpy
+import anuga
+
+
+#----------------------------
+# Parallel interface
+#---------------------------
+from anuga_parallel import myid, numprocs, finalize, barrier
+
+from anuga_parallel.sequential_distribute import sequential_distribute_load
+
+
+#--------------------------------------------------------------------------
+# Setup functions for topograpy etc
+#--------------------------------------------------------------------------
+scale_me=1.0
+
+def topography(x,y):
+	return (-x/2.0 +0.05*numpy.sin((x+y)*200.0))*scale_me
+
+def stagefun(x,y):
+    stge=-0.2*scale_me #+0.01*(x>0.9)
+    #topo=topography(x,y)
+    return stge#*(stge>topo) + (topo)*(stge<=topo)
+
+
+
+
+verbose = True
+basename = 'sw_rectangle'
+#basename = 'test'
+
+#--------------------------------------------------------------------------
+# Create domains
+#--------------------------------------------------------------------------
+#import cPickle
+#pickle_name = pickle_base+'_P%g_%g.pickle'% (numprocs,myid)
+#f = file(pickle_name, 'rb')
+# = cPickle.load(f)
+#f.close()
+
+
+t1 = time.time()
+
+if myid == 0 and verbose:
+    print 'Loading DISTRIBUTING DOMAIN'
+    sys.stdout.flush()
+
+#-------------------------------------------------------------------------
+# Distribute domain
+#-------------------------------------------------------------------------
+domain = sequential_distribute_load(filename = basename, verbose=verbose)
+
+
+t2 = time.time()
+
+if myid == 0 :
+    print 'Load Distribute domain ',t2-t1
+    
+if myid == 0 : print 'after parallel domain'
+
+
+#Boundaries
+T = anuga.Transmissive_boundary(domain)
+R = anuga.Reflective_boundary(domain)
+D = anuga.Dirichlet_boundary([-0.1*scale_me,0.,0.])
+
+
+domain.set_boundary( {'left': R, 'right': D, 'bottom': R, 'top': R, 'ghost': None} )
+
+
+if myid == 0 : print 'after set_boundary'
+
+
+yieldstep = 0.2
+finaltime = 20.0
+
+barrier()
+
+t0 = time.time()
+
+#===========================================================================
+# Main Evolve Loop
+#===========================================================================
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    if myid == 0:
+        domain.write_time()
+        sys.stdout.flush()
+						
+
+
+
+for p in range(numprocs):
+    barrier()
+    if myid == p:
+        print 50*'='
+        print 'P%g' %(myid)
+        print 'That took %.2f seconds' %(time.time()-t0)
+        print 'Communication time %.2f seconds'%domain.communication_time
+        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
+        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
+        sys.stdout.flush()
+
+
+if domain.number_of_global_triangles < 50000:
+    if myid == 0 :
+        print 'Create dump of triangulation for %g triangles' % domain.number_of_global_triangles
+    domain.dump_triangulation(filename="rectangular_cross_%g.png"% numprocs)
+
+
+#--------------------------------------------------
+# Merge the individual sww files into one file
+#--------------------------------------------------
+domain.sww_merge(delete_old=False)
+
+
+finalize()
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_sw_lost_mass.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_sw_lost_mass.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_sw_lost_mass.py	(revision 9424)
@@ -0,0 +1,108 @@
+#!/usr/bin/env/python
+#########################################################
+#
+# Shows problem of conservation of mass. Due to material being
+# removed if too shallow. Can lead to problems if timestep
+# set very small.
+#
+#
+#
+#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
+# June 2005
+#
+#
+#
+#########################################################
+
+
+##############################################
+# Set a small yieldstep (say 0.005) to see
+# lose of conservation
+# Set it to a larger value (say 0.1) and have
+# conservation "restored"
+##############################################
+yieldstep = 0.005
+#yieldstep = 0.08
+#yieldstep = 0.05
+finaltime = 100.0
+
+import sys
+import time
+
+
+from Numeric import array, zeros, Float
+
+#from shallow_water import Domain
+
+from anuga.shallow_water import Domain
+
+# mesh partition routines
+
+from anuga.abstract_2d_finite_volumes.pmesh2domain\
+     import pmesh_to_domain_instance
+
+# read in the processor information
+
+
+#-------
+# Domain
+rect = zeros( 4, Float) # Buffer for results
+
+class Set_Stage:
+    """Set an initial condition with constant water height, for x0<x<x1
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))
+
+
+
+    # read in the test files
+
+filename = 'test-100.tsh'
+
+nx = 1
+ny = 1
+
+domain = pmesh_to_domain_instance(filename, Domain)
+
+domain.set_quantity('stage', Set_Stage(200.0,300.0,5.0))
+rect = array(domain.xy_extent, Float)
+
+try:
+    domain.initialise_visualiser(rect=rect)
+    #domain.visualiser.coloring['stage'] = True
+    domain.visualiser.scale_z['stage'] = 0.2
+    domain.visualiser.scale_z['elevation'] = 0.05
+except:
+    print 'No visualiser'
+
+
+domain.default_order = 2
+
+#Boundaries
+from shallow_water import Transmissive_boundary, Reflective_boundary
+
+T = Transmissive_boundary(domain)
+R = Reflective_boundary(domain)
+domain.set_boundary( {'outflow': R, 'inflow': R, 'inner':R, 'exterior': R, 'open':R} )
+
+
+
+#---------
+# Evolution
+t0 = time.time()
+
+print 'No of triangles %d'%(domain.number_of_triangles)
+
+
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    #domain.write_time()
+    print '    Integral stage = ', domain.quantities['stage'].get_integral(),' Time = ',domain.time
+
+
Index: /trunk/anuga_core/source/anuga_parallel/examples/run_sw_lost_mass_simpler.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/examples/run_sw_lost_mass_simpler.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/examples/run_sw_lost_mass_simpler.py	(revision 9424)
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+#########################################################
+#
+# Shows problem of conservation of mass. Due to material being
+# removed if too shallow. Can lead to problems if timestep
+# set very small.
+#
+#
+#
+#  Authors: Linda Stals, Steve Roberts, Matthew Hardy, Ole Nielsen
+#  July 2005
+#
+#
+#########################################################
+
+
+##############################################
+# Change min_depth and see how larger values aggravate the problem
+yieldstep = 0.1
+finaltime = 10.0
+#min_depth = 1.0e-4
+min_depth = 1.0e-2
+
+
+from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular
+from anuga.shallow_water import Domain, Reflective_boundary
+
+
+
+#Create shallow water domain
+points, vertices, boundary = rectangular(50, 50, len1=500, len2=500)
+domain = Domain(points, vertices, boundary)
+domain.smooth = False
+domain.visualise = True
+domain.default_order = 1
+domain.minimum_allowed_height = min_depth
+print 'Extent', domain.get_extent()
+
+# Set initial condition
+class Set_IC:
+    """Set an initial condition with a constant value, for x0<x<x1
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))
+
+
+domain.set_quantity('stage', Set_IC(200.0,300.0,5.0))
+
+try:
+    domain.initialise_visualiser()
+except:
+    print 'No visualiser'
+else:    
+    domain.visualiser.scale_z['stage'] = 0.2
+    domain.visualiser.scale_z['elevation'] = 0.05
+    
+
+#Boundaries
+R = Reflective_boundary(domain)
+domain.set_boundary( {'left': R, 'right': R, 'top':R, 'bottom': R} )
+
+
+# Evolution
+print 'Minimal allowed water height = ', domain.minimum_allowed_height 
+for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+    print 'Integral stage = ', domain.quantities['stage'].get_integral(),' Time = ',domain.time
+
+
Index: unk/anuga_core/source/anuga_parallel/run_advection.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_advection.py	(revision 9423)
+++ 	(revision )
@@ -1,50 +1,0 @@
-#import pdb
-#pdb.set_trace()
-
-#========================================================================
-from anuga.config import g, epsilon
-
-#from Numeric import allclose, array, zeros, ones, Float
-
-from anuga.advection import Domain, Transmissive_boundary, Dirichlet_boundary
-
-#from Numeric import array
-import numpy as num
-
-
-from anuga.interface import rectangular_cross
-
-#points, vertices, boundary = rectangular(60, 60)
-points, vertices, boundary = rectangular_cross(10, 10)
-
-#Create advection domain with direction (1,-1)
-domain = Domain(points, vertices, boundary, velocity=[1.0, 0.0])
-
-# Initial condition is zero by default
-
-#domain.initialise_visualiser(scale_z=1.0)
-#domain.visualise_range_z = 0.5
-#domain.visualise_color_stage = True
-
-
-#Boundaries
-T = Transmissive_boundary(domain)
-D = Dirichlet_boundary(num.array([1.0]))
-
-#turn on the visualisation
-rect = [0.0, 0.0, 1.0, 1.0]
-#domain.initialise_visualiser(rect=rect)
-domain.visualise = False
-
-
-domain.default_order = 2
-
-print domain.quantities.keys()
-
-domain.set_boundary( {'left': D, 'right': T, 'bottom': T, 'top': T} )
-domain.check_integrity()
-
-#Check that the boundary value gets propagated to all elements
-for t in domain.evolve(yieldstep = 0.1, finaltime = 1.5):
-    domain.write_time()
-    #pdb.set_trace()
Index: unk/anuga_core/source/anuga_parallel/run_parallel_advection.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_advection.py	(revision 9423)
+++ 	(revision )
@@ -1,164 +1,0 @@
-#!/usr/bin/env python
-#########################################################
-#
-#  Main file for parallel mesh testing. Runs an advection
-# flow simulation using a rectangular mesh
-#
-#
-#  Authors: Steve Roberts June 2005
-#  Modified by Linda Stals April 2006
-#
-#
-#########################################################
-
-
-
-# Parallel communication routines
-
-import pypar
-
-#  Mesh partition routines
-
-from parallel_meshes import parallel_rectangle
-
-# Parallel Domain
- 
-from parallel_advection import Parallel_Domain
-from parallel_advection import Transmissive_boundary
-
-############################
-# Set the initial conditions
-############################
-class Set_Stage:
-    """Set an initial condition with constant water height, for x<x0
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))
-
-###############################
-# Read in processor information
-###############################
-
-numprocs = pypar.size()
-myid = pypar.rank()
-processor_name = pypar.get_processor_name()
-
-N = 5
-M = 2
-
-N = 10
-M = 5
-
-#######################
-# Partition the mesh
-#######################
-
-# Build a unit mesh, subdivide it over numproces processors with each
-# submesh containing M*N nodes
-
-points, vertices, boundary, full_send_dict, ghost_recv_dict =  \
-    parallel_rectangle(N, M, len1_g=1.0)
-
-#print 'ghost ',ghost_recv_dict
-#print 'full  ',full_send_dict
-
-
-
-print "Myid = ", myid, "no points = ", len(points), \
-      "no vertices = ", len(vertices), "no boundaries = ", len(boundary)
-
-###########################################
-# Start the computations on each subpartion
-###########################################
-
-# Create advection domain with direction (1,-1)
-# Initial condition is zero by default
-
-domain = Parallel_Domain(points, vertices, boundary,
-                         full_send_dict, ghost_recv_dict, velocity=[1.0, 0.0])
-
-#print 'ghost ',ghost_recv_dict
-#print 'full  ',full_send_dict
-
-
-# Boundaries
-
-T = Transmissive_boundary(domain)
-
-
-
-
-domain.set_boundary( {'left': T, 'right': T, 'bottom': T, 'top': T, \
-                      'ghost': None} )
-
-
-# Set Evolve parameters
-domain.set_default_order(2)
-domain.set_timestepping_method('rk2')
-
-print domain.get_timestepping_method()
-
-#domain.use_edge_limiter = True
-#domain.tight_slope_limiters = True
-#domain.use_centroid_velocities = False
-
-domain.CFL = 1.0
-
-domain.set_beta(0.8)
-
-
-# Ensure that the domain definitions make sense
-
-domain.check_integrity()
-
-# Set the inititial conditions
-
-domain.set_quantity('stage', Set_Stage(0.2,0.4,1.0))
-
-# Let processor 0 output some timing information
-
-visualise = False
-if visualise:
-    from anuga.visualiser import RealtimeVisualiser
-    vis = RealtimeVisualiser(domain)
-    vis.render_quantity_height("stage", zScale = 5.0, dynamic=True)
-    vis.colour_height_quantity('stage', (0.2, 0.2, 0.8))
-    vis.start()
-    import time
-    time.sleep(2.0)
-    
-
-
-if myid == 0:
-    import time
-    t0 = time.time()
-
-for t in domain.evolve(yieldstep = 5.0, finaltime = 30.0):
-    if myid == 0:
-        domain.write_time()
-        
-    if visualise:
-        vis.update()						
-
-        
-
-if visualise: vis.evolveFinished()
-    
-
-# Output some computation statistics
-
-if myid == 0:
-    print 'That took %.2f seconds' %(time.time()-t0)
-    print 'Communication time %.2f seconds'\
-          %domain.communication_time
-    print 'Reduction Communication time %.2f seconds'\
-          %domain.communication_reduce_time
-        
-  
-if visualise: vis.join()
Index: unk/anuga_core/source/anuga_parallel/run_parallel_advection_prof.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_advection_prof.py	(revision 9423)
+++ 	(revision )
@@ -1,122 +1,0 @@
-#!/usr/bin/env python
-#########################################################
-#
-#  Main file for parallel mesh testing. Runs an advection
-# flow simulation using a rectangular mesh
-#
-#
-#  Authors: Steve Roberts June 2005
-#  Modified by Linda Stals April 2006
-#
-#
-#########################################################
-
-
-
-# Parallel communication routines
-
-import pypar
-
-#  Mesh partition routines
-
-from parallel_meshes import parallel_rectangle
-
-# Parallel Domain
- 
-from parallel_advection import Parallel_Domain
-from parallel_advection import Transmissive_boundary
-
-############################
-# Set the initial conditions
-############################
-class Set_Stage:
-    """Set an initial condition with constant water height, for x<x0
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))
-
-###############################
-# Read in processor information
-###############################
-
-numprocs = pypar.size()
-myid = pypar.rank()
-processor_name = pypar.Get_processor_name()
-
-N = 80
-M = 80
-
-#######################
-# Partition the mesh
-#######################
-
-# Build a unit mesh, subdivide it over numproces processors with each
-# submesh containing M*N nodes
-
-points, vertices, boundary, full_send_dict, ghost_recv_dict =  \
-    parallel_rectangle(N, M, len1_g=1.0)
-
-print "Myid = ", myid, "no points = ", len(points), \
-      "no vertices = ", len(vertices), "no boundaries = ", len(boundary)
-
-###########################################
-# Start the computations on each subpartion
-###########################################
-
-# Create advection domain with direction (1,-1)
-# Initial condition is zero by default
-
-domain = Parallel_Domain(points, vertices, boundary,
-                         full_send_dict, ghost_recv_dict, velocity=[1.0, 0.0])
-
-# Boundaries
-
-T = Transmissive_boundary(domain)
-domain.default_order = 2
-domain.set_boundary( {'left': T, 'right': T, 'bottom': T, 'top': T, \
-                      'ghost': None} )
-
-# Ensure that the domain definitions make sense
-
-domain.check_integrity()
-
-# Set the inititial conditions
-
-domain.set_quantity('stage', Set_Stage(0.2,0.4,1.0))
-
-# Let processor 0 output some timing information
-
-if myid == 0:
-    import time
-    t0 = time.time()
-
-# Start the evolve computions
-import hotshot
-profiler = hotshot.Profile("hotshot." + str(numprocs) + "." + str(myid) + ".prof")
-s = '''for t in domain.evolve(yieldstep = 0.1, finaltime = 2.0):
-  if myid == 0:
-    domain.write_time()
-'''
-    
-
-result = profiler.runctx(s, globals(), locals())
-profiler.close()
-
-# for t in domain.evolve(yieldstep = 0.1, finaltime = 3.0):
-#     if myid == 0:
-#         domain.write_time()
-
-# Output some computation statistics
-
-if myid == 0:
-    print 'That took %.2f seconds' %(time.time()-t0)
-    print 'Communication time %.2f seconds'\
-          %domain.communication_time
-    print 'Reduction Communication time %.2f seconds'\
-          %domain.communication_reduce_time
Index: unk/anuga_core/source/anuga_parallel/run_parallel_boyd_box_op.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_boyd_box_op.py	(revision 9423)
+++ 	(revision )
@@ -1,167 +1,0 @@
-import os.path
-import sys
-
-
-import anuga
-
-from anuga import myid, numprocs, finalize
-from anuga import Inlet_operator, Boyd_box_operator
-
-
-
-
-"""
-This test exercises the parallel culvert
-"""
-verbose = True
-
-    
-
-length = 40.
-width = 16.
-
-dx = dy = 2           # Resolution: Length of subdivisions on both axes
-
-#----------------------------------------------------------------------
-# Setup initial conditions
-#----------------------------------------------------------------------
-
-def topography(x, y):
-    """Set up a weir
-    
-    A culvert will connect either side
-    """
-    # General Slope of Topography
-    z=-x/1000
-    
-    N = len(x)
-    for i in range(N):
-
-       # Sloping Embankment Across Channel
-        if 5.0 < x[i] < 10.1:
-            # Cut Out Segment for Culvert face                
-            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
-               z[i]=z[i]
-            else:
-               z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
-        if 10.0 < x[i] < 12.1:
-           z[i] +=  2.5                    # Flat Crest of Embankment
-        if 12.0 < x[i] < 14.5:
-            # Cut Out Segment for Culvert face                
-            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
-               z[i]=z[i]
-            else:
-               z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
-                   
-        
-    return z
-
-##-----------------------------------------------------------------------
-## Setup domain
-##-----------------------------------------------------------------------
-if myid == 0:
-    points, vertices, boundary = anuga.rectangular_cross(int(length/dx),
-                                                   int(width/dy),
-                                                   len1=length, 
-                                                   len2=width)
-
-    domain = anuga.Domain(points, vertices, boundary)   
-    domain.set_name()                 # Output name output_script_name.sww
-    domain.set_flow_algorithm('1_5')
-else:
-    domain = None
-    
-##-----------------------------------------------------------------------
-## Distribute domain
-##-----------------------------------------------------------------------
-
-
-domain = anuga.distribute(domain)
-domain.dump_triangulation("run_parallel_boyd_box_op_domain.png")
-    
-
-##-----------------------------------------------------------------------
-## Setup boundary conditions
-##-----------------------------------------------------------------------
-    
-domain.set_quantity('elevation', topography) 
-domain.set_quantity('friction', 0.01)         # Constant friction 
-domain.set_quantity('stage',
-                        expression='elevation')   # Dry initial condition
-
-    
-Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
-Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
-domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
-
-
-
-    ################ Define Fractional Operators ##########################
-line0 = [[10.0, 10.0], [30.0, 10.0]]
-#line0 = [[29.0, 10.0], [30.0, 10.0]]
-poly1 = [[0.0, 10.0], [0.0, 15.0], [5.0, 15.0], [5.0, 10.0]]
-Q0 = anuga.file_function('test_hydrograph.tms', quantities=['hydrograph'])
-Q1 = 5.0
-
-samples = 50
-
-
-inlet0 = None
-inlet1 = None
-boyd_box0 = None
-
-inlet0 = Inlet_operator(domain, line0, Q0, logging=True, description='inlet0', verbose = False)
-inlet1 = Inlet_operator(domain, poly1, Q1, logging=True, description='inlet1', verbose = False)
-
-# Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
-
-boyd_box0 = Boyd_box_operator(domain,
-                              end_points=[[9.0, 2.5],[19.0, 2.5]],
-                              losses=1.5,
-                              width=5.0,
-                              apron=5.0,
-                              use_momentum_jet=True,
-                              use_velocity_head=False,
-                              manning=0.013,
-                              logging=True,
-                              description='boyd_box_0',
-                              verbose=False)
-
-if inlet0 is not None and verbose: inlet0.print_statistics()
-if inlet1 is not None and verbose: inlet1.print_statistics()
-if boyd_box0 is not None and verbose: boyd_box0.print_statistics()
-
-
-##-----------------------------------------------------------------------
-## Evolve system through time
-##-----------------------------------------------------------------------
-
-for t in domain.evolve(yieldstep = 2.0, finaltime = 20.0):
-    if myid == 0 and verbose:
-        domain.write_time()
-
-    #print domain.volumetric_balance_statistics()
-
-    stage = domain.get_quantity('stage')
-
-
-    if boyd_box0 is not None and verbose : boyd_box0.print_timestepping_statistics()
-
-    #for i in range(samples):
-    #    if tri_ids[i] >= 0:                
-    #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
-
-    sys.stdout.flush()
-
-    pass
-
-##-----------------------------------------------------------------------
-## Assign/Test Control data
-##-----------------------------------------------------------------------
-
-domain.sww_merge()
-
-finalize()
-
-    
-
Index: unk/anuga_core/source/anuga_parallel/run_parallel_gate_operator.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_gate_operator.py	(revision 9423)
+++ 	(revision )
@@ -1,192 +1,0 @@
-import os.path
-import sys
-
-#from anuga.utilities.system_tools import get_pathname_from_package
-#from anuga.geometry.polygon_function import Polygon_function
-        
-#from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
-#from anuga.abstract_2d_finite_volumes.quantity import Quantity
-
-import anuga
-import time
-
-                            
-from math import pi, pow, sqrt
-
-import numpy as num
-
-from anuga import distribute, myid, numprocs, finalize, barrier
-
-from anuga import Inlet_operator, Boyd_box_operator
-
-
-
-#----------------------------------------------------------------------
-# Setup initial conditions
-#----------------------------------------------------------------------
-
-def topography(x, y):
-    """Set up a weir
-
-    A culvert will connect either side
-    """
-    # General Slope of Topography
-    z=-x/1000
-
-    N = len(x)
-    for i in range(N):
-
-        # Sloping Embankment Across Channel
-        if 5.0 < x[i] < 10.1:
-            # Cut Out Segment for Culvert face
-            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0:
-                z[i]=z[i]
-            else:
-                z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
-        if 10.0 < x[i] < 12.1:
-            z[i] +=  2.5                    # Flat Crest of Embankment
-        if 12.0 < x[i] < 14.5:
-            # Cut Out Segment for Culvert face
-            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
-                z[i]=z[i]
-            else:
-                z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
-
-    return z
-
-  
-
-length = 40.
-width = 15.
-
-dx = dy = 0.5          # Resolution: Length of subdivisions on both axes
-
-
-if myid == 0:
-
-    points, vertices, boundary = anuga.rectangular_cross(int(length/dx),
-                                                         int(width/dy),
-                                                         len1=length, 
-                                                         len2=width)
-    domain = anuga.Domain(points, vertices, boundary)
-    domain.set_name()                 # Output name
-    domain.set_flow_algorithm('DE0')
-    domain.set_store_vertices_uniquely(True)
-    domain.set_quantity('elevation', topography)
-    domain.set_quantity('friction', 0.01)         # Constant friction
-    domain.set_quantity('stage',
-                        expression='elevation')   # Dry initial condition
-
-
-
-else:
-
-    domain = None
-
-
-domain = distribute(domain, verbose = True)
-
-#domain.set_store_vertices_uniquely(False)
-
-
-gate = Boyd_box_operator(domain,
-                            end_points=[[9.0, 2.5],[13.0, 2.5]],
-                            losses=1.5,
-                            width=1.5,
-                            height = 0.0001,
-                            apron=5.0,
-                            use_momentum_jet=True,
-                            use_velocity_head=False,
-                            manning=0.013,
-                            verbose=False)
-
-# Close the gate
-if gate is not None:
-    gate.set_culvert_height(0.0)
-
-line = [[0.0, 5.0], [0.0, 10.0]]
-Q = 1.0
-in0 = Inlet_operator(domain, line, Q)
-
-
-
-
-##-----------------------------------------------------------------------
-## Setup boundary conditions
-##-----------------------------------------------------------------------
-
-## Inflow based on Flow Depth and Approaching Momentum
-Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
-
-domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
-
-
-##-----------------------------------------------------------------------
-## Evolve system through time
-##-----------------------------------------------------------------------
-barrier()
-
-t0 = time.time()
-
-for t in domain.evolve(yieldstep = 1.0, finaltime = 50):
-
-    if myid == 0:
-        print 80*'='
-        domain.write_time()
-
-    #================================================
-    # gate operator lives on a number of processors
-    #
-    # Need to run parallel gate operations on all
-    # processors where gate lives.
-    # But true result only on master processor.
-    #================================================
-    if gate is not None:
-        [s0, s1] = gate.get_enquiry_stages()
-        [d0, d1] = gate.get_enquiry_depths()
-        [e0, e1] = gate.get_enquiry_elevations()
-        [i0, i1] = gate.get_enquiry_invert_elevations()
-        [w0, w1] = gate.get_enquiry_water_depths()
-
-        output = gate.discharge_routine()
-
-        if myid == gate.get_master_proc():
-            print 'myid ', myid, s0,s1
-            print 'myid ', myid, d0,d1
-            print 'myid ', myid, e0,e1
-            print 'myid ', myid, i0,i1
-            print 'myid ', myid, w0,w1
-
-            print 'myid ',myid, output
-
-
-            if d0 > 0.2: gate.set_culvert_height(10.0)
-
-            
-    
-
-barrier()
-
-
-domain.sww_merge(delete_old=True)
-
-
-
-
-barrier()
-
-for p in range(numprocs):
-    if myid == p:
-        print 'Processor %g ' %myid
-        print 'That took %.2f seconds' %(time.time()-t0)
-        print 'Communication time %.2f seconds'%domain.communication_time
-        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
-        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
-    else:
-        pass
-
-    barrier()
-
-
-finalize()
-
Index: unk/anuga_core/source/anuga_parallel/run_parallel_job.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_job.py	(revision 9423)
+++ 	(revision )
@@ -1,95 +1,0 @@
-#!/usr/bin/env python
-
-"""Run job in parallel using PBS batch system with
-   specified number of processors
-
-   Syntax
-     run_parallel_job.py 4 <prog>
-"""
-
-queue = 'normal'
-walltime = '00:10:00'
-vmem = '1Mb'
-
-import sys, string, time, os
-arg = sys.argv
-command = arg[0]
-ext = '.py'
-
-if len(arg)<3:
-    print __doc__
-    print 'Usage:'
-    print command + ' p <filename>' + ext
-    sys.exit()
-
-numproc = int(arg[1])    
-progname = arg[2]
-
-
-#Create batchfile
-
-s = """
-#!/bin/csh
-
-# This file is automatically generated - don't edit
-#
-
-#PBS -wd  
-#PBS -q %s
-#PBS -l ncpus=%d
-#PBS -l walltime=%s,vmem=%s
-#PBS -l software=python
-
-prun %s""" %(queue, numproc, walltime, vmem, progname)
-
-
-scriptname = progname + '.run'
-fid = open(scriptname, 'w')
-fid.write(s)
-fid.close()
-
-#Cleanup
-os.system('/bin/rm *.o* *.e*')
-
-# Run
-os.system('qsub %s' %scriptname)
-os.system('/bin/rm %s' %scriptname)
-
-#Create check scripts
-fid = open('q', 'w')
-fid.write("""#!/bin/csh
-#This file is generated automatically.
-#Any modifications will be lost.
-#Edit runjob.py instead.
-echo '--------------------------------------------'
-nqstat %s\n""" %queue)
-fid.close()
-
-#fid = open('o', 'w')
-#fid.write("""#!/bin/csh
-##This file is generated automatically.
-##Any modifications will be lost.
-##Edit runjob.py instead.
-#cat %s.e* %s.o*\n""" %((progname,)*2))
-#fid.close()
-
-
-fid = open('o', 'w')
-fid.write("""#!/bin/csh
-#This file is generated automatically.
-#Any modifications will be lost.
-#Edit runjob.py instead.
-cat *.e* *.o*\n""")
-
-fid.close()
-
-os.system('chmod +x o q')
-
-
-
-
-
-
-
-
-
Index: unk/anuga_core/source/anuga_parallel/run_parallel_merimbula_test.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_merimbula_test.py	(revision 9423)
+++ 	(revision )
@@ -1,173 +1,0 @@
-#!/usr/bin/env python
-#########################################################
-#
-#  Main file for parallel mesh testing. Runs an advection
-# flow simulation using a rectangular mesh
-#
-#  This is a modification of the run_parallel_advection.py
-# file
-#
-#
-# *) The (new) files that have been added to manage the
-# grid partitioning are
-#    +) pmesh_divide_metis.py: subdivide a pmesh
-#    +) build_submesh.py: build the submeshes on the host
-# processor.
-#    +) build_local.py: build the GA mesh datastructure
-# on each processor.
-#    +) build_commun.py: handle the communication between
-# the host and processors
-#
-#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
-# June 2005
-#
-#
-#
-#########################################################
-
-import pypar    # The Python-MPI interface
-import time
-
-from sys import path
-
-print path
-
-# Numeric arrays
-from Numeric import array, zeros, Float
-
-from print_stats import print_test_stats, build_full_flag
-
-from anuga.abstract_2d_finite_volumes.pmesh2domain\
-     import pmesh_to_domain_instance
-from anuga.advection.advection import Domain as Advection_Domain
-from parallel_advection import Parallel_Domain
-
-from anuga.abstract_2d_finite_volumes.generic_boundary_conditions\
-     import Transmissive_boundary
-
-# mesh partition routines
-
-from pmesh_divide  import pmesh_divide_metis
-from build_submesh import build_submesh
-from build_local   import build_local_mesh
-from build_commun  import send_submesh, rec_submesh, extract_submesh
-
-
-
-class Set_Stage:
-    """Set an initial condition with constant water height, for x<x0
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))
-
-# read in the processor information
-
-numprocs = pypar.size()
-myid = pypar.rank()
-processor_name = pypar.Get_processor_name()
-
-
-#-------
-# Domain
-rect = zeros( 4, Float) # Buffer for results
-
-if myid == 0:
-
-    # read in the test files
-
-    filename = 'test-100.tsh'
-#    filename = 'merimbula_10785.tsh'
-    nx = numprocs
-    ny = 1
-    if nx*ny != numprocs:
-        print "WARNING: number of subboxes is not equal to the number of proc"
-
-    domain_full = pmesh_to_domain_instance(filename, Advection_Domain)
-    domain_full.set_quantity('stage', Set_Stage(200.0,300.0,1.0))
-#    domain_full.set_quantity('stage', Set_Stage(756000.0,756500.0,4.0))
-
-    nodes, triangles, boundary, triangles_per_proc, quantities  =\
-            pmesh_divide_metis(domain_full, numprocs)
-    
-    # subdivide the mesh
-    rect = array(domain_full.xy_extent, Float)
-#    rect = array(rect, Float)
-
-    submesh = build_submesh(nodes, triangles, boundary, quantities, \
-                            triangles_per_proc)
-
-    # send the mesh partition to the appropriate processor
-
-    for p in range(1, numprocs):
-      send_submesh(submesh, triangles_per_proc, p)
-
-    # Build the local mesh for processor 0
-    
-    points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict = \
-              extract_submesh(submesh, triangles_per_proc)
-
-# read in the mesh partition that belongs to this
-# processor (note that the information is in the
-# correct form for the GA data structure
-
-else:
-    [points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict] = \
-             rec_submesh(0)
-
-pypar.broadcast(rect,0)
-
-domain = Parallel_Domain(points, vertices, boundary,
-                                   full_send_dict  = full_send_dict,
-                                   ghost_recv_dict = ghost_recv_dict,
-                                   velocity = [0.1,0.0])
-
-# Make a notes of which triangles are full and which are ghost
-
-tri_full_flag = build_full_flag(domain, ghost_recv_dict)
-
-#domain.initialise_visualiser(rect=rect)
-
-#Boundaries
-
-T = Transmissive_boundary(domain)
-#R = Reflective_boundary(domain)
-domain.set_boundary( {'outflow': T, 'inflow': T, 'inner':T, 'exterior': T, 'open':T, 'ghost':None} )
-
-
-
-domain.set_quantity('stage', quantities['stage'])
-
-#---------
-# Evolution
-t0 = time.time()
-
-
-try:
-    domain.initialise_visualiser(rect=rect)
-    #domain.visualiser.coloring['stage'] = True
-    #domain.visualiser.scale_z['stage'] = 0.2
-except:
-    print 'No visualiser'
-
-
-from norms import linf_norm
-
-yieldstep = 1
-finaltime = 200
-
-#yieldstep = 1000
-#finaltime = 50000
-
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    if myid == 0:
-        domain.write_time()
-    print_test_stats(domain, tri_full_flag)
-
-if myid == 0:
-    print 'That took %.2f seconds' %(time.time()-t0)
Index: unk/anuga_core/source/anuga_parallel/run_parallel_sw_merimbula.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_merimbula.py	(revision 9423)
+++ 	(revision )
@@ -1,200 +1,0 @@
-
-"""Run parallel shallow water domain.
-
-   run using command like:
-
-   mpirun -np m python run_parallel_sw_merimbula.py
-
-   where m is the number of processors to be used.
-   
-   Will produce sww files with names domain_Pn_m.sww where m is number of processors and
-   n in [0, m-1] refers to specific processor that owned this part of the partitioned mesh.
-"""
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-
-import os
-import sys
-import time
-import numpy as num
-
-#------------------------
-# ANUGA Modules
-#------------------------
-	
-from anuga import Domain
-from anuga import Reflective_boundary
-from anuga import Dirichlet_boundary
-from anuga import Time_boundary
-from anuga import Transmissive_boundary
-from anuga import Transmissive_n_momentum_zero_t_momentum_set_stage_boundary
-
-from anuga import rectangular_cross
-from anuga import create_domain_from_file
-
-
-from anuga_parallel import distribute, myid, numprocs, finalize, barrier
-
-
-#--------------------------------------------------------------------------
-# Setup parameters
-#--------------------------------------------------------------------------
-
-#mesh_filename = "merimbula_10785_1.tsh" ; x0 = 756000.0 ; x1 = 756500.0; yieldstep = 10; finaltime = 100
-mesh_filename = "merimbula_17156.tsh"   ; x0 = 756000.0 ; x1 = 756500.0; yieldstep = 50; finaltime = 500
-#mesh_filename = "merimbula_43200_1.tsh"   ; x0 = 756000.0 ; x1 = 756500.0; yieldstep = 50; finaltime = 500
-#mesh_filename = "test-100.tsh" ; x0 = 200.0 ; x1 = 300.0; yieldstep = 1; finaltime = 10
-#mesh_filename = "test-20.tsh" ; x0 = 250.0 ; x1 = 350.0; yieldstep = 1; finaltime = 50
-
-verbose = False
-
-#--------------------------------------------------------------------------
-# Setup procedures
-#--------------------------------------------------------------------------
-class Set_Stage:
-    """Set an initial condition with constant water height, for x0<x<x1
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))+1.0
-
-
-class Set_Elevation:
-    """Set an elevation
-    """
-
-    def __init__(self, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return x/self.h
-    
-
-#--------------------------------------------------------------------------
-# Setup Domain only on processor 0
-#--------------------------------------------------------------------------
-if myid == 0:
-    domain = create_domain_from_file(mesh_filename)
-    domain.set_quantity('stage', Set_Stage(x0, x1, 1.0))
-    #domain.set_quantity('stage', 0.0)
-    #domain.set_datadir('.')
-    domain.set_name('merimbula_new')
-    domain.set_store(True)
-    #domain.set_store_vertices_smoothly(True)
-    domain.set_store_vertices_smoothly(False)
-    #domain.set_quantity('elevation', Set_Elevation(500.0))
-
-    #print domain.statistics()
-    #print domain.get_extent()
-    #print domain.get_extent(absolute=True)
-    #print domain.geo_reference
-else:
-    domain = None
-
-#--------------------------------------------------------------------------
-# Distribute sequential domain on processor 0 to other processors
-#--------------------------------------------------------------------------
-
-if myid == 0 and verbose: print 'DISTRIBUTING DOMAIN'
-domain = distribute(domain, verbose=verbose)
-
-#--------------------------------------------------------------------------
-# On all processors, setup evolve parameters for domains on all processors
-# (all called "domain"
-#--------------------------------------------------------------------------
-
-domain.set_flow_algorithm('2_0')
-domain.set_store_centroids()
-
-#for p in range(numprocs):
-#    if myid == p:
-#        print 'P%d'%p
-#        print domain.get_extent()
-#        print domain.get_extent(absolute=True)
-#        print domain.geo_reference
-#        print domain.s2p_map
-#        print domain.p2s_map
-#        print domain.tri_l2g
-#        print domain.node_l2g
-#    else:
-#        pass
-#
-#    barrier()
-
-
-domain.set_quantities_to_be_stored({'elevation':1,
-                                    'friction':1,
-                                    'stage':2,
-                                    'xmomentum':2,
-                                    'ymomentum':2})
-                                 
-#------------------------------------------------------------------------------
-# Setup boundary conditions
-# This must currently happen *after* domain has been distributed
-#------------------------------------------------------------------------------
-Br = Reflective_boundary(domain)      # Solid reflective wall
-
-from math import sin
-Bts = Transmissive_n_momentum_zero_t_momentum_set_stage_boundary(domain, lambda t: 10*sin(t/60))
-
-domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Bts})
-
-
-#bdy_ids = domain.tag_boundary_cells['open']
-#vol_ids = domain.boundary_cells[bdy_ids]
-#edge_ids = domain.boundary_edges[bdy_ids]
-
-#print domain.mesh.edge_midpoint_coordinates[3*vol_ids+edge_ids]
-
-
-
-#------------------------------------------------------------------------------
-# Evolution
-#------------------------------------------------------------------------------
-if myid == 0 and verbose: print 'EVOLVE'
-
-barrier()
-t0 = time.time()
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    #print 'P:'+ str(myid) + ' '+ domain.timestepping_statistics()
-    if myid == 0:
-        domain.write_time()
-
-
-barrier()
-
-for p in range(numprocs):
-    if myid == p:
-        print 'Processor %g ' %myid
-        print 'That took %.2f seconds' %(time.time()-t0)
-        print 'Communication time %.2f seconds'%domain.communication_time
-        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
-        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
-    else:
-        pass
-
-    barrier()
-
-
-#--------------------------------------------------
-# Merge the individual sww files into one file
-#--------------------------------------------------
-domain.sww_merge(delete_old=True)
-
-
-domain.dump_triangulation()
-
-finalize()
-
-
-
-
Index: unk/anuga_core/source/anuga_parallel/run_parallel_sw_merimbula_profile.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_merimbula_profile.py	(revision 9423)
+++ 	(revision )
@@ -1,186 +1,0 @@
-
-"""Run parallel shallow water domain.
-
-   run using command like:
-
-   mpirun -np m python run_parallel_sw_merimbula.py
-
-   where m is the number of processors to be used.
-   
-   Will produce sww files with names domain_Pn_m.sww where n is number of processors and
-   m in [0, n-1] refers to specific processor that owned this part of the partitioned mesh.
-"""
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-
-import os
-import sys
-import time
-import numpy as num
-
-#------------------------
-# ANUGA Modules
-#------------------------
-	
-from anuga import Domain
-from anuga import Reflective_boundary
-from anuga import Dirichlet_boundary
-from anuga import Time_boundary
-from anuga import Transmissive_boundary
-
-from anuga import rectangular_cross
-from anuga import create_domain_from_file
-
-
-from anuga_parallel import distribute, myid, numprocs, finalize, barrier
-
-
-#--------------------------------------------------------------------------
-# Setup parameters
-#--------------------------------------------------------------------------
-
-#mesh_filename = "merimbula_10785_1.tsh" ; x0 = 756000.0 ; x1 = 756500.0
-mesh_filename = "merimbula_43200.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
-#mesh_filename = "test-100.tsh" ; x0 = 0.25 ; x1 = 0.5
-
-finaltime = 500
-yieldstep = 50
-verbose = True
-
-#--------------------------------------------------------------------------
-# Setup procedures
-#--------------------------------------------------------------------------
-class Set_Stage:
-    """Set an initial condition with constant water height, for x0<x<x1
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1)) + 1.0
-
-#--------------------------------------------------------------------------
-# Setup Domain only on processor 0
-#--------------------------------------------------------------------------
-if myid == 0:
-    domain = create_domain_from_file(mesh_filename)
-    domain.set_quantity('stage', Set_Stage(x0, x1, 2.0))
-else:
-    domain = None
-
-#--------------------------------------------------------------------------
-# Distribute sequential domain on processor 0 to other processors
-#--------------------------------------------------------------------------
-
-if myid == 0 and verbose: print 'DISTRIBUTING DOMAIN'
-domain = distribute(domain)
-
-#domain.smooth = False
-barrier()
-for p in range(numprocs):
-    if myid == p:
-        print 'Process ID %g' %myid
-        print 'Number of triangles %g ' %domain.get_number_of_triangles()
-
-    barrier()
-
-
-domain.set_flow_algorithm(2.0)
-
-if myid == 0:
-    domain.print_algorithm_parameters()
-
-barrier()
-
-domain.set_name('meribula')
-
-#------------------------------------------------------------------------------
-# Setup boundary conditions
-# This must currently happen *after* domain has been distributed
-#------------------------------------------------------------------------------
-Br = Reflective_boundary(domain)      # Solid reflective wall
-
-domain.set_boundary({'exterior' :Br, 'open' :Br})
-
-
-barrier()
-for p in range(numprocs):
-    if myid == p:
-        print domain.boundary_statistics()
-
-    barrier()
-
-#------------------------------------------------------------------------------
-# Evolution
-#------------------------------------------------------------------------------
-if myid == 0 and verbose: print 'EVOLVE'
-
-t0 = time.time()
-
-
-s = """
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    if myid == 0:
-        domain.write_time()
-"""
-
-
-# Profiling
-import cProfile
-prof_file = 'evolve-prof'+ str(numprocs) + '_' + str(myid) + '.dat'
-cProfile.run(s,prof_file)
-
-
-barrier()
-
-if myid == 0:
-    import pstats
-    p = pstats.Stats(prof_file)
-    #p.sort_stats('cumulative').print_stats(25)
-
-
-    p.sort_stats('time').print_stats(25)
-
-barrier()
-
-
-if myid == 1:
-    import pstats
-    p = pstats.Stats(prof_file)
-    #p.sort_stats('cumulative').print_stats(25)
-
-
-    p.sort_stats('time').print_stats(25)
-
-    #p.print_stats()
-
-# Run evolve loop
-#result = profiler.runctx(s, globals(), locals())
-
-#result.dump_stats("profile." + str(numprocs) + "." + str(myid) + ".dat")
-#profiler.close()
-
-barrier()
-
-for p in range(numprocs):
-    if myid == p:
-        print 'Process ID %g' %myid
-        print 'Number of processors %g ' %numprocs
-        print 'That took %.2f seconds' %(time.time()-t0)
-        print 'Communication time %.2f seconds'%domain.communication_time
-        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
-        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
-
-    barrier()
-
-
-finalize()
-
-
-
-
Index: unk/anuga_core/source/anuga_parallel/run_parallel_sw_merimbula_test.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_merimbula_test.py	(revision 9423)
+++ 	(revision )
@@ -1,283 +1,0 @@
-#!/usr/bin/env python
-#########################################################
-#
-#  Main file for parallel mesh testing.
-#
-#  This is a modification of the run_parallel_advection.py
-# file.
-#
-#
-# *) The (new) files that have been added to manage the
-# grid partitioning are
-#    +) pmesh_divide_metis.py: subdivide a pmesh
-#    +) build_submesh.py: build the submeshes on the host
-# processor.
-#    +) build_local.py: build the GA mesh datastructure
-# on each processor.
-#    +) build_commun.py: handle the communication between
-# the host and processors
-#
-# *) Things still to do:
-#    +) Overlap the communication and computation: The
-# communication routines in build_commun.py should be
-# interdispersed in the build_submesh.py and build_local.py
-# files. This will overlap the communication and
-# computation and will be far more efficient. This should
-# be done after more testing and there more confidence in
-# the subpartioning.
-#    +) Much more testing especially with large numbers of
-# processors.
-#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
-# June 2005
-#
-#
-#
-#########################################################
-import sys
-import pypar    # The Python-MPI interface
-import time
-
-# Numeric arrays
-import numpy as num
-#from numpy import array, zeros, float
-
-# Print debugging information
-from print_stats import print_test_stats, build_full_flag
-
-# pmesh
-from anuga.shallow_water import Domain
-from parallel_shallow_water import Parallel_domain
-from anuga.abstract_2d_finite_volumes.pmesh2domain\
-     import pmesh_to_domain_instance
-
-# Reuse previous mesh import
-from anuga.caching import cache
-
-# Mesh partition routines
-from distribute_mesh  import pmesh_divide_metis
-from distribute_mesh  import build_submesh
-from distribute_mesh  import build_local_mesh
-from distribute_mesh  import send_submesh, rec_submesh, extract_submesh
-
-
-###############################
-# Read in processor information
-###############################
-
-numprocs = pypar.size()
-myid = pypar.rank()
-processor_name = pypar.get_processor_name()
-
-############################
-# Set the initial conditions
-############################
-
-rect = num.zeros( 4, num.float) # Buffer for results
-
-class Set_Stage:
-    """Set an initial condition with constant water height, for x<x0
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))
-
-#######################
-# Partition the domain
-#######################
-
-if myid == 0:
-
-    # Read in the test files
-
-#    filename = 'test-100.tsh'
-#    filename = 'merimbula_10785_1.tsh'
-    filename = 'merimbula_43200.tsh'
-
-    # Build the whole domain
-    
-    domain_full = pmesh_to_domain_instance(filename, Domain)
-
-#    domain_full = cache(pmesh_to_domain_instance,
-#               (filename, Domain),
-#              dependencies = [filename])
-
-    rect = num.array(domain_full.get_extent(), num.float)
-    print rect
-
-    # Initialise the wave
-
-    #domain_full.set_quantity('stage', Set_Stage(200.0,300.0,1.0))
-    domain_full.set_quantity('stage', Set_Stage(756000.0,756500.0,2.0))
-#    domain_full.set_quantity('stage', Set_Stage(756000.0,756500.0,0.0))
-
-    # Subdivide the domain
-
-    # Note the different arguments compared with pmesh_divide,
-    # pmesh_divide_steve etc.
-    
-    nodes, triangles, boundary, triangles_per_proc, quantities = \
-         pmesh_divide_metis(domain_full, numprocs)
-
-    print triangles_per_proc
-    
-    rect = num.array(domain_full.get_extent(), num.float)
-
-    submesh = build_submesh(nodes, triangles, boundary,\
-                            quantities, triangles_per_proc)
-
-    # Send the mesh partition to the appropriate processor
-
-    for p in range(1, numprocs):
-      send_submesh(submesh, triangles_per_proc, p)
-
-    # Build the local mesh for processor 0
-
-    points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict = \
-             extract_submesh(submesh, triangles_per_proc)
-
-# Read in the mesh partition that belongs to this
-# processor (note that the information is in the
-# correct form for the GA data structure
-
-else:
-    points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict , \
-            no_full_nodes, no_full_trigs = rec_submesh(0)
-
-
-###########################################
-# Start the computations on each subpartion
-###########################################
-
-#if myid == 0:
-#    print 'ghost'
-#    print ghost_recv_dict
-#processor_name
-#if myid == 0:
-#    print 'full'
-#    print full_send_dict
-
-# The visualiser needs to know the size of the whole domain
-
-pypar.broadcast(rect,0)
-
-domain = Parallel_domain(points, vertices, boundary,
-                         full_send_dict  = full_send_dict,
-                         ghost_recv_dict = ghost_recv_dict)
-
-# Make a note of which triangles are full and which are ghost
-
-tri_full_flag = build_full_flag(domain, ghost_recv_dict)
-
-try:
-    #domain.initialise_visualiser(rect=rect)
-    #domain.visualiser.coloring['stage'] = True
-    #domain.visualiser.scale_z['stage'] = 0.2
-    #domain.visualiser.scale_z['elevation'] = 0.05
-    pass
-except:
-    print 'No visualiser'
-
-
-
-domain.default_order = 1
-
-#Boundaries
-from anuga.interface import Transmissive_boundary, Reflective_boundary
-
-T = Transmissive_boundary(domain)
-R = Reflective_boundary(domain)
-domain.set_boundary( {'outflow': R, 'inflow': R, 'inner':R, 'exterior': R, 'open':R, 'ghost':None} )
-
-
-domain.set_quantity('stage', quantities['stage'])
-domain.set_quantity('elevation', quantities['elevation'])
-
-domain.store = False
-
-#---------
-# Evolution
-t0 = time.time()
-
-print 'Processor %d on %s: No of elements %d'%(domain.processor,processor_name,domain.number_of_elements)
-yieldstep = 50.0
-finaltime = 500.0
-
-#yieldstep = 1000
-#finaltime = 40000
-
-#yieldstep = 1
-#finaltime = 1
-#processor_name
-#for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-#    if myid == 0:
-#        domain.write_time()
-        #print 'Processor %d, Integral of stage %d'%\
-        #       (domain.processor,domain.quantities['stage'].get_integral())
-        #    print_test_stats(domain, tri_full_flag)
-
-
-# Profiling
-#import profile
-#profiler = profile.Profile()
-#result.dump_stats("profile." + str(numprocs) + "." + str(myid) + ".dat")
-
-## #New hotshot profiling
-## import hotshot
-## profiler = hotshot.Profile("hotshot." + str(numprocs) + "." + str(myid) + ".prof")
-## s = '''for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-##   if myid == 0:
-##     domain.write_time()
-##   print_test_stats(domain, tri_full_flag)
-
-## '''
-## result = profiler.runctx(s, globals(), locals())
-## profiler.close()
-
-#from vtk_realtime_visualiser import Visualiser
-#V = Visualiser(domain,default_scale_z=100.0)
-#V.coloring['stage'] = True
-#V.coloring['elevation'] = False
-#V.setup['elevation']=True
-#V.updating['stage']=True
-#V.qcolor['stage'] = (0.1,0.4,0.99)
-
-
-#V.start()
-#V.idle.wait()
-#V.idle.clear()
-
-
-
-
-
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    if myid == 0:
-        domain.write_time()
-        #print_test_stats(domain, tri_full_flag)
-
-#    V.redraw_ready.set()
-#    V.idle.wait()
-#    V.idle.clear()
-#    V.unpaused.wait()
-
-
-#print 'P%d: That took %.2f seconds' %(myid, time.time()-t0)
-#print 'P%d: Communication time %.2f seconds' %(myid, domain.communication_time)
-#print 'P%d: Reduction Communication time %.2f seconds' %(myid, domain.communication_reduce_time)
-#print 'P%d: Broadcast time %.2f seconds' %(myid, domain.communication_broadcast_time)
-
-
-
-if myid == 0:
-    print 'That took %.2f seconds' %(time.time()-t0)
-    print 'Communication time %.2f seconds'%domain.communication_time
-    print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
-    print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
-
-
-pypar.finalize()
Index: unk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangle.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangle.py	(revision 9423)
+++ 	(revision )
@@ -1,162 +1,0 @@
-#########################################################
-#
-#  Main file for parallel mesh testing.
-#
-#  This is a modification of the run_parallel_advection.py
-# file.
-#
-#
-#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
-# June 2005
-#
-#
-#
-#########################################################
-
-import time
-
-import numpy as num
-
-from print_stats import print_test_stats, build_full_flag
-
-#----------------------------
-# Sequential interface
-#---------------------------
-from anuga import Domain
-from anuga import Transmissive_boundary, Reflective_boundary
-
-#----------------------------
-# Parallel interface
-#---------------------------
-from anuga_parallel import Parallel_shallow_water_domain
-from anuga_parallel import parallel_rectangle
-from anuga_parallel import myid, numprocs, finalize, get_processor_name
-
-
-processor_name = get_processor_name()
-
-M = 50
-N = M*numprocs
-
-
-if myid == 0:
-    print 'N == %d' %N
-
-points, vertices, boundary, full_send_dict, ghost_recv_dict =\
-        parallel_rectangle(N, M, len1_g=1.0*numprocs, len2_g = 1.0)
-
-print "Myid = ", myid, "no points = ", len(points), \
-      "no vertices = ", len(vertices), "no boundaries = ", len(boundary)
-
-###########################################
-# Start the computations on each subpartion
-###########################################
-
-domain = Parallel_shallow_water_domain(points, vertices, boundary,
-                                       full_send_dict  = full_send_dict,
-                                       ghost_recv_dict = ghost_recv_dict)
-
-
-print 'after parallel domain'
-
-
-
-domain.set_name('sw_rectangle')
-
-#Boundaries
-T = Transmissive_boundary(domain)
-R = Reflective_boundary(domain)
-
-
-domain.set_boundary( {'left': R, 'right': R, 'bottom': R, 'top': R, 'ghost': None} )
-
-
-print 'after set_boundary'
-
-
-
-domain.check_integrity()
-
-class Set_Stage:
-    """Set an initial condition with constant water height, for x<x0
-    """
-
-    def __init__(self, x0=0.25, x1=0.75, y0=0.0, y1=1.0, h=5.0, h0=0.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.y0 = y0
-        self.y1 = y1
-        self.h  = h
-        self.h0 = h0
-
-    def __call__(self, x, y):
-        return self.h0 + self.h*((x>self.x0)&(x<self.x1)&(y>self.y0)&(y<self.y1))
-
-print 'after check_integrity'
-
-domain.set_quantity('stage', Set_Stage(0.2, 0.4, 0.25, 0.75, 1.0, 0.00))
-
-
-print 'after set quantity'
-
-# Set Evolve parameters
-domain.set_default_order(2)
-domain.set_timestepping_method('rk2')
-domain.set_CFL(0.7)
-domain.set_beta(1.5)
-
-#print domain.get_timestepping_method()
-#domain.use_edge_limiter = True
-#domain.tight_slope_limiters = True
-#domain.use_centroid_velocities = False
-
-
-print 'after evolve parameters'
-
-
-import pdb; pdb.set_trace()
-
-if myid == 0:
-    import time
-    t0 = time.time()
-
-
-# Turn on the visualisation
-visualise = False
-if visualise:
-    from anuga.visualiser import RealtimeVisualiser
-    vis = RealtimeVisualiser(domain)
-    vis.render_quantity_height("elevation", offset=0.001, dynamic=False)
-    vis.render_quantity_height("stage", dynamic=True)
-    vis.colour_height_quantity('stage', (0.2, 0.2, 0.8))
-    vis.start()
-    import time
-    time.sleep(2.0)
-
-
-
-yieldstep = 0.05
-finaltime = 2.0
-
-#Check that the boundary value gets propagated to all elements
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    if myid == 0:
-        domain.write_time()
-    #print_test_stats(domain, tri_full_flag)
-    if visualise:
-        vis.update()						
-
-
-
-if visualise: vis.evolveFinished()
-
-if myid == 0:
-    print 'That took %.2f seconds' %(time.time()-t0)
-    print 'Communication time %.2f seconds'%domain.communication_time
-    print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
-    print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
-
-
-if visualise: vis.join()
-
-finalize()
Index: unk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangular_cross.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangular_cross.py	(revision 9423)
+++ 	(revision )
@@ -1,162 +1,0 @@
-#########################################################
-#
-#  Main file for parallel mesh testing.
-#
-#  This is a modification of the run_parallel_advection.py
-# file.
-#
-#
-#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
-# June 2005
-#
-#
-#
-#########################################################
-
-import time
-import sys
-
-
-#----------------------------
-# Sequential interface
-#---------------------------
-from anuga import Transmissive_boundary, Reflective_boundary
-from anuga import rectangular_cross_domain
-from anuga import Set_quantity
-
-#----------------------------
-# Parallel interface
-#---------------------------
-from anuga_parallel import distribute, myid, numprocs, finalize, barrier
-
-
-t0 = time.time()
-
-verbose = True
-
-#--------------------------------------------------------------------------
-# Setup Domain only on processor 0
-#--------------------------------------------------------------------------
-if myid == 0:
-    length = 2.0
-    width = 2.0
-    #dx = dy = 0.005
-    #dx = dy = 0.00125
-    dx = dy  = 0.5
-    domain = rectangular_cross_domain(int(length/dx), int(width/dy),
-                                              len1=length, len2=width, origin=(-length/2, -width/2), verbose=verbose)
-
-
-    domain.set_store(True)
-    domain.set_quantity('elevation', lambda x,y : -1.0-x )
-    domain.set_quantity('stage', 1.0)
-    domain.set_flow_algorithm('DE0')
-    domain.print_statistics()
-else:
-    domain = None
-
-t1 = time.time()
-
-if myid == 0 :
-    print 'Create sequential domain: Time',t1-t0
-
-if myid == 0 and verbose: 
-    print 'DISTRIBUTING DOMAIN'
-    sys.stdout.flush()
-    
-barrier()
-
-#-------------------------------------------------------------------------
-# Distribute domain
-#-------------------------------------------------------------------------
-domain = distribute(domain,verbose=verbose)
-
-
-t2 = time.time()
-
-if myid == 0 :
-    print 'Distribute domain: Time ',t2-t1
-    
-if myid == 0 : print 'after parallel domain'
-
-
-
-domain.set_name('sw_rectangle')
-
-#Boundaries
-T = Transmissive_boundary(domain)
-R = Reflective_boundary(domain)
-
-
-domain.set_boundary( {'left': R, 'right': R, 'bottom': R, 'top': R, 'ghost': None} )
-
-
-if myid == 0 : print 'after set_boundary'
-
-
-
-#domain.check_integrity()
-
-if myid == 0 : print 'after check_integrity'
-
-class Set_Stage:
-    """Set an initial condition with constant water height, for x<x0
-    """
-
-    def __init__(self, x0=0.25, x1=0.75, y0=0.0, y1=1.0, h=5.0, h0=0.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.y0 = y0
-        self.y1 = y1
-        self.h  = h
-        self.h0 = h0
-
-    def __call__(self, x, y):
-        return self.h0 + self.h*((x>self.x0)&(x<self.x1)&(y>self.y0)&(y<self.y1))
-
-
-
-#domain.set_quantity('stage', Set_Stage(0.2, 0.4, 0.25, 0.75, 1.0, 0.00))
-
-Set_quantity(domain,'stage',center=(0.0,0.0), radius=0.5, value = 2.0)
-
-if myid == 0 : print 'after set quantity'
-
-
-
-yieldstep = 0.005
-finaltime = 0.05
-
-barrier()
-
-t0 = time.time()
-
-#===========================================================================
-# Main Evolve Loop
-#===========================================================================
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    if myid == 0:
-        domain.write_time()
-        sys.stdout.flush()
-        
-        
-
-
-for p in range(numprocs):
-    barrier()
-    if myid == p:
-        print 50*'='
-        print 'P%g' %(myid)
-        print 'That took %.2f seconds' %(time.time()-t0)
-        print 'Communication time %.2f seconds'%domain.communication_time
-        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
-        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
-        sys.stdout.flush()
-
-
-if domain.number_of_global_triangles < 50000:
-    if myid == 0 :
-        print 'Create dump of triangulation for %g triangles' % domain.number_of_global_triangles
-    domain.dump_triangulation(filename="rectangular_cross_%g.png"% numprocs)
-
-finalize()
Index: unk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangular_cross_profile.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangular_cross_profile.py	(revision 9423)
+++ 	(revision )
@@ -1,182 +1,0 @@
-
-"""Run parallel shallow water domain.
-
-   run using command like:
-
-   mpirun -np m python run_parallel_sw_rectangular_cross.py
-
-   where m is the number of processors to be used.
-   
-   Will produce sww files with names domain_Pn_m.sww where n is number of processors and
-   m in [0, n-1] refers to specific processor that owned this part of the partitioned mesh.
-"""
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-
-import os
-import sys
-import time
-import numpy as num
-
-#------------------------
-# ANUGA Modules
-#------------------------
-	
-from anuga import Domain
-from anuga import Reflective_boundary
-from anuga import Dirichlet_boundary
-from anuga import Time_boundary
-from anuga import Transmissive_boundary
-
-from anuga import rectangular_cross_domain
-from anuga import create_domain_from_file
-
-
-from anuga_parallel import distribute, myid, numprocs, finalize, barrier
-
-
-#--------------------------------------------------------------------------
-# Setup parameters
-#--------------------------------------------------------------------------
-
-#mesh_filename = "merimbula_10785_1.tsh" ; x0 = 756000.0 ; x1 = 756500.0
-#mesh_filename = "merimbula_43200.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
-#mesh_filename = "test-100.tsh" ; x0 = 0.25 ; x1 = 0.5
-
-
-verbose = True
-
-#--------------------------------------------------------------------------
-# Setup procedures
-#--------------------------------------------------------------------------
-class Set_Stage:
-    """Set an initial condition with constant water height, for x0<x<x1
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))
-
-#--------------------------------------------------------------------------
-# Setup Domain only on processor 0
-#--------------------------------------------------------------------------
-if myid == 0:
-    length = 2.0
-    width = 2.0
-    dx = dy = 0.01
-    domain = rectangular_cross_domain(int(length/dx), int(width/dy),
-                                              len1=length, len2=width)
-
-    domain.set_quantity('elevation', -1.0)
-    domain.set_quantity('stage', Set_Stage(h=2.0))
-else:
-    domain = None
-
-#--------------------------------------------------------------------------
-# Distribute sequential domain on processor 0 to other processors
-#--------------------------------------------------------------------------
-
-if myid == 0 and verbose: print 'DISTRIBUTING DOMAIN'
-domain = distribute(domain)
-
-#domain.smooth = False
-barrier()
-for p in range(numprocs):
-    if myid == p:
-        print 'Process ID %g' %myid
-        print 'Number of triangles %g ' %domain.get_number_of_triangles()
-        sys.stdout.flush()
-        
-    barrier()
-
-
-domain.set_flow_algorithm(2.0)
-
-if myid == 0:
-    domain.print_algorithm_parameters()
-    sys.stdout.flush()
-    
-barrier()
-
-domain.set_name('rectangular_cross')
-domain.set_store(False)
-
-#------------------------------------------------------------------------------
-# Setup boundary conditions
-# This must currently happen *after* domain has been distributed
-#------------------------------------------------------------------------------
-Br = Reflective_boundary(domain)      # Solid reflective wall
-
-domain.set_boundary({'top' :Br, 'bottom' :Br, 'left' :Br, 'right' :Br, 'ghost' : None })
-
-"""
-barrier()
-for p in range(numprocs):
-    if myid == p:
-        print domain.boundary_statistics()
-        sys.stdout.flush()
-        
-    barrier()
-"""
-
-
-domain.dump_triangulation()
-
-#------------------------------------------------------------------------------
-# Evolution
-#------------------------------------------------------------------------------
-if myid == 0 and verbose: print 'EVOLVE'
-
-t0 = time.time()
-finaltime = 0.25
-yieldstep = 0.05
-
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    if myid == 0:
-        domain.write_time()
-
-
-
-## Profiling
-#import cProfile
-#prof_file = 'evolve-prof'+ str(numprocs) + '_' + str(myid) + '.dat'
-#cProfile.run(s,prof_file)
-
-
-barrier()
-
-#for id in range(numprocs):
-#    if myid == id:
-#        import pstats
-#        p = pstats.Stats(prof_file)
-#        #p.sort_stats('cumulative').print_stats(25)
-#        p.sort_stats('time').print_stats(25)
-#        sys.stdout.flush
-#
-#    barrier()
-#
-#barrier()
-
-for p in range(numprocs):
-    if myid == p:
-        print 'Process ID %g' %myid
-        print 'Number of processors %g ' %numprocs
-        print 'That took %.2f seconds' %(time.time()-t0)
-        print 'Communication time %.2f seconds'%domain.communication_time
-        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
-        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
-
-    barrier()
-
-
-finalize()
-
-
-
-
Index: unk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangular_tsunami.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_rectangular_tsunami.py	(revision 9423)
+++ 	(revision )
@@ -1,159 +1,0 @@
-#########################################################
-#
-#  Main file for parallel mesh testing.
-#
-#  This is a modification of the run_parallel_advection.py
-# file.
-#
-#
-#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
-# June 2005
-#
-#
-#
-#########################################################
-
-import time
-import sys
-import numpy
-import anuga
-
-
-#----------------------------
-# Parallel interface
-#---------------------------
-from anuga_parallel import distribute, myid, numprocs, finalize, barrier
-
-
-#--------------------------------------------------------------------------
-# Setup functions for topograpy etc
-#--------------------------------------------------------------------------
-scale_me=1.0
-
-def topography(x,y):
-	return (-x/2.0 +0.05*numpy.sin((x+y)*200.0))*scale_me
-
-def stagefun(x,y):
-    stge=-0.2*scale_me #+0.01*(x>0.9)
-    #topo=topography(x,y)
-    return stge#*(stge>topo) + (topo)*(stge<=topo)
-
-
-
-#--------------------------------------------------------------------------
-# Create domains
-#--------------------------------------------------------------------------
-
-
-
-t0 = time.time()
-
-verbose = True
-
-#--------------------------------------------------------------------------
-# Setup Domain only on processor 0
-#--------------------------------------------------------------------------
-if myid == 0:
-    length = 2.0
-    width = 2.0
-    dx = dy = 0.005
-    #dx = dy = 0.00125
-    dx = dy  = 0.05
-    domain = anuga.rectangular_cross_domain(int(length/dx), int(width/dy),
-                                              len1=length, len2=width, verbose=verbose)
-
-    #---------------------------------------
-    # Add these two commands to use Gareth's
-    # tsunami algorithm. Play with the 
-    # minimum allowed height to remove possible 
-    # unrealistic large velocities
-    #---------------------------------------
-    domain.set_flow_algorithm('tsunami')
-    domain.set_minimum_allowed_height(0.01)
-
-    domain.set_store(True)
-    domain.set_quantity('elevation',topography)     # Use function for elevation
-    domain.get_quantity('elevation').smooth_vertex_values()
-    domain.set_quantity('friction',0.03)            # Constant friction
-    domain.set_quantity('stage', stagefun)          # Constant negative initial stage
-    domain.get_quantity('stage').smooth_vertex_values()
-
-    domain.set_name('rectangular_tsunami')
-
-    domain.print_statistics()
-else:
-    domain = None
-
-t1 = time.time()
-
-if myid == 0 :
-    print 'Create sequential domain ',t1-t0
-
-if myid == 0 and verbose: 
-    print 'DISTRIBUTING DOMAIN'
-    sys.stdout.flush()
-    
-barrier()
-
-#-------------------------------------------------------------------------
-# Distribute domain
-#-------------------------------------------------------------------------
-domain = distribute(domain,verbose=verbose)
-
-
-t2 = time.time()
-
-if myid == 0 :
-    print 'Distribute domain ',t2-t1
-    
-if myid == 0 : print 'after parallel domain'
-
-
-#Boundaries
-T = anuga.Transmissive_boundary(domain)
-R = anuga.Reflective_boundary(domain)
-D = anuga.Dirichlet_boundary([-0.1*scale_me,0.,0.])
-
-
-domain.set_boundary( {'left': R, 'right': D, 'bottom': R, 'top': R, 'ghost': None} )
-
-
-if myid == 0 : print 'after set_boundary'
-
-
-yieldstep = 0.2
-finaltime = 20.0
-
-barrier()
-
-t0 = time.time()
-
-#===========================================================================
-# Main Evolve Loop
-#===========================================================================
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    if myid == 0:
-        domain.write_time()
-        sys.stdout.flush()
-						
-
-
-
-for p in range(numprocs):
-    barrier()
-    if myid == p:
-        print 50*'='
-        print 'P%g' %(myid)
-        print 'That took %.2f seconds' %(time.time()-t0)
-        print 'Communication time %.2f seconds'%domain.communication_time
-        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
-        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
-        sys.stdout.flush()
-
-
-if domain.number_of_global_triangles < 50000:
-    if myid == 0 :
-        print 'Create dump of triangulation for %g triangles' % domain.number_of_global_triangles
-    domain.dump_triangulation(filename="rectangular_cross_%g.png"% numprocs)
-
-finalize()
Index: unk/anuga_core/source/anuga_parallel/run_sequential_dist_all_merimbula.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_sequential_dist_all_merimbula.py	(revision 9423)
+++ 	(revision )
@@ -1,159 +1,0 @@
-"""Run parallel shallow water domain.
-
-   run using command like:
-
-   mpirun -np m python run_sequential_dist_all.py
-
-   where m is the number of processors to be used.
-   
-"""
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-
-import os
-import sys
-import time
-import numpy as num
-
-#------------------------
-# ANUGA Modules
-#------------------------
-	
-from anuga import Domain
-from anuga import Reflective_boundary
-from anuga import Dirichlet_boundary
-from anuga import Time_boundary
-from anuga import Transmissive_boundary
-
-from anuga import rectangular_cross
-from anuga import create_domain_from_file
-
-
-#from anuga_parallel.sequential_distribute import sequential_distribute
-
-from anuga_parallel import distribute, myid, numprocs, finalize, barrier
-
-
-#--------------------------------------------------------------------------
-# Setup parameters
-#--------------------------------------------------------------------------
-
-#mesh_filename = "merimbula_10785_1.tsh" ; x0 = 756000.0 ; x1 = 756500.0
-mesh_filename = "merimbula_17156.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
-#mesh_filename = "merimbula_43200_1.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
-#mesh_filename = "test-100.tsh" ; x0 = 0.25 ; x1 = 0.5
-#mesh_filename = "test-20.tsh" ; x0 = 250.0 ; x1 = 350.0
-yieldstep = 50
-finaltime = 1500
-verbose = True
-
-#--------------------------------------------------------------------------
-# Setup procedures
-#--------------------------------------------------------------------------
-class Set_Stage:
-    """Set an initial condition with constant water height, for x0<x<x1
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))+1.0
-
-
-class Set_Elevation:
-    """Set an elevation
-    """
-
-    def __init__(self, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return x/self.h
-    
-
-#--------------------------------------------------------------------------
-# Setup Sequential Domain
-#--------------------------------------------------------------------------
-if myid == 0 :
-    domain = create_domain_from_file(mesh_filename)
-    domain.set_quantity('stage', Set_Stage(x0, x1, 2.0))
-    #domain.set_datadir('.')
-    domain.set_name('merimbula_new')
-    domain.set_store(True)
-else:
-    domain = None
-
-
-#--------------------------------------------------------------------------
-# Distribute sequential domain on processor 0 to other processors
-#--------------------------------------------------------------------------
-if verbose: print 'DISTRIBUTING DOMAIN'
-domain = distribute(domain)
-
-
-#--------------------------------------------------------------------------
-# On all processors, setup evolve parameters for domains on all processors
-# (all called "domain"
-#--------------------------------------------------------------------------
-
-domain.set_flow_algorithm('DE0')
-
-#------------------------------------------------------------------------------
-# Setup boundary conditions
-# This must currently happen *after* domain has been distributed
-#------------------------------------------------------------------------------
-Br = Reflective_boundary(domain)      # Solid reflective wall
-
-domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Br})
-
-#------------------------------------------------------------------------------
-# Evolution
-#------------------------------------------------------------------------------
-if myid == 0 and verbose: print 'EVOLVE'
-
-barrier()
-t0 = time.time()
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    if myid == 0:
-        domain.write_time()
-
-
-barrier()
-
-for p in range(numprocs):
-    if myid == p:
-        print 'Processor %g ' %myid
-        print 'That took %.2f seconds' %(time.time()-t0)
-        print 'Communication time %.2f seconds'%domain.communication_time
-        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
-        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
-    else:
-        pass
-
-    barrier()
-
-
-#--------------------------------------------------
-# Merge the individual sww files into one file
-#--------------------------------------------------
-domain.sww_merge(delete_old=False)
-
-finalize()
-
-
-
-
-
-
-
-
-
-
-
Index: unk/anuga_core/source/anuga_parallel/run_sequential_dist_distribute_merimbula.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_sequential_dist_distribute_merimbula.py	(revision 9423)
+++ 	(revision )
@@ -1,100 +1,0 @@
-
-"""Run parallel shallow water domain.
-
-   run using command like:
-
-   python run_sequential_dist_distribute_merimbula.py
-
-   will produce 4 pickled files corresponding partitioned domains
-   
-"""
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-
-import os
-import sys
-import time
-import numpy as num
-
-#------------------------
-# ANUGA Modules
-#------------------------
-	
-from anuga import Domain
-from anuga import Reflective_boundary
-from anuga import Dirichlet_boundary
-from anuga import Time_boundary
-from anuga import Transmissive_boundary
-
-from anuga import rectangular_cross
-from anuga import create_domain_from_file
-
-
-from anuga_parallel.sequential_distribute import sequential_distribute_dump
-
-
-#--------------------------------------------------------------------------
-# Setup parameters
-#--------------------------------------------------------------------------
-
-#mesh_filename = "merimbula_10785_1.tsh" ; x0 = 756000.0 ; x1 = 756500.0
-mesh_filename = "merimbula_17156.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
-#mesh_filename = "merimbula_43200_1.tsh"   ; x0 = 756000.0 ; x1 = 756500.0
-#mesh_filename = "test-100.tsh" ; x0 = 0.25 ; x1 = 0.5
-#mesh_filename = "test-20.tsh" ; x0 = 250.0 ; x1 = 350.0
-yieldstep = 50
-finaltime = 1500
-verbose = True
-
-#--------------------------------------------------------------------------
-# Setup procedures
-#--------------------------------------------------------------------------
-class Set_Stage:
-    """Set an initial condition with constant water height, for x0<x<x1
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))+1.0
-
-
-class Set_Elevation:
-    """Set an elevation
-    """
-
-    def __init__(self, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return x/self.h
-    
-
-#--------------------------------------------------------------------------
-# Setup Sequential Domain
-#--------------------------------------------------------------------------
-domain = create_domain_from_file(mesh_filename)
-domain.set_quantity('stage', Set_Stage(x0, x1, 2.0))
-#domain.set_datadir('.')
-domain.set_name('merimbula_new')
-domain.set_store(True)
-
-
-#--------------------------------------------------------------------------
-# Distribute sequential domain on processor 0 to other processors
-#--------------------------------------------------------------------------
-
-if verbose: print 'DISTRIBUTING DOMAIN'
-sequential_distribute_dump(domain, 4, verbose=True)
-
-
-
-
-
Index: unk/anuga_core/source/anuga_parallel/run_sequential_dist_distribute_rectangular.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_sequential_dist_distribute_rectangular.py	(revision 9423)
+++ 	(revision )
@@ -1,96 +1,0 @@
-#########################################################
-#
-#  Main file for parallel mesh testing.
-#
-#  This is a modification of the run_parallel_advection.py
-# file.
-#
-#
-#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
-# June 2005
-#
-#
-#
-#########################################################
-
-import time
-import sys
-import numpy
-
-
-#----------------------------
-# Sequential interface
-#---------------------------
-from anuga import Transmissive_boundary, Reflective_boundary
-from anuga import rectangular_cross_domain
-
-#----------------------------
-# Parallel interface
-#---------------------------
-from anuga_parallel.sequential_distribute import sequential_distribute_dump
-
-
-t0 = time.time()
-
-verbose = True
-
-#--------------------------------------------------------------------------
-# Setup functions for topograpy etc
-#--------------------------------------------------------------------------
-scale_me=1.0
-
-def topography(x,y):
-	return (-x/2.0 +0.05*numpy.sin((x+y)*200.0))*scale_me
-
-def stagefun(x,y):
-    stge=-0.2*scale_me #+0.01*(x>0.9)
-    #topo=topography(x,y)
-    return stge#*(stge>topo) + (topo)*(stge<=topo)
-
-
-#--------------------------------------------------------------------------
-# Setup Domain only on processor 0
-#--------------------------------------------------------------------------
-myid = 0
-numprocs = 3
-
-
-
-length = 2.0
-width = 2.0
-dx = dy = 0.005  # 640,000
-dx = dy  = 0.05
-domain = rectangular_cross_domain(int(length/dx), int(width/dy),
-                                          len1=length, len2=width, verbose=verbose)
-
-
-print domain.number_of_global_triangles
-
-domain.set_store(True)
-domain.set_flow_algorithm('DE0')
-domain.set_minimum_allowed_height(0.01)
-domain.set_quantity('elevation',topography)     # Use function for elevation
-domain.get_quantity('elevation').smooth_vertex_values()
-domain.set_quantity('friction',0.03)            # Constant friction
-domain.set_quantity('stage', stagefun)          # Constant negative initial stage
-domain.get_quantity('stage').smooth_vertex_values()
-
-domain.set_name('sw_rectangle')
-
-
-t1 = time.time()
-print 'Create sequential domain: time ',t1-t0
-    
-
-#-------------------------------------------------------------------------
-# Distribute domain
-#----------------------------------------------------------------------
-sequential_distribute_dump(domain, numprocs, verbose = True)
-
-
-t2 = time.time()
-print 'Sequential Distribute domain: time ',t2-t1
-    
-
-
-
Index: unk/anuga_core/source/anuga_parallel/run_sequential_dist_evolve_merimbula.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_sequential_dist_evolve_merimbula.py	(revision 9423)
+++ 	(revision )
@@ -1,109 +1,0 @@
-
-"""Run parallel shallow water domain.
-
-   run using command like:
-
-   mpirun -np 4  python run_sequential_dist_evolve_merimbula.py
-
-   Need to have run run_sequential_dist_distribute_merimbula.py
-
-   first to produce the required pickle files encoding the partitioning
-   """
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-
-import os
-import sys
-import time
-import numpy as num
-
-#------------------------
-# ANUGA Modules
-#------------------------
-	
-from anuga import Domain
-from anuga import Reflective_boundary
-from anuga import Dirichlet_boundary
-from anuga import Time_boundary
-from anuga import Transmissive_boundary
-
-from anuga import rectangular_cross
-from anuga import create_domain_from_file
-
-
-from anuga_parallel import distribute, myid, numprocs, finalize, barrier
-
-from anuga_parallel.sequential_distribute import sequential_distribute_load
-
-#--------------------------------------------------------------------------
-# Setup parameters
-#--------------------------------------------------------------------------
-yieldstep = 50
-finaltime = 1500
-verbose = True
-
-
-
-#-------------------------------------------------------------------------------
-# Read in cPickled distribute mesh data
-#-------------------------------------------------------------------------------
-domain = sequential_distribute_load(filename='merimbula_new', verbose = True)
-
-#--------------------------------------------------------------------------
-# On all processors, setup evolve parameters for domains on all processors
-# (all called "domain"
-#--------------------------------------------------------------------------
-
-domain.set_flow_algorithm('DE0')
-
-
-
-#------------------------------------------------------------------------------
-# Setup boundary conditions
-# This must currently happen *after* domain has been distributed
-#------------------------------------------------------------------------------
-Br = Reflective_boundary(domain)      # Solid reflective wall
-
-domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Br})
-
-#------------------------------------------------------------------------------
-# Evolution
-#------------------------------------------------------------------------------
-if myid == 0 and verbose: print 'EVOLVE'
-
-barrier()
-t0 = time.time()
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    if myid == 0:
-        domain.write_time()
-
-
-barrier()
-
-for p in range(numprocs):
-    if myid == p:
-        print 'Processor %g ' %myid
-        print 'That took %.2f seconds' %(time.time()-t0)
-        print 'Communication time %.2f seconds'%domain.communication_time
-        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
-        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
-    else:
-        pass
-
-    barrier()
-
-
-#--------------------------------------------------
-# Merge the individual sww files into one file
-#--------------------------------------------------
-
-
-
-
-finalize()
-
-
-
-
Index: unk/anuga_core/source/anuga_parallel/run_sequential_dist_parallel_evolve_rectangular.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_sequential_dist_parallel_evolve_rectangular.py	(revision 9423)
+++ 	(revision )
@@ -1,134 +1,0 @@
-#########################################################
-#
-#  Main file for parallel mesh testing.
-#
-#  This is a modification of the run_parallel_advection.py
-# file.
-#
-#
-#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
-# June 2005
-#
-#
-#
-#########################################################
-
-import time
-import sys
-import numpy
-import anuga
-
-
-#----------------------------
-# Parallel interface
-#---------------------------
-from anuga_parallel import myid, numprocs, finalize, barrier
-
-from anuga_parallel.sequential_distribute import sequential_distribute_load
-
-
-#--------------------------------------------------------------------------
-# Setup functions for topograpy etc
-#--------------------------------------------------------------------------
-scale_me=1.0
-
-def topography(x,y):
-	return (-x/2.0 +0.05*numpy.sin((x+y)*200.0))*scale_me
-
-def stagefun(x,y):
-    stge=-0.2*scale_me #+0.01*(x>0.9)
-    #topo=topography(x,y)
-    return stge#*(stge>topo) + (topo)*(stge<=topo)
-
-
-
-
-verbose = True
-basename = 'sw_rectangle'
-#basename = 'test'
-
-#--------------------------------------------------------------------------
-# Create domains
-#--------------------------------------------------------------------------
-#import cPickle
-#pickle_name = pickle_base+'_P%g_%g.pickle'% (numprocs,myid)
-#f = file(pickle_name, 'rb')
-# = cPickle.load(f)
-#f.close()
-
-
-t1 = time.time()
-
-if myid == 0 and verbose:
-    print 'Loading DISTRIBUTING DOMAIN'
-    sys.stdout.flush()
-
-#-------------------------------------------------------------------------
-# Distribute domain
-#-------------------------------------------------------------------------
-domain = sequential_distribute_load(filename = basename, verbose=verbose)
-
-
-t2 = time.time()
-
-if myid == 0 :
-    print 'Load Distribute domain ',t2-t1
-    
-if myid == 0 : print 'after parallel domain'
-
-
-#Boundaries
-T = anuga.Transmissive_boundary(domain)
-R = anuga.Reflective_boundary(domain)
-D = anuga.Dirichlet_boundary([-0.1*scale_me,0.,0.])
-
-
-domain.set_boundary( {'left': R, 'right': D, 'bottom': R, 'top': R, 'ghost': None} )
-
-
-if myid == 0 : print 'after set_boundary'
-
-
-yieldstep = 0.2
-finaltime = 20.0
-
-barrier()
-
-t0 = time.time()
-
-#===========================================================================
-# Main Evolve Loop
-#===========================================================================
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    if myid == 0:
-        domain.write_time()
-        sys.stdout.flush()
-						
-
-
-
-for p in range(numprocs):
-    barrier()
-    if myid == p:
-        print 50*'='
-        print 'P%g' %(myid)
-        print 'That took %.2f seconds' %(time.time()-t0)
-        print 'Communication time %.2f seconds'%domain.communication_time
-        print 'Reduction Communication time %.2f seconds'%domain.communication_reduce_time
-        print 'Broadcast time %.2f seconds'%domain.communication_broadcast_time
-        sys.stdout.flush()
-
-
-if domain.number_of_global_triangles < 50000:
-    if myid == 0 :
-        print 'Create dump of triangulation for %g triangles' % domain.number_of_global_triangles
-    domain.dump_triangulation(filename="rectangular_cross_%g.png"% numprocs)
-
-
-#--------------------------------------------------
-# Merge the individual sww files into one file
-#--------------------------------------------------
-domain.sww_merge(delete_old=False)
-
-
-finalize()
Index: unk/anuga_core/source/anuga_parallel/run_sw_lost_mass.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_sw_lost_mass.py	(revision 9423)
+++ 	(revision )
@@ -1,108 +1,0 @@
-#!/usr/bin/env/python
-#########################################################
-#
-# Shows problem of conservation of mass. Due to material being
-# removed if too shallow. Can lead to problems if timestep
-# set very small.
-#
-#
-#
-#  Authors: Linda Stals, Steve Roberts and Matthew Hardy,
-# June 2005
-#
-#
-#
-#########################################################
-
-
-##############################################
-# Set a small yieldstep (say 0.005) to see
-# lose of conservation
-# Set it to a larger value (say 0.1) and have
-# conservation "restored"
-##############################################
-yieldstep = 0.005
-#yieldstep = 0.08
-#yieldstep = 0.05
-finaltime = 100.0
-
-import sys
-import time
-
-
-from Numeric import array, zeros, Float
-
-#from shallow_water import Domain
-
-from anuga.shallow_water import Domain
-
-# mesh partition routines
-
-from anuga.abstract_2d_finite_volumes.pmesh2domain\
-     import pmesh_to_domain_instance
-
-# read in the processor information
-
-
-#-------
-# Domain
-rect = zeros( 4, Float) # Buffer for results
-
-class Set_Stage:
-    """Set an initial condition with constant water height, for x0<x<x1
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))
-
-
-
-    # read in the test files
-
-filename = 'test-100.tsh'
-
-nx = 1
-ny = 1
-
-domain = pmesh_to_domain_instance(filename, Domain)
-
-domain.set_quantity('stage', Set_Stage(200.0,300.0,5.0))
-rect = array(domain.xy_extent, Float)
-
-try:
-    domain.initialise_visualiser(rect=rect)
-    #domain.visualiser.coloring['stage'] = True
-    domain.visualiser.scale_z['stage'] = 0.2
-    domain.visualiser.scale_z['elevation'] = 0.05
-except:
-    print 'No visualiser'
-
-
-domain.default_order = 2
-
-#Boundaries
-from shallow_water import Transmissive_boundary, Reflective_boundary
-
-T = Transmissive_boundary(domain)
-R = Reflective_boundary(domain)
-domain.set_boundary( {'outflow': R, 'inflow': R, 'inner':R, 'exterior': R, 'open':R} )
-
-
-
-#---------
-# Evolution
-t0 = time.time()
-
-print 'No of triangles %d'%(domain.number_of_triangles)
-
-
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    #domain.write_time()
-    print '    Integral stage = ', domain.quantities['stage'].get_integral(),' Time = ',domain.time
-
-
Index: unk/anuga_core/source/anuga_parallel/run_sw_lost_mass_simpler.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_sw_lost_mass_simpler.py	(revision 9423)
+++ 	(revision )
@@ -1,74 +1,0 @@
-#!/usr/bin/env python
-#########################################################
-#
-# Shows problem of conservation of mass. Due to material being
-# removed if too shallow. Can lead to problems if timestep
-# set very small.
-#
-#
-#
-#  Authors: Linda Stals, Steve Roberts, Matthew Hardy, Ole Nielsen
-#  July 2005
-#
-#
-#########################################################
-
-
-##############################################
-# Change min_depth and see how larger values aggravate the problem
-yieldstep = 0.1
-finaltime = 10.0
-#min_depth = 1.0e-4
-min_depth = 1.0e-2
-
-
-from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular
-from anuga.shallow_water import Domain, Reflective_boundary
-
-
-
-#Create shallow water domain
-points, vertices, boundary = rectangular(50, 50, len1=500, len2=500)
-domain = Domain(points, vertices, boundary)
-domain.smooth = False
-domain.visualise = True
-domain.default_order = 1
-domain.minimum_allowed_height = min_depth
-print 'Extent', domain.get_extent()
-
-# Set initial condition
-class Set_IC:
-    """Set an initial condition with a constant value, for x0<x<x1
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))
-
-
-domain.set_quantity('stage', Set_IC(200.0,300.0,5.0))
-
-try:
-    domain.initialise_visualiser()
-except:
-    print 'No visualiser'
-else:    
-    domain.visualiser.scale_z['stage'] = 0.2
-    domain.visualiser.scale_z['elevation'] = 0.05
-    
-
-#Boundaries
-R = Reflective_boundary(domain)
-domain.set_boundary( {'left': R, 'right': R, 'top':R, 'bottom': R} )
-
-
-# Evolution
-print 'Minimal allowed water height = ', domain.minimum_allowed_height 
-for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-    print 'Integral stage = ', domain.quantities['stage'].get_integral(),' Time = ',domain.time
-
-
Index: unk/anuga_core/source/anuga_parallel/test_all.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_all.py	(revision 9423)
+++ 	(revision )
@@ -1,223 +1,0 @@
-"""Regression testing framework
-This module will search for scripts in the same directory named
-test_*.py.  Each such script should be a test suite that tests a
-module through PyUnit. This script will aggregate all
-found test suites into one big test suite and run them all at once.
-"""
-
-# Author: Mark Pilgrim
-# Modified by Ole Nielsen
-
-import unittest
-import os
-import sys
-import tempfile
-import time
-import anuga.utilities.system_tools as aust
-from anuga.utilities.terminal_width import terminal_width
-
-
-
-
-
-#List files that should be excluded from the testing process.
-#E.g. if they are known to fail and under development
-exclude_files = ['test_failure.py', 'test_parallel_boyd_pipe_operator.py', 'test_parallel_boyd_box_op_apron.py']
-
-# Directories that should not be searched for test files.
-exclude_dirs = ['documentation',               # Special requirements
-                '.svn',                              # subversion
-                'props', 'wcprops', 'prop-base', 'text-base', 'tmp']
-
-
-##
-# @brief List a string sequence on the screen in columns.
-# @param names Sequence of strings to list.
-# @param func Function to apply to each string in sequence.
-# @param col_width Force columns to this width (default calculated).
-# @param page_width Set displayable page width to this (default 132).
-def list_names(names, func=None, col_width=None, page_width=None):
-    # set defaults
-    p_width = page_width - 1            # set page width
-    if p_width is None:
-        p_width = 132                   # default page width
-
-    c_width = col_width                 # set column width
-    if c_width is None:
-        c_width = 0
-        for name in names:
-            if func:
-                name = func(name)
-            c_width = max(c_width, len(name))
-    c_width += 2                        # 2 column padding
-
-    # calculate number of columns allowed
-    max_columns = int(p_width / c_width)
-
-    # print columns
-    column = 0
-    for name in names:
-        if func:
-            name = func(name)
-        print '%-*s' % (c_width-1, name),
-        column += 1
-        if column >= max_columns:
-            column = 0
-            print
-
-    # if last line not finished, end it here
-    if column > 0:
-        print
-
-
-##
-# @brief Get 'test_*.py' files and paths to directories.
-# @param path Path to directory to start walk in.
-# @return A tuple (<files>, <dirs>).
-# @note Don't include any files in and below forbidden directories.
-def get_test_files(path):
-    walk = os.walk(path)
-
-    test_files = []
-    path_files = []
-
-    for (dirpath, dirnames, filenames) in walk:
-        # exclude forbidden directories
-        for e_dir in exclude_dirs:
-            try:
-                dirnames.remove(e_dir)
-            except ValueError:
-                pass
-
-        # check for test_*.py files
-        for filename in filenames:
-            if filename.startswith('test_') and filename.endswith('.py'):
-                test_files.append(filename)
-                if dirpath not in path_files:
-                    path_files.append(dirpath)
-
-    return test_files, path_files
-
-
-def regressionTest(test_verbose=False):
-    # start off with where we are
-    path = os.getcwd()
-    print
-    print 'Testing path: %s' % path
-
-    # get the terminal width
-    term_width = terminal_width()
-
-    # explain what we are doing
-    print
-    print "The following directories will be skipped over:"
-    exclude_dirs.sort()
-    list_names(exclude_dirs, page_width=term_width)
-
-    # get all test_*.py and enclosing directories
-    test_files, path_files = get_test_files(path)
-    path_files.sort()
-
-    files = [x for x in test_files if not x == 'test_all.py']
-    files.sort()        # Ensure same order on all platforms
-
-    print
-    print 'Paths searched:'
-    list_names(path_files, os.path.basename, page_width=term_width)
-
-    print    
-    print 'Files tested:'
-    list_names(files, page_width=term_width)
-    print
-
-    # update system path with found paths
-    for path in path_files:
-        sys.path.append(path)
-   
-    # exclude files that we can't handle 
-    for file in exclude_files:
-        print 'WARNING: File '+ file + ' to be excluded from testing'
-        try:
-            files.remove(file)
-        except ValueError, e:
-            msg = 'File "%s" was not found in test suite.\n' % file
-            msg += 'Original error is "%s"\n' % e
-            msg += 'Perhaps it should be removed from exclude list?'
-            raise Exception, msg
-
-    # import all test_*.py files
-    # NOTE: This implies that test_*.py files MUST HAVE UNIQUE NAMES!
-    filenameToModuleName = lambda f: os.path.splitext(f)[0]
-    moduleNames = map(filenameToModuleName, files)
-    modules = map(__import__, moduleNames)
-
-    # Fix up the system path
-    for file in path_files:
-        sys.path.remove(file)
-
-    # bundle up all the tests
-    load = unittest.defaultTestLoader.loadTestsFromModule
-    testCaseClasses = map(load, modules)
-
-    if test_verbose is True:
-        # Test the code by setting verbose to True.
-        # The test cases have to be set up for this to work.
-        # See test data manager for an example.
-        for test_suite in testCaseClasses:
-            for tests in test_suite._tests:
-                # tests is of class TestSuite
-                if len(tests._tests) > 1:
-                    # these are the test functions
-                    try:
-                        # Calls class method set_verbose in test case classes
-                        tests._tests[0].set_verbose()
-                    except:
-                        pass                # No all classes have set_verbose
-
-    return unittest.TestSuite(testCaseClasses)
-
-
-##
-# @brief Check that the environment is sane.
-# @note Stops here if there is an error.
-def check_anuga_import():
-    try:
-        # importing something that loads quickly
-        import anuga.anuga_exceptions
-    except ImportError:
-        print "Python cannot import ANUGA module."
-        print "Check you have followed all steps of its installation."
-        import sys
-        sys.exit()
-
-
-if __name__ == '__main__':
-    check_anuga_import()
-
-    if len(sys.argv) > 1 and sys.argv[1][0].upper() == 'V':
-        test_verbose = True
-        saveout = sys.stdout
-        filename = ".temp"
-        fid = open(filename, 'w')
-        sys.stdout = fid
-    else:
-        test_verbose = False
-    suite = regressionTest(test_verbose)
-    runner = unittest.TextTestRunner(verbosity=1)
-    runner.run(suite)
-
-    # timestamp at the end
-    timestamp = time.asctime()
-    version = aust.get_revision_number()
-    print
-    print 'Finished at %s, version %s' % (timestamp, version)
-
-    # Cleaning up
-    if len(sys.argv) > 1 and sys.argv[1][0].upper() == 'V':
-        sys.stdout = saveout
-        #fid.close() # This was causing an error in windows
-        #os.remove(filename)
-
-    
-    if sys.platform == 'win32':
-        raw_input('Press the RETURN key')
Index: unk/anuga_core/source/anuga_parallel/test_distribute_mesh.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_distribute_mesh.py	(revision 9423)
+++ 	(revision )
@@ -1,1518 +1,0 @@
-#!/usr/bin/env python
-
-import unittest
-import sys
-from math import sqrt
-
-
-from anuga import Domain
-from anuga import rectangular_cross
-
-from anuga_parallel.distribute_mesh import pmesh_divide_metis
-from anuga_parallel.distribute_mesh import build_submesh
-from anuga_parallel.distribute_mesh import submesh_full, submesh_ghost, submesh_quantities
-from anuga_parallel.distribute_mesh import extract_submesh, rec_submesh, send_submesh
-
-import numpy as num
-
-
-def topography(x,y): 
-    return -x/2
-
-
-def xcoord(x,y):
-    return x
-
-def ycoord(x,y):
-    return y
-
-
-
-class Test_Distribute_Mesh(unittest.TestCase):
-    def setUp(self):
-        pass
-
-
-    def tearDown(self):
-        pass
-
-
-
-    def test_pmesh_1(self):
-        """
-        test distributing with just one processor
-        """
-        
-        points, vertices, boundary = rectangular_cross(2,2)
-
-
-        true_points = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], \
-        [0.5, 0.5], [0.5, 1.0], [1.0, 0.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], \
-        [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
-
-        true_vertices = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [1, 10, 2], \
-        [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7],
-        [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]]
-
-
-        assert num.allclose(points,true_points)
-        assert num.allclose(vertices,true_vertices)
-
-        domain = Domain(points, vertices, boundary)
-
-
-        domain.set_quantity('elevation', topography) # Use function for elevation
-        domain.set_quantity('friction', 0.0)         # Constant friction 
-        domain.set_quantity('stage', expression='elevation') # Dry initial stage
-        domain.set_quantity('xmomentum', expression='friction + 2.0') # 
-        domain.set_quantity('ymomentum', ycoord) #
-
-
-        #print domain.quantities['ymomentum'].centroid_values
- 
-        nodes, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(domain,1)
-
-
-        true_nodes = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], \
-        [1.0, 0.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
-
-        true_triangles = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [1, 10, 2], [4, 10, 1], \
-        [5, 10, 4], [2, 10, 5], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7], [4, 12, 5], \
-        [7, 12, 4], [8, 12, 7], [5, 12, 8]]
-
-
-        assert num.allclose(nodes,true_nodes)
-        assert num.allclose(triangles,true_triangles)
-
-
-        assert num.allclose(triangles_per_proc,[16])
-        
-        
-
-    def test_pmesh_2(self):
-        """
-        Test 2 way pmesh
-        """
-        points, vertices, boundary = rectangular_cross(2,2)
-
-
-        true_points = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], \
-        [1.0, 0.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
-
-        true_vertices = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [1, 10, 2], [4, 10, 1], \
-        [5, 10, 4], [2, 10, 5], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7], [4, 12, 5], \
-        [7, 12, 4], [8, 12, 7], [5, 12, 8]]
-
-
-        assert num.allclose(points,true_points)
-        assert num.allclose(vertices,true_vertices)
-
-        domain = Domain(points, vertices, boundary)
-
-
-        domain.set_quantity('elevation', topography) # Use function for elevation
-        domain.set_quantity('friction', 0.0)         # Constant friction 
-        domain.set_quantity('stage', expression='elevation') # Dry initial stage
-        domain.set_quantity('xmomentum', expression='friction + 2.0') # 
-        domain.set_quantity('ymomentum', ycoord) #
-
-
-        #print domain.quantities['ymomentum'].centroid_values
- 
-        nodes, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(domain,2)
-
-
-        true_nodes = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], \
-        [1.0, 0.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
-
-
-        true_triangles = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [4, 10, 1], [3, 11, 4], \
-        [4, 11, 7], [4, 12, 5], [1, 10, 2], [5, 10, 4], [2, 10, 5], [6, 11, 3], [7, 11, 6], \
-        [7, 12, 4], [8, 12, 7], [5, 12, 8]]
-
-
-        assert num.allclose(nodes,true_nodes)
-        assert num.allclose(triangles,true_triangles)
-
-
-        assert num.allclose(triangles_per_proc,[8,8])
-
-
-            
-    def test_build_submesh_3(self):
-        """
-        Test 3 way build_submesh
-        """
-
-        nodes = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], [1.0, 0.0], \
-        [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
-
-
-        triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], \
-        [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], \
-        [6, 11, 3], [7, 11, 6], [4, 11, 7]]
-
-
-        boundary = {(13, 1): 'bottom', (7, 1): 'left', (3, 1): 'right', (14, 1): 'right', \
-        (11, 1): 'bottom', (10, 1): 'top', (5, 1): 'left', (4, 1): 'top'}
-
-        triangles_per_proc = [5, 6, 5]
-
-
-        quantities = {'stage': num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]),  'elevation': num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]),  'ymomentum': num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.5 ]]),  'friction': num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]),  'xmomentum': num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]])}
-
-
-        
-        true_submesh = {'full_boundary': [{(3, 1): 'right', (4, 1): 'top'},\
-        {(5, 1): 'left', (10, 1): 'top', (7, 1): 'left'}, \
-        {(13, 1): 'bottom', (14, 1): 'right', (11, 1): 'bottom'}],
-                        'ghost_nodes': [num.array([[  0.  ,   0.  ,   0.  ],
-       [  1.  ,   0.  ,   0.5 ],
-       [  2.  ,   0.  ,   1.  ],
-       [  6.  ,   1.  ,   0.  ],
-       [ 10.  ,   0.25,   0.75],
-       [ 11.  ,   0.75,   0.25]]), num.array([[  3.  ,   0.5 ,   0.  ],
-       [  7.  ,   1.  ,   0.5 ],
-       [  8.  ,   1.  ,   1.  ],
-       [ 11.  ,   0.75,   0.25],
-       [ 12.  ,   0.75,   0.75]]), num.array([[  1.  ,   0.  ,   0.5 ],
-       [  5.  ,   0.5 ,   1.  ],
-       [  8.  ,   1.  ,   1.  ],
-       [ 12.  ,   0.75,   0.75]])],
-                        'full_nodes': [num.array([[  3.  ,   0.5 ,   0.  ],
-       [  4.  ,   0.5 ,   0.5 ],
-       [  5.  ,   0.5 ,   1.  ],
-       [  7.  ,   1.  ,   0.5 ],
-       [  8.  ,   1.  ,   1.  ],
-       [  9.  ,   0.25,   0.25],
-       [ 12.  ,   0.75,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
-       [  1.  ,   0.  ,   0.5 ],
-       [  2.  ,   0.  ,   1.  ],
-       [  4.  ,   0.5 ,   0.5 ],
-       [  5.  ,   0.5 ,   1.  ],
-       [  9.  ,   0.25,   0.25],
-       [ 10.  ,   0.25,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
-       [  3.  ,   0.5 ,   0.  ],
-       [  4.  ,   0.5 ,   0.5 ],
-       [  6.  ,   1.  ,   0.  ],
-       [  7.  ,   1.  ,   0.5 ],
-       [  9.  ,   0.25,   0.25],
-       [ 11.  ,   0.75,   0.25]])],
-                        'ghost_triangles': [num.array([[ 5,  0,  9,  1],
-       [ 6,  1,  9,  4],
-       [ 8,  4, 10,  1],
-       [ 9,  5, 10,  4],
-       [10,  2, 10,  5],
-       [11,  3,  9,  0],
-       [12,  3, 11,  4],
-       [13,  6, 11,  3],
-       [14,  7, 11,  6],
-       [15,  4, 11,  7]]), num.array([[ 0,  4,  9,  3],
-       [ 1,  4, 12,  5],
-       [ 2,  7, 12,  4],
-       [ 4,  5, 12,  8],
-       [11,  3,  9,  0],
-       [12,  3, 11,  4]]), num.array([[ 0,  4,  9,  3],
-       [ 1,  4, 12,  5],
-       [ 2,  7, 12,  4],
-       [ 3,  8, 12,  7],
-       [ 5,  0,  9,  1],
-       [ 6,  1,  9,  4]])],
-                        'ghost_boundary': [{(13, 1): 'ghost', (8, 0): 'ghost', (14, 1): 'ghost', \
-                        (11, 1): 'ghost', (10, 1): 'ghost', (5, 1): 'ghost', (10, 2): 'ghost'}, \
-                        {(12, 2): 'ghost', (12, 0): 'ghost', (2, 1): 'ghost', (11, 1): 'ghost',\
-                        (2, 2): 'ghost', (4, 1): 'ghost', (4, 0): 'ghost'}, {(3, 2): 'ghost', \
-                        (6, 1): 'ghost', (3, 1): 'ghost', (5, 1): 'ghost', (1, 0): 'ghost', (1, 1): 'ghost'}],
-                        'full_triangles': [[[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]], \
-                        [[0, 9, 1], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5]], \
-                        [[3, 9, 0], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7]]],
-                        'full_commun': [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, \
-                        {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, \
-                        {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}],
-                        'ghost_commun': [num.array([[ 5,  1],
-       [ 6,  1],
-       [ 8,  1],
-       [ 9,  1],
-       [10,  1],
-       [11,  2],
-       [12,  2],
-       [13,  2],
-       [14,  2],
-       [15,  2]]), num.array([[ 0,  0],
-       [ 1,  0],
-       [ 2,  0],
-       [ 4,  0],
-       [11,  2],
-       [12,  2]]), num.array([[0, 0],
-       [1, 0],
-       [2, 0],
-       [3, 0],
-       [5, 1],
-       [6, 1]])],  'ghost_quan': {'stage': [num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ]])],  'elevation': [num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ]])],  'ymomentum': [num.array([[ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]])], 'xmomentum': [num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]])]},  'full_quan': {'stage': [num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]])],  'elevation': [num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]])],  'ymomentum': [num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]])],  'xmomentum': [num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]])]}}
-                                  
-
-        from anuga.abstract_2d_finite_volumes.neighbour_mesh import Mesh
-        
-        mesh = Mesh(nodes, triangles, boundary)
-        boundary_polygon = mesh.get_boundary_polygon()
-
-
-        # Subdivide into non-overlapping partitions
-
-        submesh = submesh_full(mesh, triangles_per_proc)
-
-        #print submesh
-
-
-        for i in range(3):
-            assert num.allclose(true_submesh['full_triangles'][i],submesh['full_triangles'][i])
-            assert num.allclose(true_submesh['full_nodes'][i],submesh['full_nodes'][i])
-        assert true_submesh['full_boundary'] == submesh['full_boundary']
-
-        # Add any extra ghost boundary layer information
-
-        submesh = submesh_ghost(submesh, mesh, triangles_per_proc)
-
-        for i in range(3):
-            assert num.allclose(true_submesh['ghost_triangles'][i],submesh['ghost_triangles'][i])
-            assert num.allclose(true_submesh['ghost_nodes'][i],submesh['ghost_nodes'][i])
-            assert num.allclose(true_submesh['ghost_commun'][i],submesh['ghost_commun'][i])
-
-        assert true_submesh['full_commun'] == submesh['full_commun']
-
-
-        # Order the quantities information to be the same as the triangle
-        # information
-
-
-        submesh = submesh_quantities(submesh, quantities, \
-                                 triangles_per_proc)
-
-
-
-        for key, value in true_submesh['ghost_quan'].iteritems():
-            for i in range(3):
-                assert num.allclose(true_submesh['ghost_quan'][key][i],submesh['ghost_quan'][key][i])
-                assert num.allclose(true_submesh['full_quan'][key][i],submesh['full_quan'][key][i])
-        
-
-        submesh["boundary_polygon"] = boundary_polygon
-
-
-
-    def test_build_submesh_3_layer_4(self):
-        """
-        Test 3 Layer 4 way build_submesh with ghost_layer_width = 4
-        """
-
-        nodes = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], \
-        [1.0, 0.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
-
-
-        triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], \
-        [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], \
-        [6, 11, 3], [7, 11, 6], [4, 11, 7]]
-
-
-        boundary = {(13, 1): 'bottom', (7, 1): 'left', (3, 1): 'right', (14, 1): 'right', \
-        (11, 1): 'bottom', (10, 1): 'top', (5, 1): 'left', (4, 1): 'top'}
-
-        """
-                     top
-            (10,1)       (4,1)
-        2 ---------- 5 ---------- 8
-        | \   10   / | \   04   / |
-        |   \    /   |   \    /   |
- (7,1)  | 07  10  09 | 01  12  03 | (3,1)
-        |   /    \   |   /    \   |
-        | /   08   \ | /   02   \ |
- left   1 ---------- 4 ---------- 7  right
-        | \   06   / | \   15   / |
-        |   \    /   |   \    /   |
- (5,1)  | 05  09  00 | 12  11  14 | (14,1)
-        |   /    \   |   /    \   |
-        | /   11   \ | /   13   \ |
-        0 ---------- 3 ---------- 6
-            (11,1)        (13,1)
-                   bottom
-
-
-        Processor 0 Full Triangles 00,01,02,03,04
-        Processor 1 Full Triangles 05,06,07,08,09,10
-        Processor 2 Full Triangles 11,12,13,14,15
-        """
-
-
-        triangles_per_proc = [5, 6, 5]
-
-
-        quantities = {'stage': num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]),  'elevation': num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]),  'ymomentum': num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.5 ]]),  'friction': num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]),  'xmomentum': num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]])}
-
-
-
-        true_submesh = {'full_boundary': [{(3, 1): 'right', (4, 1): 'top'}, {(5, 1): 'left', \
-        (10, 1): 'top', (7, 1): 'left'}, {(13, 1): 'bottom', (14, 1): 'right', (11, 1): 'bottom'}],
-                        'ghost_nodes': [num.array([[  0.  ,   0.  ,   0.  ],
-       [  1.  ,   0.  ,   0.5 ],
-       [  2.  ,   0.  ,   1.  ],
-       [  6.  ,   1.  ,   0.  ],
-       [ 10.  ,   0.25,   0.75],
-       [ 11.  ,   0.75,   0.25]]), num.array([[  3.  ,   0.5 ,   0.  ],
-       [  7.  ,   1.  ,   0.5 ],
-       [  8.  ,   1.  ,   1.  ],
-       [ 11.  ,   0.75,   0.25],
-       [ 12.  ,   0.75,   0.75]]), num.array([[  1.  ,   0.  ,   0.5 ],
-       [  5.  ,   0.5 ,   1.  ],
-       [  8.  ,   1.  ,   1.  ],
-       [ 12.  ,   0.75,   0.75]])],
-                        'full_nodes': [num.array([[  3.  ,   0.5 ,   0.  ],
-       [  4.  ,   0.5 ,   0.5 ],
-       [  5.  ,   0.5 ,   1.  ],
-       [  7.  ,   1.  ,   0.5 ],
-       [  8.  ,   1.  ,   1.  ],
-       [  9.  ,   0.25,   0.25],
-       [ 12.  ,   0.75,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
-       [  1.  ,   0.  ,   0.5 ],
-       [  2.  ,   0.  ,   1.  ],
-       [  4.  ,   0.5 ,   0.5 ],
-       [  5.  ,   0.5 ,   1.  ],
-       [  9.  ,   0.25,   0.25],
-       [ 10.  ,   0.25,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
-       [  3.  ,   0.5 ,   0.  ],
-       [  4.  ,   0.5 ,   0.5 ],
-       [  6.  ,   1.  ,   0.  ],
-       [  7.  ,   1.  ,   0.5 ],
-       [  9.  ,   0.25,   0.25],
-       [ 11.  ,   0.75,   0.25]])],
-                        'ghost_triangles': [num.array([[ 5,  0,  9,  1],
-       [ 6,  1,  9,  4],
-       [ 8,  4, 10,  1],
-       [ 9,  5, 10,  4],
-       [10,  2, 10,  5],
-       [11,  3,  9,  0],
-       [12,  3, 11,  4],
-       [13,  6, 11,  3],
-       [14,  7, 11,  6],
-       [15,  4, 11,  7]]), num.array([[ 0,  4,  9,  3],
-       [ 1,  4, 12,  5],
-       [ 2,  7, 12,  4],
-       [ 4,  5, 12,  8],
-       [11,  3,  9,  0],
-       [12,  3, 11,  4]]), num.array([[ 0,  4,  9,  3],
-       [ 1,  4, 12,  5],
-       [ 2,  7, 12,  4],
-       [ 3,  8, 12,  7],
-       [ 5,  0,  9,  1],
-       [ 6,  1,  9,  4]])],
-                        'ghost_boundary': [{(13, 1): 'ghost', (8, 0): 'ghost', \
-                        (14, 1): 'ghost', (11, 1): 'ghost', (10, 1): 'ghost', \
-                        (5, 1): 'ghost', (10, 2): 'ghost'}, {(12, 2): 'ghost', \
-                        (12, 0): 'ghost', (2, 1): 'ghost', (11, 1): 'ghost', \
-                        (2, 2): 'ghost', (4, 1): 'ghost', (4, 0): 'ghost'}, \
-                        {(3, 2): 'ghost', (6, 1): 'ghost', (3, 1): 'ghost', \
-                        (5, 1): 'ghost', (1, 0): 'ghost', (1, 1): 'ghost'}],
-                        'full_triangles': [[[4, 9, 3], [4, 12, 5], [7, 12, 4], \
-                        [8, 12, 7], [5, 12, 8]], [[0, 9, 1], [1, 9, 4], [1, 10, 2], \
-                        [4, 10, 1], [5, 10, 4], [2, 10, 5]], [[3, 9, 0], [3, 11, 4], \
-                        [6, 11, 3], [7, 11, 6], [4, 11, 7]]],
-                        'full_commun': [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, \
-                        {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, \
-                        {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}],
-                        'ghost_commun': [num.array([[ 5,  1],
-       [ 6,  1],
-       [ 8,  1],
-       [ 9,  1],
-       [10,  1],
-       [11,  2],
-       [12,  2],
-       [13,  2],
-       [14,  2],
-       [15,  2]]), num.array([[ 0,  0],
-       [ 1,  0],
-       [ 2,  0],
-       [ 4,  0],
-       [11,  2],
-       [12,  2]]), num.array([[0, 0],
-       [1, 0],
-       [2, 0],
-       [3, 0],
-       [5, 1],
-       [6, 1]])],  'ghost_quan': {'stage': [num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ]])],  'elevation': [num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ]])],  'ymomentum': [num.array([[ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]])], 'xmomentum': [num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]])]},  'full_quan': {'stage': [num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]])],  'elevation': [num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]])],  'ymomentum': [num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]])],  'xmomentum': [num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]])]}}
-
-
-        # setup parameters to test using different ghost_layer_widths
-        parameters = dict(ghost_layer_width = 1)
-
-        from anuga.abstract_2d_finite_volumes.neighbour_mesh import Mesh
-
-        mesh = Mesh(nodes, triangles, boundary)
-        boundary_polygon = mesh.get_boundary_polygon()
-
-
-        # Subdivide into non-overlapping partitions
-
-        submesh = submesh_full(mesh, triangles_per_proc)
-
-
-
-        for i in range(3):
-            assert num.allclose(true_submesh['full_triangles'][i],submesh['full_triangles'][i])
-            assert num.allclose(true_submesh['full_nodes'][i],submesh['full_nodes'][i])
-        assert true_submesh['full_boundary'] == submesh['full_boundary']
-
-        # Add any extra ghost boundary layer information
-
-        submesh = submesh_ghost(submesh, mesh, triangles_per_proc, parameters)
-
-        #print submesh
-
-        #print 'ghost_triangles', submesh['ghost_triangles']
-        #print 'ghost_boundary', submesh['ghost_boundary']
-        #print 'ghost_nodes', submesh['ghost_nodes']
-        #print 'ghost_commun', submesh['ghost_commun']
-
-
-        true_submesh['ghost_boundary'] = [{(12, 2): 'ghost', (9, 0): 'ghost', (9, 2): 'ghost', \
-        (6, 1): 'ghost', (15, 0): 'ghost', (11, 1): 'bottom', (11, 0): 'ghost', (6, 2): 'ghost'}, \
-        {(0, 1): 'ghost', (1, 2): 'ghost', (1, 0): 'ghost', (11, 1): 'bottom'},\
-        {(5, 1): 'left', (2, 0): 'ghost', (0, 2): 'ghost', (5, 0): 'ghost', (2, 2): 'ghost'}]
-
-
-        true_submesh['ghost_triangles'] = [num.array([[ 6,  1,  9,  4],\
-                                                      [ 9,  5, 10,  4],\
-                                                      [11,  3,  9,  0],\
-                                                      [12,  3, 11,  4],\
-                                                      [15,  4, 11,  7]]), \
-                                           num.array([[ 0,  4,  9,  3],\
-                                                      [ 1,  4, 12,  5],\
-                                                     [11,  3,  9,  0]]),\
-                                           num.array([[ 0,  4,  9,  3],\
-                                                      [ 2,  7, 12,  4],\
-                                                      [ 5,  0,  9,  1]])]
-
-        true_submesh['ghost_nodes'] =  [num.array([[  0.  ,   0.  ,   0.  ],\
-                                                   [  1.  ,   0.  ,   0.5 ],\
-                                                   [ 10.  ,   0.25,   0.75],\
-                                                   [ 11.  ,   0.75,   0.25]]),\
-                                        num.array([[  3.  ,   0.5 ,   0.  ],\
-                                                   [ 12.  ,   0.75,   0.75]]),\
-                                        num.array([[  1.  ,   0.  ,   0.5 ],\
-                                                   [ 12.  ,   0.75,   0.75]])]
-
-        true_submesh['ghost_commun'] = [num.array([[ 6,  1],\
-                                                   [ 9,  1],\
-                                                   [11,  2],\
-                                                   [12,  2],\
-                                                   [15,  2]]),\
-                                        num.array([[ 0,  0],\
-                                                   [ 1,  0],\
-                                                   [11,  2]]),\
-                                        num.array([[0, 0],\
-                                                   [2, 0],\
-                                                   [5, 1]])]
-
-
-
-
-        for i in range(3):
-            assert true_submesh['ghost_boundary'][i] == submesh['ghost_boundary'][i]
-            assert num.allclose(true_submesh['ghost_triangles'][i],submesh['ghost_triangles'][i])
-            assert num.allclose(true_submesh['ghost_nodes'][i],submesh['ghost_nodes'][i])
-            assert num.allclose(true_submesh['ghost_commun'][i],submesh['ghost_commun'][i])
-
-
-
-        true_submesh['full_commun'] = [{0: [1, 2], 1: [1], 2: [2], 3: [], 4: []},\
-                                       {5: [2], 6: [0], 7: [], 8: [], 9: [0], 10: []},\
-                                       {11: [0, 1], 12: [0], 13: [], 14: [], 15: [0]}]
-
-        assert true_submesh['full_commun'] == submesh['full_commun']
-
-
-        # Order the quantities information to be the same as the triangle
-        # information
-
-
-        submesh = submesh_quantities(submesh, quantities, \
-                                 triangles_per_proc)
-
-
-        #print submesh['full_quan']
-
-
-
-        true_submesh['full_quan'] = {'stage': [num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]])], 'elevation': [num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]])], 'ymomentum': [num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.5 ]])], 'friction': [num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]])], 'xmomentum': [num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]])]}
-
-
-        true_submesh['ghost_quan'] = {'stage': [num.array([[-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.25 , -0.125, -0.   ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.   , -0.125, -0.   ]])], 'elevation': [num.array([[-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.25 , -0.125, -0.   ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.   , -0.125, -0.   ]])], 'ymomentum': [num.array([[ 0.5 ,  0.25,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.  ]]), num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 0.  ,  0.25,  0.5 ]])], 'friction': [num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]])], 'xmomentum': [num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]])]}
-
-
-        for key, value in true_submesh['ghost_quan'].iteritems():
-            for i in range(3):
-                assert num.allclose(true_submesh['ghost_quan'][key][i],submesh['ghost_quan'][key][i])
-                assert num.allclose(true_submesh['full_quan'][key][i],submesh['full_quan'][key][i])
-
-
-        submesh["boundary_polygon"] = boundary_polygon
-
-
-    def test_build_extract_submesh_3(self):
-        """
-        Test 3 way extract_submesh
-        """
-
-        nodes = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], [1.0, 0.0], \
-        [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
-
-
-        triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], \
-        [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], \
-        [6, 11, 3], [7, 11, 6], [4, 11, 7]]
-
-
-        boundary = {(13, 1): 'bottom', (7, 1): 'left', (3, 1): 'right', (14, 1): 'right', \
-        (11, 1): 'bottom', (10, 1): 'top', (5, 1): 'left', (4, 1): 'top'}
-
-        triangles_per_proc = [5, 6, 5]
-
-
-        quantities = {'stage': num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]),  'elevation': num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]),  'ymomentum': num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.5 ]]),  'friction': num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]),  'xmomentum': num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]])}
-
-
-
-        true_submesh = {'full_boundary': [{(3, 1): 'right', (4, 1): 'top'},\
-        {(5, 1): 'left', (10, 1): 'top', (7, 1): 'left'}, \
-        {(13, 1): 'bottom', (14, 1): 'right', (11, 1): 'bottom'}],
-                        'ghost_nodes': [num.array([[  0.  ,   0.  ,   0.  ],
-       [  1.  ,   0.  ,   0.5 ],
-       [  2.  ,   0.  ,   1.  ],
-       [  6.  ,   1.  ,   0.  ],
-       [ 10.  ,   0.25,   0.75],
-       [ 11.  ,   0.75,   0.25]]), num.array([[  3.  ,   0.5 ,   0.  ],
-       [  7.  ,   1.  ,   0.5 ],
-       [  8.  ,   1.  ,   1.  ],
-       [ 11.  ,   0.75,   0.25],
-       [ 12.  ,   0.75,   0.75]]), num.array([[  1.  ,   0.  ,   0.5 ],
-       [  5.  ,   0.5 ,   1.  ],
-       [  8.  ,   1.  ,   1.  ],
-       [ 12.  ,   0.75,   0.75]])],
-                        'full_nodes': [num.array([[  3.  ,   0.5 ,   0.  ],
-       [  4.  ,   0.5 ,   0.5 ],
-       [  5.  ,   0.5 ,   1.  ],
-       [  7.  ,   1.  ,   0.5 ],
-       [  8.  ,   1.  ,   1.  ],
-       [  9.  ,   0.25,   0.25],
-       [ 12.  ,   0.75,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
-       [  1.  ,   0.  ,   0.5 ],
-       [  2.  ,   0.  ,   1.  ],
-       [  4.  ,   0.5 ,   0.5 ],
-       [  5.  ,   0.5 ,   1.  ],
-       [  9.  ,   0.25,   0.25],
-       [ 10.  ,   0.25,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
-       [  3.  ,   0.5 ,   0.  ],
-       [  4.  ,   0.5 ,   0.5 ],
-       [  6.  ,   1.  ,   0.  ],
-       [  7.  ,   1.  ,   0.5 ],
-       [  9.  ,   0.25,   0.25],
-       [ 11.  ,   0.75,   0.25]])],
-                        'ghost_triangles': [num.array([[ 5,  0,  9,  1],
-       [ 6,  1,  9,  4],
-       [ 8,  4, 10,  1],
-       [ 9,  5, 10,  4],
-       [10,  2, 10,  5],
-       [11,  3,  9,  0],
-       [12,  3, 11,  4],
-       [13,  6, 11,  3],
-       [14,  7, 11,  6],
-       [15,  4, 11,  7]]), num.array([[ 0,  4,  9,  3],
-       [ 1,  4, 12,  5],
-       [ 2,  7, 12,  4],
-       [ 4,  5, 12,  8],
-       [11,  3,  9,  0],
-       [12,  3, 11,  4]]), num.array([[ 0,  4,  9,  3],
-       [ 1,  4, 12,  5],
-       [ 2,  7, 12,  4],
-       [ 3,  8, 12,  7],
-       [ 5,  0,  9,  1],
-       [ 6,  1,  9,  4]])],
-                        'ghost_boundary': [{(13, 1): 'ghost', (8, 0): 'ghost', (14, 1): 'ghost', \
-                        (11, 1): 'ghost', (10, 1): 'ghost', (5, 1): 'ghost', (10, 2): 'ghost'}, \
-                        {(12, 2): 'ghost', (12, 0): 'ghost', (2, 1): 'ghost', (11, 1): 'ghost',\
-                        (2, 2): 'ghost', (4, 1): 'ghost', (4, 0): 'ghost'}, {(3, 2): 'ghost', \
-                        (6, 1): 'ghost', (3, 1): 'ghost', (5, 1): 'ghost', (1, 0): 'ghost', (1, 1): 'ghost'}],
-                        'full_triangles': [[[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]], \
-                        [[0, 9, 1], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5]], \
-                        [[3, 9, 0], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7]]],
-                        'full_commun': [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, \
-                        {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, \
-                        {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}],
-                        'ghost_commun': [num.array([[ 5,  1],
-       [ 6,  1],
-       [ 8,  1],
-       [ 9,  1],
-       [10,  1],
-       [11,  2],
-       [12,  2],
-       [13,  2],
-       [14,  2],
-       [15,  2]]), num.array([[ 0,  0],
-       [ 1,  0],
-       [ 2,  0],
-       [ 4,  0],
-       [11,  2],
-       [12,  2]]), num.array([[0, 0],
-       [1, 0],
-       [2, 0],
-       [3, 0],
-       [5, 1],
-       [6, 1]])],  'ghost_quan': {'stage': [num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ]])],  'elevation': [num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.25 , -0.375, -0.5  ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ]])],  'ymomentum': [num.array([[ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]])], 'xmomentum': [num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]])]},  'full_quan': {'stage': [num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]])],  'elevation': [num.array([[-0.25 , -0.125, -0.25 ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
-       [-0.   , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.125, -0.25 ],
-       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
-       [-0.25 , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.25 ],
-       [-0.5  , -0.375, -0.5  ],
-       [-0.25 , -0.375, -0.5  ]])],  'ymomentum': [num.array([[ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.5 ],
-       [ 0.5 ,  0.25,  0.5 ],
-       [ 0.5 ,  0.75,  1.  ],
-       [ 0.5 ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  0.5 ],
-       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.  ],
-       [ 0.  ,  0.25,  0.5 ],
-       [ 0.  ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.  ],
-       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.],
-       [ 0.,  0.,  0.]])],  'xmomentum': [num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.],
-       [ 2.,  2.,  2.]])]}}
-
-
-        # Subdivide into non-overlapping partitions
-        submesh = build_submesh(nodes, triangles, boundary, quantities,
-                  triangles_per_proc, parameters = None)
-
-
-        for i in range(3):
-            assert num.allclose(true_submesh['full_triangles'][i],submesh['full_triangles'][i])
-            assert num.allclose(true_submesh['full_nodes'][i],submesh['full_nodes'][i])
-            assert num.allclose(true_submesh['ghost_triangles'][i],submesh['ghost_triangles'][i])
-            assert num.allclose(true_submesh['ghost_nodes'][i],submesh['ghost_nodes'][i])
-            assert num.allclose(true_submesh['ghost_commun'][i],submesh['ghost_commun'][i])
-
-        assert true_submesh['full_boundary'] == submesh['full_boundary']
-        assert true_submesh['full_commun'] == submesh['full_commun']
-
-        for key, value in true_submesh['ghost_quan'].iteritems():
-            for i in range(3):
-                assert num.allclose(true_submesh['ghost_quan'][key][i],submesh['ghost_quan'][key][i])
-                assert num.allclose(true_submesh['full_quan'][key][i],submesh['full_quan'][key][i])
-
-
-        # Now test the extract_submesh for the 3 processors
-
-        submesh_cell_0 = extract_submesh(submesh,triangles_per_proc,p=0)
-        submesh_cell_1 = extract_submesh(submesh,triangles_per_proc,p=1)
-        submesh_cell_2 = extract_submesh(submesh,triangles_per_proc,p=2)
-
-
-        from pprint import pprint
-
-        #pprint(submesh_cell_1)
-
-#-------------------------------------------------------------
-
-if __name__ == "__main__":
-    suite = unittest.makeSuite(Test_Distribute_Mesh,'test')
-    runner = unittest.TextTestRunner()
-    runner.run(suite)
Index: unk/anuga_core/source/anuga_parallel/test_failure.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_failure.py	(revision 9423)
+++ 	(revision )
@@ -1,324 +1,0 @@
-import os.path
-import sys
-
-from anuga.utilities.system_tools import get_pathname_from_package
-from anuga.geometry.polygon_function import Polygon_function
-        
-from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
-from anuga.abstract_2d_finite_volumes.quantity import Quantity
-from anuga.abstract_2d_finite_volumes.util import file_function
-
-import anuga
-
-from anuga.structures.boyd_box_operator import Boyd_box_operator
-from anuga.structures.inlet_operator import Inlet_operator
-                            
-#from anuga.culvert_flows.culvert_routines import boyd_generalised_culvert_model
-     
-from math import pi, pow, sqrt
-
-import numpy as num
-#from parallel_inlet_operator import Parallel_Inlet_operator
-from anuga_parallel import distribute, myid, numprocs, finalize
-from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
-
-#from parallel_operator_factory import Inlet_operator, Boyd_box_operator
-import pypar
-import random
-
-
-"""test_that_culvert_runs_rating
-
-This test exercises the culvert and checks values outside rating curve
-are dealt with       
-"""
-verbose = True
-path = get_pathname_from_package('anuga.culvert_flows')    
-
-length = 40.
-width = 15.
-
-dx = dy = 0.5          # Resolution: Length of subdivisions on both axes
-
-#----------------------------------------------------------------------
-# Setup initial conditions
-#----------------------------------------------------------------------
-
-def topography(x, y):
-    """Set up a weir
-    
-    A culvert will connect either side
-    """
-    # General Slope of Topography
-    z=-x/1000
-    
-    N = len(x)
-    for i in range(N):
-
-       # Sloping Embankment Across Channel
-        if 5.0 < x[i] < 10.1:
-            # Cut Out Segment for Culvert face                
-            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
-               z[i]=z[i]
-            else:
-               z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
-        if 10.0 < x[i] < 12.1:
-           z[i] +=  2.5                    # Flat Crest of Embankment
-        if 12.0 < x[i] < 14.5:
-            # Cut Out Segment for Culvert face                
-            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
-               z[i]=z[i]
-            else:
-               z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
-                   
-        
-    return z
-
-filename=os.path.join(path, 'example_rating_curve.csv')
-
-line0 = [[10.0, 10.0], [30.0, 10.0]]
-#line0 = [[29.0, 10.0], [30.0, 10.0]]
-line1 = [[0.0, 10.0], [0.0, 15.0]]
-Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
-Q1 = 5.0
-
-samples = 50
-
-def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
-    success = True
-
-##-----------------------------------------------------------------------
-## Setup domain
-##-----------------------------------------------------------------------
-
-    points, vertices, boundary = rectangular_cross(int(length/dx),
-                                                   int(width/dy),
-                                                   len1=length, 
-                                                   len2=width)
-
-    domain = anuga.Domain(points, vertices, boundary)   
-    domain.set_name('Test_Parallel_Frac_Op')                 # Output name
-    domain.set_default_order(2)
-
-##-----------------------------------------------------------------------
-## Distribute domain
-##-----------------------------------------------------------------------
-
-    if parallel: domain = distribute(domain)
-    
-
-##-----------------------------------------------------------------------
-## Setup boundary conditions
-##-----------------------------------------------------------------------
-    
-    domain.set_quantity('elevation', topography) 
-    domain.set_quantity('friction', 0.01)         # Constant friction 
-    domain.set_quantity('stage',
-                        expression='elevation')   # Dry initial condition
-
-    
-    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
-    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
-    domain.set_boundary({'left': Bi, 'right': Br, 'top': Br, 'bottom': Br})
-
-
-##-----------------------------------------------------------------------
-## Determine triangle index coinciding with test points
-##-----------------------------------------------------------------------
-
-    assert(test_points is not None)
-    assert(len(test_points) == samples)
-
-    tri_ids = []
-
-    for point in test_points:
-        try:
-            k = domain.get_triangle_containing_point(point)
-            if domain.tri_full_flag[k] == 1:
-                tri_ids.append(k)
-            else:
-                tri_ids.append(-1)            
-        except:
-            tri_ids.append(-2)
-
-    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
-
-    if not parallel: control_data = []
-
-    ################ Define Fractional Operators ##########################
-
-    inlet0 = None
-    inlet1 = None
-    boyd_box0 = None
-    
-    #inlet0 = Inlet_operator(domain, line0, Q0, debug = True)
-    #inlet1 = Inlet_operator(domain, line1, Q1, debug = True)
-    
-    ## # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
-    ## if myid == 5 and parallel:
-    ##     boyd_box0 = Boyd_box_operator(domain,
-    ##                                   end_points=[[9.0, 2.5],[13.0, 2.5]],
-    ##                                   losses=1.5,
-    ##                                   width=1.0,
-    ##                                   apron=0.5,
-    ##                                   use_momentum_jet=True,
-    ##                                   use_velocity_head=False,
-    ##                                   manning=0.013,
-    ##                                   verbose=False)
-    ## elif not parallel:
-    ##     boyd_box0 = Boyd_box_operator(domain,
-    ##                                   end_points=[[9.0, 2.5],[13.0, 2.5]],
-    ##                                   losses=1.5,
-    ##                                   width=1.0,
-    ##                                   apron=0.5,
-    ##                                   use_momentum_jet=True,
-    ##                                   use_velocity_head=False,
-    ##                                   manning=0.013,
-    ##                                   verbose=False)
-
-#    if parallel:
-#        factory = Parallel_operator_factory(domain, debug = True)
-#
-#        inlet0 = factory.inlet_operator_factory(line0, Q0)
-#        inlet1 = factory.inlet_operator_factory(line1, Q1)
-#        
-#        boyd_box0 = factory.boyd_box_operator_factory(end_points=[[9.0, 2.5],[19.0, 2.5]],
-#                                          losses=1.5,
-#                                          width=1.5,
-#                                          apron=5.0,
-#                                          use_momentum_jet=True,
-#                                          use_velocity_head=False,
-#                                          manning=0.013,
-#                                          verbose=False)
-#
-#    else:
-#        inlet0 = Inlet_operator(domain, line0, Q0)
-#        inlet1 = Inlet_operator(domain, line1, Q1)
-#
-#        # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
-#        boyd_box0 = Boyd_box_operator(domain,
-#                          end_points=[[9.0, 2.5],[19.0, 2.5]],
-#                          losses=1.5,
-#                          width=1.5,
-#                          apron=5.0,
-#                          use_momentum_jet=True,
-#                          use_velocity_head=False,
-#                          manning=0.013,
-#                          verbose=False)
-    
-    #######################################################################
-
-    ##-----------------------------------------------------------------------
-    ## Evolve system through time
-    ##-----------------------------------------------------------------------
-
-    for t in domain.evolve(yieldstep = 1.0, finaltime = 4):
-        domain.write_time()
-
-        #print domain.volumetric_balance_statistics()
-    
-        stage = domain.get_quantity('stage')
-
-        #for i in range(samples):
-        #    if tri_ids[i] >= 0:                
-        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
-                    
-        sys.stdout.flush()
- 
-        pass
-
- 
-
-    success = True
-
-##-----------------------------------------------------------------------
-## Assign/Test Control data
-##-----------------------------------------------------------------------
-
-    if not parallel:
-        stage = domain.get_quantity('stage')
-
-        for i in range(samples):
-            assert(tri_ids[i] >= 0)
-            control_data.append(stage.centroid_values[tri_ids[i]])
-
-        if inlet0 is not None:
-            control_data.append(inlet0.inlet.get_average_stage())
-            control_data.append(inlet0.inlet.get_average_xmom())
-            control_data.append(inlet0.inlet.get_average_ymom())
-            control_data.append(inlet0.inlet.get_total_water_volume())
-            control_data.append(inlet0.inlet.get_average_depth())
-
-        if verbose: print 'P%d control_data = %s' %(myid, control_data)
-    else:
-        stage = domain.get_quantity('stage')
-        
-        for i in range(samples):
-            if tri_ids[i] >= 0:
-                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
-                success = success and local_success
-                if verbose and not local_success: 
-                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success)
-                    sys.stdout.flush()
-                    
-                
-                
-        if inlet0 is not None:
-            inlet_master_proc = inlet0.inlet.get_master_proc()
-            average_stage = inlet0.inlet.get_global_average_stage()
-            average_xmom = inlet0.inlet.get_global_average_xmom()
-            average_ymom = inlet0.inlet.get_global_average_ymom()
-            average_volume = inlet0.inlet.get_global_total_water_volume()
-            average_depth = inlet0.inlet.get_global_average_depth()
-
-            if myid == inlet_master_proc:
-                if verbose: 
-                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
-
-                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
-
-                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
-
-                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
-
-                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
-
-
-        #assert(success)
-
-    return control_data
-
-
-if __name__=="__main__":
-    
-    test_points = []
-
-    if myid == 0:
-
-        for i in range(samples):
-            x = random.randrange(0,1000)/1000.0 * length
-            y = random.randrange(0,1000)/1000.0 * width
-            point = [x, y]
-            test_points.append(point)
-        
-        for i in range(1,numprocs):
-            pypar.send(test_points, i)
-    else:
-        test_points = pypar.receive(0)
-
-    print "Test Points::"
-    print test_points
-
-    if myid == 0:
-        control_data = run_test(parallel=False, test_points = test_points, verbose = True)
-        
-        for proc in range(1,numprocs):
-            pypar.send(control_data, proc)
-    else:
-        control_data = pypar.receive(0)
-
-    pypar.barrier()
-    run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = True)
-
-
-finalize()
Index: unk/anuga_core/source/anuga_parallel/test_parallel_boyd_box_op_apron.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_boyd_box_op_apron.py	(revision 9423)
+++ 	(revision )
@@ -1,365 +1,0 @@
-import os.path
-import sys
-
-from anuga.utilities.system_tools import get_pathname_from_package
-from anuga.geometry.polygon_function import Polygon_function
-        
-from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
-from anuga.abstract_2d_finite_volumes.quantity import Quantity
-from anuga.abstract_2d_finite_volumes.util import file_function
-
-import anuga
-
-import warnings
-warnings.simplefilter("ignore")
-
-#------------------------------------------
-# Import pypar without the initial output
-#------------------------------------------
-class NullStream:
-    def write(self,text):
-        pass
-sys.stdout = NullStream()
-import pypar
-sys.stdout = sys.__stdout__
-
-
-from math import pi, pow, sqrt
-
-import numpy as num
-from parallel_inlet_operator import Parallel_Inlet_operator
-from anuga_parallel import distribute, myid, numprocs, finalize
-from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
-
-from parallel_operator_factory import Inlet_operator, Boyd_box_operator
-
-import random
-import unittest
-
-
-"""
-
-This test exercises the parallel culvert and checks values
-"""
-verbose = False
-nprocs = 3
-    
-
-length = 40.
-width = 16.
-
-dx = dy = 4           # Resolution: Length of subdivisions on both axes
-
-#----------------------------------------------------------------------
-# Setup initial conditions
-#----------------------------------------------------------------------
-
-def topography(x, y):
-    """Set up a weir
-    
-    A culvert will connect either side
-    """
-    # General Slope of Topography
-    z=-x/1000
-    
-    N = len(x)
-    for i in range(N):
-
-       # Sloping Embankment Across Channel
-        if 5.0 < x[i] < 10.1:
-            # Cut Out Segment for Culvert face                
-            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
-               z[i]=z[i]
-            else:
-               z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
-        if 10.0 < x[i] < 12.1:
-           z[i] +=  2.5                    # Flat Crest of Embankment
-        if 12.0 < x[i] < 14.5:
-            # Cut Out Segment for Culvert face                
-            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
-               z[i]=z[i]
-            else:
-               z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
-                   
-        
-    return z
-
-#filename=os.path.join(path, 'example_rating_curve.csv')
-
-line0 = [[10.0, 10.0], [30.0, 10.0]]
-#line0 = [[29.0, 10.0], [30.0, 10.0]]
-line1 = [[0.0, 10.0], [0.0, 15.0]]
-Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
-Q1 = 5.0
-
-samples = 50
-
-def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
-    success = True
-
-##-----------------------------------------------------------------------
-## Setup domain
-##-----------------------------------------------------------------------
-
-    points, vertices, boundary = rectangular_cross(int(length/dx),
-                                                   int(width/dy),
-                                                   len1=length, 
-                                                   len2=width)
-
-    domain = anuga.Domain(points, vertices, boundary)   
-    domain.set_name('Test_Parallel_Frac_Op')                 # Output name
-    domain.set_default_order(2)
-
-##-----------------------------------------------------------------------
-## Distribute domain
-##-----------------------------------------------------------------------
-
-    if parallel:
-        domain = distribute(domain)
-        domain.dump_triangulation("frac_op_domain.png")
-    
-
-##-----------------------------------------------------------------------
-## Setup boundary conditions
-##-----------------------------------------------------------------------
-    
-    domain.set_quantity('elevation', topography) 
-    domain.set_quantity('friction', 0.01)         # Constant friction 
-    domain.set_quantity('stage',
-                        expression='elevation')   # Dry initial condition
-
-    
-    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
-    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
-    domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
-
-
-##-----------------------------------------------------------------------
-## Determine triangle index coinciding with test points
-##-----------------------------------------------------------------------
-
-    assert(test_points is not None)
-    assert(len(test_points) == samples)
-
-    tri_ids = []
-
-    for point in test_points:
-        try:
-            k = domain.get_triangle_containing_point(point)
-            if domain.tri_full_flag[k] == 1:
-                tri_ids.append(k)
-            else:
-                tri_ids.append(-1)            
-        except:
-            tri_ids.append(-2)
-
-    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
-
-    if not parallel: control_data = []
-
-    ################ Define Fractional Operators ##########################
-
-    inlet0 = None
-    inlet1 = None
-    boyd_box0 = None
-    
-    inlet0 = Inlet_operator(domain, line0, Q0, verbose = False, label = 'Inlet_0')
-    inlet1 = Inlet_operator(domain, line1, Q1, verbose = False, label = 'Inlet_1')
-    
-    # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
-    
-    boyd_box0 = Boyd_box_operator(domain,
-                                  end_points=[[9.0, 2.5],[19.0, 2.5]],
-                                  losses=1.5,
-                                  width=5.0,
-                                  apron=5.0,
-                                  use_momentum_jet=True,
-                                  use_velocity_head=False,
-                                  manning=0.013,
-                                  label='Boyd_Box_0',
-                                  verbose=False)
-        
-    #if inlet0 is not None and verbose: inlet0.print_statistics()
-    #if inlet1 is not None and verbose: inlet1.print_statistics()
-    
-    if boyd_box0 is not None and verbose:
-        print "++++", myid
-        boyd_box0.print_statistics()
-
-#    if parallel:
-#        factory = Parallel_operator_factory(domain, verbose = True)
-#
-#        inlet0 = factory.inlet_operator_factory(line0, Q0)
-#        inlet1 = factory.inlet_operator_factory(line1, Q1)
-#        
-#        boyd_box0 = factory.boyd_box_operator_factory(end_points=[[9.0, 2.5],[19.0, 2.5]],
-#                                          losses=1.5,
-#                                          width=1.5,
-#                                          apron=5.0,
-#                                          use_momentum_jet=True,
-#                                          use_velocity_head=False,
-#                                          manning=0.013,
-#                                          verbose=False)
-#
-#    else:
-#        inlet0 = Inlet_operator(domain, line0, Q0)
-#        inlet1 = Inlet_operator(domain, line1, Q1)
-#
-#        # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
-#        boyd_box0 = Boyd_box_operator(domain,
-#                          end_points=[[9.0, 2.5],[19.0, 2.5]],
-#                          losses=1.5,
-#                          width=1.5,
-#                          apron=5.0,
-#                          use_momentum_jet=True,
-#                          use_velocity_head=False,
-#                          manning=0.013,
-#                          verbose=False)
-    
-    #######################################################################
-
-    ##-----------------------------------------------------------------------
-    ## Evolve system through time
-    ##-----------------------------------------------------------------------
-
-    for t in domain.evolve(yieldstep = 2.0, finaltime = 20.0):
-        if myid == 0 and verbose:
-            domain.write_time()
-
-        #print domain.volumetric_balance_statistics()
-    
-        stage = domain.get_quantity('stage')
-
-
-        if boyd_box0 is not None and verbose :
-            if myid == boyd_box0.master_proc:
-                print 'master_proc ',myid
-                boyd_box0.print_timestepping_statistics()
- 
-        #for i in range(samples):
-        #    if tri_ids[i] >= 0:                
-        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
-                    
-        sys.stdout.flush()
- 
-        pass
-
-    success = True
-
-##-----------------------------------------------------------------------
-## Assign/Test Control data
-##-----------------------------------------------------------------------
-
-    if not parallel:
-        stage = domain.get_quantity('stage')
-
-        for i in range(samples):
-            assert(tri_ids[i] >= 0)
-            control_data.append(stage.centroid_values[tri_ids[i]])
-        
-        if inlet0 is not None:
-            control_data.append(inlet0.inlet.get_average_stage())
-            control_data.append(inlet0.inlet.get_average_xmom())
-            control_data.append(inlet0.inlet.get_average_ymom())
-            control_data.append(inlet0.inlet.get_total_water_volume())
-            control_data.append(inlet0.inlet.get_average_depth())
-
-        if verbose: print 'P%d control_data = %s' %(myid, control_data)
-    else:
-        stage = domain.get_quantity('stage')
-        
-        for i in range(samples):
-            if tri_ids[i] >= 0:
-                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
-                success = success and local_success
-                if verbose: 
-                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
-                
-                
-        if inlet0 is not None:
-            inlet_master_proc = inlet0.inlet.get_master_proc()
-            average_stage = inlet0.inlet.get_global_average_stage()
-            average_xmom = inlet0.inlet.get_global_average_xmom()
-            average_ymom = inlet0.inlet.get_global_average_ymom()
-            average_volume = inlet0.inlet.get_global_total_water_volume()
-            average_depth = inlet0.inlet.get_global_average_depth()
-
-            if myid == inlet_master_proc:
-                if verbose: 
-                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
-
-                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
-
-                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
-
-                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
-
-                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
-
-
-        assert(success)
-
-    return control_data
-
-
-# Test an nprocs-way run of the shallow water equations
-# against the sequential code.
-
-
-
-class Test_parallel_boyd_box_apron(unittest.TestCase):
-    def test_parallel_operator(self):
-        
-
-        #print "Expect this test to fail if not run from the parallel directory."
-        result = os.system("mpirun -np %d python test_parallel_boyd_box_op_apron.py" % nprocs)
-        assert_(result == 0)
-
-
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #pypar.finalize()
-        raise AssertionError, msg
-
-if __name__=="__main__":
-    if numprocs == 1:
-        runner = unittest.TextTestRunner()
-        suite = unittest.makeSuite(Test_parallel_boyd_box_apron, 'test')
-        #print "Running for numproc = 1"
-        runner.run(suite)
-    else:
-        #print "Running for numproc > 1"
-        pypar.barrier()
-        test_points = []
-
-        if myid == 0:
-            if verbose: print 'PARALLEL START'
-            for i in range(samples):
-                x = random.randrange(0,1000)/1000.0 * length
-                y = random.randrange(0,1000)/1000.0 * width
-                point = [x, y]
-                test_points.append(point)
-
-            for i in range(1,numprocs):
-                pypar.send(test_points, i)
-        else:
-            test_points = pypar.receive(0)
-
-        if myid == 0:
-            control_data = run_test(parallel=False, test_points = test_points, verbose = verbose)
-
-            for proc in range(1,numprocs):
-                pypar.send(control_data, proc)
-        else:
-            control_data = pypar.receive(0)
-
-
-        pypar.barrier()
-        run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = verbose)
-
-
-    finalize()
-    
-
Index: unk/anuga_core/source/anuga_parallel/test_parallel_boyd_box_operator.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_boyd_box_operator.py	(revision 9423)
+++ 	(revision )
@@ -1,367 +1,0 @@
-import os.path
-import sys
-
-from anuga.utilities.system_tools import get_pathname_from_package
-from anuga.geometry.polygon_function import Polygon_function
-        
-from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
-from anuga.abstract_2d_finite_volumes.quantity import Quantity
-from anuga.abstract_2d_finite_volumes.util import file_function
-
-import anuga
-
-import warnings
-warnings.simplefilter("ignore")
-
-#from anuga.structures.boyd_box_operator import Boyd_box_operator
-#from anuga.structures.inlet_operator import Inlet_operator
-                            
-#from anuga.culvert_flows.culvert_routines import boyd_generalised_culvert_model
-
-
-#------------------------------------------
-# Import pypar without the initial output
-#------------------------------------------
-class NullStream:
-    def write(self,text):
-        pass
-sys.stdout = NullStream()
-import pypar
-sys.stdout = sys.__stdout__
-
-
-from math import pi, pow, sqrt
-
-import numpy as num
-from parallel_inlet_operator import Parallel_Inlet_operator
-from anuga_parallel import distribute, myid, numprocs, finalize
-from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
-
-from parallel_operator_factory import Inlet_operator, Boyd_box_operator
-
-import random
-import unittest
-
-
-"""
-
-This test exercises the parallel culvert and checks values
-"""
-verbose = False
-nprocs = 3
-    
-
-length = 40.
-width = 16.
-
-dx = dy = 4           # Resolution: Length of subdivisions on both axes
-
-#----------------------------------------------------------------------
-# Setup initial conditions
-#----------------------------------------------------------------------
-
-def topography(x, y):
-    """Set up a weir
-    
-    A culvert will connect either side
-    """
-    # General Slope of Topography
-    z=-x/1000
-    
-    N = len(x)
-    for i in range(N):
-
-       # Sloping Embankment Across Channel
-        if 5.0 < x[i] < 10.1:
-            # Cut Out Segment for Culvert face                
-            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
-               z[i]=z[i]
-            else:
-               z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
-        if 10.0 < x[i] < 12.1:
-           z[i] +=  2.5                    # Flat Crest of Embankment
-        if 12.0 < x[i] < 14.5:
-            # Cut Out Segment for Culvert face                
-            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
-               z[i]=z[i]
-            else:
-               z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
-                   
-        
-    return z
-
-#filename=os.path.join(path, 'example_rating_curve.csv')
-
-line0 = [[10.0, 10.0], [30.0, 10.0]]
-#line0 = [[29.0, 10.0], [30.0, 10.0]]
-line1 = [[0.0, 10.0], [0.0, 15.0]]
-Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
-Q1 = 5.0
-
-samples = 50
-
-def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
-    success = True
-
-##-----------------------------------------------------------------------
-## Setup domain
-##-----------------------------------------------------------------------
-
-    points, vertices, boundary = rectangular_cross(int(length/dx),
-                                                   int(width/dy),
-                                                   len1=length, 
-                                                   len2=width)
-
-    domain = anuga.Domain(points, vertices, boundary)   
-    domain.set_name('Test_Parallel_Frac_Op')                 # Output name
-    domain.set_default_order(2)
-
-##-----------------------------------------------------------------------
-## Distribute domain
-##-----------------------------------------------------------------------
-
-    if parallel:
-        domain = distribute(domain)
-        domain.dump_triangulation("frac_op_domain.png")
-    
-
-##-----------------------------------------------------------------------
-## Setup boundary conditions
-##-----------------------------------------------------------------------
-    
-    domain.set_quantity('elevation', topography) 
-    domain.set_quantity('friction', 0.01)         # Constant friction 
-    domain.set_quantity('stage',
-                        expression='elevation')   # Dry initial condition
-
-    
-    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
-    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
-    domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
-
-
-##-----------------------------------------------------------------------
-## Determine triangle index coinciding with test points
-##-----------------------------------------------------------------------
-
-    assert(test_points is not None)
-    assert(len(test_points) == samples)
-
-    tri_ids = []
-
-    for point in test_points:
-        try:
-            k = domain.get_triangle_containing_point(point)
-            if domain.tri_full_flag[k] == 1:
-                tri_ids.append(k)
-            else:
-                tri_ids.append(-1)            
-        except:
-            tri_ids.append(-2)
-
-    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
-
-    if not parallel: control_data = []
-
-    ################ Define Fractional Operators ##########################
-
-    inlet0 = None
-    inlet1 = None
-    boyd_box0 = None
-    
-    inlet0 = Inlet_operator(domain, line0, Q0, verbose = False, label = 'Inlet_0')
-    inlet1 = Inlet_operator(domain, line1, Q1, verbose = False, label = 'Inlet_1')
-    
-    # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
-    
-    boyd_box0 = Boyd_box_operator(domain,
-                                  end_points=[[9.0, 2.5],[19.0, 2.5]],
-                                  losses=1.5,
-                                  width=5.0,
-                                  #apron=5.,
-                                  use_momentum_jet=True,
-                                  use_velocity_head=False,
-                                  manning=0.013,
-                                  label='Boyd_Box_0',
-                                  verbose=False)
-        
-    #if inlet0 is not None and verbose: inlet0.print_statistics()
-    #if inlet1 is not None and verbose: inlet1.print_statistics()
-    
-    if boyd_box0 is not None and verbose:
-        print "++++", myid
-        boyd_box0.print_statistics()
-
-#    if parallel:
-#        factory = Parallel_operator_factory(domain, verbose = True)
-#
-#        inlet0 = factory.inlet_operator_factory(line0, Q0)
-#        inlet1 = factory.inlet_operator_factory(line1, Q1)
-#        
-#        boyd_box0 = factory.boyd_box_operator_factory(end_points=[[9.0, 2.5],[19.0, 2.5]],
-#                                          losses=1.5,
-#                                          width=1.5,
-#                                          apron=5.0,
-#                                          use_momentum_jet=True,
-#                                          use_velocity_head=False,
-#                                          manning=0.013,
-#                                          verbose=False)
-#
-#    else:
-#        inlet0 = Inlet_operator(domain, line0, Q0)
-#        inlet1 = Inlet_operator(domain, line1, Q1)
-#
-#        # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
-#        boyd_box0 = Boyd_box_operator(domain,
-#                          end_points=[[9.0, 2.5],[19.0, 2.5]],
-#                          losses=1.5,
-#                          width=1.5,
-#                          apron=5.0,
-#                          use_momentum_jet=True,
-#                          use_velocity_head=False,
-#                          manning=0.013,
-#                          verbose=False)
-    
-    #######################################################################
-
-    ##-----------------------------------------------------------------------
-    ## Evolve system through time
-    ##-----------------------------------------------------------------------
-
-    for t in domain.evolve(yieldstep = 2.0, finaltime = 20.0):
-        if myid == 0 and verbose:
-            domain.write_time()
-
-        #print domain.volumetric_balance_statistics()
-    
-        stage = domain.get_quantity('stage')
-
-
-        if boyd_box0 is not None and verbose :
-            if myid == boyd_box0.master_proc:
-                print 'master_proc ',myid
-                boyd_box0.print_timestepping_statistics()
- 
-        #for i in range(samples):
-        #    if tri_ids[i] >= 0:                
-        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
-                    
-        sys.stdout.flush()
- 
-        pass
-
-    success = True
-
-##-----------------------------------------------------------------------
-## Assign/Test Control data
-##-----------------------------------------------------------------------
-
-    if not parallel:
-        stage = domain.get_quantity('stage')
-
-        for i in range(samples):
-            assert(tri_ids[i] >= 0)
-            control_data.append(stage.centroid_values[tri_ids[i]])
-        
-        if inlet0 is not None:
-            control_data.append(inlet0.inlet.get_average_stage())
-            control_data.append(inlet0.inlet.get_average_xmom())
-            control_data.append(inlet0.inlet.get_average_ymom())
-            control_data.append(inlet0.inlet.get_total_water_volume())
-            control_data.append(inlet0.inlet.get_average_depth())
-
-        if verbose: print 'P%d control_data = %s' %(myid, control_data)
-    else:
-        stage = domain.get_quantity('stage')
-        
-        for i in range(samples):
-            if tri_ids[i] >= 0:
-                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
-                success = success and local_success
-                if verbose: 
-                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
-                
-                
-        if inlet0 is not None:
-            inlet_master_proc = inlet0.inlet.get_master_proc()
-            average_stage = inlet0.inlet.get_global_average_stage()
-            average_xmom = inlet0.inlet.get_global_average_xmom()
-            average_ymom = inlet0.inlet.get_global_average_ymom()
-            average_volume = inlet0.inlet.get_global_total_water_volume()
-            average_depth = inlet0.inlet.get_global_average_depth()
-
-            if myid == inlet_master_proc:
-                if verbose: 
-                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
-
-                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
-
-                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
-
-                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
-
-                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
-
-
-        assert(success)
-
-    return control_data
-
-
-# Test an nprocs-way run of the shallow water equations
-# against the sequential code.
-
-class Test_parallel_boyd_box_operator(unittest.TestCase):
-    def test_parallel_operator(self):
-        #print "Expect this test to fail if not run from the parallel directory."
-        result = os.system("mpirun -np %d python test_parallel_boyd_box_operator.py" % nprocs)
-        assert_(result == 0)
-
-
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #pypar.finalize()
-        raise AssertionError, msg
-
-if __name__=="__main__":
-    if numprocs == 1:
-        runner = unittest.TextTestRunner()
-        suite = unittest.makeSuite(Test_parallel_boyd_box_operator, 'test')
-        #print "Running for numproc = 1"
-        runner.run(suite)
-    else:
-        #print "Running for numproc > 1"
-        pypar.barrier()
-        test_points = []
-
-        if myid == 0:
-            if verbose: print 'PARALLEL START'
-            for i in range(samples):
-                x = random.randrange(0,1000)/1000.0 * length
-                y = random.randrange(0,1000)/1000.0 * width
-                point = [x, y]
-                test_points.append(point)
-
-            for i in range(1,numprocs):
-                pypar.send(test_points, i)
-        else:
-            test_points = pypar.receive(0)
-
-        if myid == 0:
-            control_data = run_test(parallel=False, test_points = test_points, verbose = verbose)
-
-            for proc in range(1,numprocs):
-                pypar.send(control_data, proc)
-        else:
-            control_data = pypar.receive(0)
-
-
-        pypar.barrier()
-        run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = verbose)
-
-
-    finalize()
-    
-
Index: unk/anuga_core/source/anuga_parallel/test_parallel_boyd_pipe_operator.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_boyd_pipe_operator.py	(revision 9423)
+++ 	(revision )
@@ -1,356 +1,0 @@
-import os.path
-import sys
-
-from anuga.utilities.system_tools import get_pathname_from_package
-from anuga.geometry.polygon_function import Polygon_function
-        
-from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
-from anuga.abstract_2d_finite_volumes.quantity import Quantity
-from anuga.abstract_2d_finite_volumes.util import file_function
-
-import anuga
-
-#------------------------------------------
-# Import pypar without the initial output
-#------------------------------------------
-class NullStream:
-    def write(self,text):
-        pass
-sys.stdout = NullStream()
-import pypar
-sys.stdout = sys.__stdout__
-
-
-from math import pi, pow, sqrt
-
-import numpy as num
-
-#from parallel_inlet_operator import Parallel_Inlet_operator
-
-from anuga_parallel import distribute, myid, numprocs, finalize
-from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
-
-from parallel_operator_factory import Inlet_operator
-from parallel_operator_factory import Boyd_pipe_operator
-
-import random
-import unittest
-
-
-"""
-
-This test exercises the parallel culvert and checks values
-"""
-verbose = False
-nprocs = 3
-    
-
-length = 40.
-width = 16.
-
-dx = dy = 2           # Resolution: Length of subdivisions on both axes
-
-#----------------------------------------------------------------------
-# Setup initial conditions
-#----------------------------------------------------------------------
-
-def topography(x, y):
-    """Set up a weir
-    
-    A culvert will connect either side
-    """
-    # General Slope of Topography
-    z=-x/1000
-    
-    N = len(x)
-    for i in range(N):
-
-       # Sloping Embankment Across Channel
-        if 5.0 < x[i] < 10.1:
-            # Cut Out Segment for Culvert face                
-            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
-               z[i]=z[i]
-            else:
-               z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
-        if 10.0 < x[i] < 12.1:
-           z[i] +=  2.5                    # Flat Crest of Embankment
-        if 12.0 < x[i] < 14.5:
-            # Cut Out Segment for Culvert face                
-            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
-               z[i]=z[i]
-            else:
-               z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
-                   
-        
-    return z
-
-#filename=os.path.join(path, 'example_rating_curve.csv')
-
-line0 = [[10.0, 10.0], [30.0, 10.0]]
-#line0 = [[29.0, 10.0], [30.0, 10.0]]
-line1 = [[0.0, 10.0], [0.0, 15.0]]
-Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
-Q1 = 5.0
-
-samples = 50
-
-def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
-    success = True
-
-##-----------------------------------------------------------------------
-## Setup domain
-##-----------------------------------------------------------------------
-
-    points, vertices, boundary = rectangular_cross(int(length/dx),
-                                                   int(width/dy),
-                                                   len1=length, 
-                                                   len2=width)
-
-    domain = anuga.Domain(points, vertices, boundary)   
-    domain.set_name('Test_Parallel_Boyd_Pipe_Op')                 # Output name
-    domain.set_default_order(2)
-
-##-----------------------------------------------------------------------
-## Distribute domain
-##-----------------------------------------------------------------------
-
-    if parallel:
-        domain = distribute(domain)
-        domain.dump_triangulation("boyd_pipe_domain.png")
-    
-
-##-----------------------------------------------------------------------
-## Setup boundary conditions
-##-----------------------------------------------------------------------
-    
-    domain.set_quantity('elevation', topography) 
-    domain.set_quantity('friction', 0.01)         # Constant friction 
-    domain.set_quantity('stage',
-                        expression='elevation')   # Dry initial condition
-
-    
-    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
-    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
-    domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
-
-
-##-----------------------------------------------------------------------
-## Determine triangle index coinciding with test points
-##-----------------------------------------------------------------------
-
-    assert(test_points is not None)
-    assert(len(test_points) == samples)
-
-    tri_ids = []
-
-    for point in test_points:
-        try:
-            k = domain.get_triangle_containing_point(point)
-            if domain.tri_full_flag[k] == 1:
-                tri_ids.append(k)
-            else:
-                tri_ids.append(-1)            
-        except:
-            tri_ids.append(-2)
-
-    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
-
-    if not parallel: control_data = []
-
-    ################ Define Fractional Operators ##########################
-
-    inlet0 = None
-    inlet1 = None
-    boyd_pipe0 = None
-    
-    inlet0 = Inlet_operator(domain, line0, Q0, verbose = False)
-    inlet1 = Inlet_operator(domain, line1, Q1, verbose = False)
-    
-    # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
-    
-    boyd_pipe0 = Boyd_pipe_operator(domain,
-                                  end_points=[[9.0, 2.5],[19.0, 2.5]],
-                                  losses=1.5,
-                                  diameter=5.0,
-                                  apron=5.0,
-                                  use_momentum_jet=True,
-                                  use_velocity_head=False,
-                                  manning=0.013,
-                                  verbose=False)
-        
-    if inlet0 is not None and verbose: inlet0.print_statistics()
-    if inlet1 is not None and verbose: inlet1.print_statistics()
-    if boyd_pipe0 is not None and verbose: boyd_pipe0.print_statistics()
-
-#    if parallel:
-#        factory = Parallel_operator_factory(domain, verbose = True)
-#
-#        inlet0 = factory.inlet_operator_factory(line0, Q0)
-#        inlet1 = factory.inlet_operator_factory(line1, Q1)
-#        
-#        boyd_box0 = factory.boyd_box_operator_factory(end_points=[[9.0, 2.5],[19.0, 2.5]],
-#                                          losses=1.5,
-#                                          width=1.5,
-#                                          apron=5.0,
-#                                          use_momentum_jet=True,
-#                                          use_velocity_head=False,
-#                                          manning=0.013,
-#                                          verbose=False)
-#
-#    else:
-#        inlet0 = Inlet_operator(domain, line0, Q0)
-#        inlet1 = Inlet_operator(domain, line1, Q1)
-#
-#        # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
-#        boyd_box0 = Boyd_box_operator(domain,
-#                          end_points=[[9.0, 2.5],[19.0, 2.5]],
-#                          losses=1.5,
-#                          width=1.5,
-#                          apron=5.0,
-#                          use_momentum_jet=True,
-#                          use_velocity_head=False,
-#                          manning=0.013,
-#                          verbose=False)
-    
-    #######################################################################
-
-    ##-----------------------------------------------------------------------
-    ## Evolve system through time
-    ##-----------------------------------------------------------------------
-
-    for t in domain.evolve(yieldstep = 2.0, finaltime = 20.0):
-        if myid == 0 and verbose:
-            domain.write_time()
-
-        #print domain.volumetric_balance_statistics()
-    
-        stage = domain.get_quantity('stage')
-
-
-        if boyd_pipe0 is not None and verbose : boyd_pipe0.print_timestepping_statistics()
- 
-        #for i in range(samples):
-        #    if tri_ids[i] >= 0:                
-        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
-                    
-        sys.stdout.flush()
- 
-        pass
-
-    success = True
-
-##-----------------------------------------------------------------------
-## Assign/Test Control data
-##-----------------------------------------------------------------------
-
-    if not parallel:
-        stage = domain.get_quantity('stage')
-
-        for i in range(samples):
-            assert(tri_ids[i] >= 0)
-            control_data.append(stage.centroid_values[tri_ids[i]])
-        
-        if inlet0 is not None:
-            control_data.append(inlet0.inlet.get_average_stage())
-            control_data.append(inlet0.inlet.get_average_xmom())
-            control_data.append(inlet0.inlet.get_average_ymom())
-            control_data.append(inlet0.inlet.get_total_water_volume())
-            control_data.append(inlet0.inlet.get_average_depth())
-
-        if verbose: print 'P%d control_data = %s' %(myid, control_data)
-    else:
-        stage = domain.get_quantity('stage')
-        
-        for i in range(samples):
-            if tri_ids[i] >= 0:
-                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
-                success = success and local_success
-                if verbose: 
-                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
-                
-                
-        if inlet0 is not None:
-            inlet_master_proc = inlet0.inlet.get_master_proc()
-            average_stage = inlet0.inlet.get_global_average_stage()
-            average_xmom = inlet0.inlet.get_global_average_xmom()
-            average_ymom = inlet0.inlet.get_global_average_ymom()
-            average_volume = inlet0.inlet.get_global_total_water_volume()
-            average_depth = inlet0.inlet.get_global_average_depth()
-
-            if myid == inlet_master_proc:
-                if verbose: 
-                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
-
-                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
-
-                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
-
-                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
-
-                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
-
-
-        assert(success)
-
-    if parallel:
-        finalize()
-
-    return control_data
-
-
-# Test an nprocs-way run of the shallow water equations
-# against the sequential code.
-
-class Test_parallel_boyd_pipe_op(unittest.TestCase):
-    def test_parallel_operator(self):
-        #print "Expect this test to fail if not run from the parallel directory."
-        result = os.system("mpirun -np %d python test_parallel_boyd_pipe_operator.py" % nprocs)
-        assert_(result == 0)
-
-
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #pypar.finalize()
-        raise AssertionError, msg
-
-if __name__=="__main__":
-    if numprocs == 1:
-        runner = unittest.TextTestRunner()
-        suite = unittest.makeSuite(Test_parallel_boyd_pipe_op, 'test')
-        #print "Running for numproc = 1"
-        runner.run(suite)
-    else:
-        #print "Running for numproc > 1"
-        pypar.barrier()
-        test_points = []
-
-        if myid == 0:
-            if verbose: print 'PARALLEL START'
-            for i in range(samples):
-                x = random.randrange(0,1000)/1000.0 * length
-                y = random.randrange(0,1000)/1000.0 * width
-                point = [x, y]
-                test_points.append(point)
-
-            for i in range(1,numprocs):
-                pypar.send(test_points, i)
-        else:
-            test_points = pypar.receive(0)
-
-        if myid == 0:
-            control_data = run_test(parallel=False, test_points = test_points, verbose = verbose)
-
-            for proc in range(1,numprocs):
-                pypar.send(control_data, proc)
-        else:
-            control_data = pypar.receive(0)
-
-
-        pypar.barrier()
-        run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = verbose)
-
-
-    
-
Index: unk/anuga_core/source/anuga_parallel/test_parallel_distribute_domain.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_distribute_domain.py	(revision 9423)
+++ 	(revision )
@@ -1,238 +1,0 @@
-
-"""Test a run of the sequential shallow water domain against
-a run of the parallel shallow water domain.
-
-WARNING: This assumes that the command to run jobs is mpirun.
-Tested with MPICH and LAM (Ole)
-"""
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-
-import unittest
-import os
-import sys
-
-
-
-import numpy as num
-
-#------------------------------------------
-# Import pypar without the initial output
-#------------------------------------------
-class NullStream:
-    def write(self,text):
-        pass
-sys.stdout = NullStream()
-import pypar
-sys.stdout = sys.__stdout__
-
-
-#------------------------------------------
-# anuga imports
-#------------------------------------------
-
-from anuga.utilities.numerical_tools import ensure_numeric
-from anuga.utilities.util_ext        import double_precision
-from anuga.utilities.norms           import l1_norm, l2_norm, linf_norm
-
-from anuga import Domain
-from anuga import Reflective_boundary
-from anuga import Dirichlet_boundary
-from anuga import Time_boundary
-from anuga import Transmissive_boundary
-
-from anuga import rectangular_cross
-from anuga import create_domain_from_file
-
-
-from anuga_parallel import distribute, myid, numprocs, finalize
-
-
-#--------------------------------------------------------------------------
-# Setup parameters
-#--------------------------------------------------------------------------
-
-mesh_filename = "merimbula_10785_1.tsh"
-#mesh_filename = "test-100.tsh"
-yieldstep = 1
-finaltime = 20
-quantity = 'stage'
-nprocs = 4
-verbose = False
-
-#--------------------------------------------------------------------------
-# Setup procedures
-#--------------------------------------------------------------------------
-class Set_Stage:
-    """Set an initial condition with constant water height, for x<x0
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))
-
-#--------------------------------------------------------------------------
-# Setup test
-#--------------------------------------------------------------------------
-def evolution_test(parallel=False):
-
-
-    domain = create_domain_from_file(mesh_filename)
-    domain.set_quantity('stage', Set_Stage(756000.0, 756500.0, 2.0))
-
-    #--------------------------------------------------------------------------
-    # Create parallel domain if requested
-    #--------------------------------------------------------------------------
-
-    if parallel:
-        if myid == 0 and verbose: print 'DISTRIBUTING PARALLEL DOMAIN'
-        domain = distribute(domain)
-
-    #------------------------------------------------------------------------------
-    # Setup boundary conditions
-    # This must currently happen *after* domain has been distributed
-    #------------------------------------------------------------------------------
-    domain.store = False
-    Br = Reflective_boundary(domain)      # Solid reflective wall
-
-    domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Br})
-
-    #------------------------------------------------------------------------------
-    # Setup diagnostic arrays
-    #------------------------------------------------------------------------------
-    l1list = []
-    l2list = []
-    linflist = []
-    l1norm = num.zeros(3, num.float)
-    l2norm = num.zeros(3, num.float)
-    linfnorm = num.zeros(3, num.float)
-    recv_norm = num.zeros(3, num.float)
-
-    #------------------------------------------------------------------------------
-    # Evolution
-    #------------------------------------------------------------------------------
-    if parallel:
-        if myid == 0 and verbose: print 'PARALLEL EVOLVE'
-    else:
-        if verbose: print 'SEQUENTIAL EVOLVE'
-        
-    for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-        edges = domain.quantities[quantity].edge_values.take(num.flatnonzero(domain.tri_full_flag),axis=0)
-        l1norm[0] = l1_norm(edges[:,0])
-        l1norm[1] = l1_norm(edges[:,1])
-        l1norm[2] = l1_norm(edges[:,2])
-        l2norm[0] = l2_norm(edges[:,0])
-        l2norm[1] = l2_norm(edges[:,1])
-        l2norm[2] = l2_norm(edges[:,2])
-        linfnorm[0] = linf_norm(edges[:,0])
-        linfnorm[1] = linf_norm(edges[:,1])
-        linfnorm[2] = linf_norm(edges[:,2])
-        if parallel:
-            l2norm[0] = pow(l2norm[0], 2)
-            l2norm[1] = pow(l2norm[1], 2)
-            l2norm[2] = pow(l2norm[2], 2)
-            if myid == 0:
-                #domain.write_time()
-
-                #print edges[:,1]            
-                for p in range(1, numprocs):
-                    recv_norm = pypar.receive(p)
-                    l1norm += recv_norm
-                    recv_norm = pypar.receive(p)
-                    l2norm += recv_norm
-                    recv_norm = pypar.receive(p)
-                    linfnorm[0] = max(linfnorm[0], recv_norm[0])
-                    linfnorm[1] = max(linfnorm[1], recv_norm[1])
-                    linfnorm[2] = max(linfnorm[2], recv_norm[2])
-
-                l2norm[0] = pow(l2norm[0], 0.5)
-                l2norm[1] = pow(l2norm[1], 0.5)
-                l2norm[2] = pow(l2norm[2], 0.5)
-
-                l1list.append(l1norm)                
-                l2list.append(l2norm)
-                linflist.append(linfnorm)                
-            else:
-                pypar.send(l1norm, 0)
-                pypar.send(l2norm, 0)
-                pypar.send(linfnorm, 0)
-        else:
-            #domain.write_time()
-            l1list.append(l1norm)                
-            l2list.append(l2norm)
-            linflist.append(linfnorm)
-            
-
-    return (l1list, l2list, linflist)
-
-# Test an nprocs-way run of the shallow water equations
-# against the sequential code.
-
-class Test_parallel_distribute_domain(unittest.TestCase):
-    def test_parallel_distribute_domain(self):
-        #print "Expect this test to fail if not run from the parallel directory."
-        result = os.system("mpirun -np %d python test_parallel_distribute_domain.py" % nprocs)
-        assert_(result == 0)
-
-
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #pypar.finalize()
-        raise AssertionError, msg
-
-if __name__=="__main__":
-    if numprocs == 1: 
-        runner = unittest.TextTestRunner()
-        suite = unittest.makeSuite(Test_parallel_distribute_domain, 'test')
-        runner.run(suite)
-    else:
-
-        pypar.barrier()
-        if myid == 0:
-            if verbose: print 'SEQUENTIAL START'
-            l1norm_seq, l2norm_seq, linfnorm_seq = evolution_test(parallel=False)
-
-        pypar.barrier()
-        if myid ==0:
-            if verbose: print 'PARALLEL START'
-        
-        l1norm_par, l2norm_par, linfnorm_par = evolution_test(parallel=True)
-        
-        if myid == 0:
-            assert_(len(l1norm_seq) == len(l1norm_par))
-            assert_(len(l2norm_seq) == len(l2norm_par))
-            assert_(len(linfnorm_seq) == len(linfnorm_par))
-            assert_(len(l1norm_seq) == len(l2norm_seq))
-            assert_(len(l2norm_seq) == len(linfnorm_seq))
-            # Anything smaller than tol we consider to be 0.
-            # This usualy comes out to be 10^-14 (DBL_DIG = 10^-15)
-            # * 10 to cater for rounding error in computation.
-            tol = pow(10, -1 * (double_precision() - 1))
-            for x in range(len(l1norm_seq)):
-                for y in range(3):
-                    # Calculate relative difference in the norms
-                    assert_(abs(l1norm_seq[x][y] - l1norm_par[x][y])/l1norm_seq[x][y] < tol)
-                    assert_(abs(l2norm_seq[x][y] - l2norm_par[x][y])/l2norm_seq[x][y] < tol)
-                    assert_(abs(linfnorm_seq[x][y] - linfnorm_par[x][y])/linfnorm_seq[x][y] < tol)
-                    if x > 0:
-                        # Verify that the quantity is being conserved across iterations.
-                        assert_(abs(l1norm_seq[x][y] - l1norm_seq[x-1][y]) < tol)
-                        assert_(abs(l2norm_seq[x][y] - l2norm_seq[x-1][y]) < tol)
-                        assert_(abs(linfnorm_seq[x][y] - linfnorm_seq[x-1][y]) < tol)
-                        assert_(abs(l1norm_par[x][y] - l1norm_par[x-1][y]) < tol)
-                        assert_(abs(l2norm_par[x][y] - l2norm_par[x-1][y]) < tol)
-                        assert_(abs(linfnorm_par[x][y] - linfnorm_par[x-1][y]) < tol)
-                
-            if verbose: print 'Parallel test OK'
-
-
-
-    finalize()
Index: unk/anuga_core/source/anuga_parallel/test_parallel_distribute_mesh.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_distribute_mesh.py	(revision 9423)
+++ 	(revision )
@@ -1,326 +1,0 @@
-#!/usr/bin/env python
-
-import unittest
-import sys
-import os
-from math import sqrt
-
-	
-from anuga import Domain
-from anuga import rectangular_cross
-
-from anuga_parallel.distribute_mesh import pmesh_divide_metis
-from anuga_parallel.distribute_mesh import build_submesh
-from anuga_parallel.distribute_mesh import submesh_full, submesh_ghost, submesh_quantities
-from anuga_parallel.distribute_mesh import extract_submesh, rec_submesh, send_submesh
-
-from anuga_parallel import myid, numprocs, barrier, finalize
-
-import numpy as num
-
-
-def topography(x,y): 
-    return -x/2
-
-
-def xcoord(x,y):
-    return x
-
-def ycoord(x,y):
-    return y
-
-
-def distibute_three_processors():
-    """
-    Do a parallel test of distributing a rectangle onto 3 processors
-    """
-
-    import pypar
-
-    myid = pypar.rank()
-    numprocs = pypar.size()
-
-    if not numprocs == 3: return
-
-    #print numprocs
-
-    barrier()
-
-    if myid == 0:
-
-        points, vertices, boundary = rectangular_cross(2,2)
-
-        domain = Domain(points, vertices, boundary)
-
-
-        domain.set_quantity('elevation', topography) # Use function for elevation
-        domain.set_quantity('friction', 0.0)         # Constant friction 
-        domain.set_quantity('stage', expression='elevation') # Dry initial stage
-        domain.set_quantity('xmomentum', expression='friction + 2.0') # 
-        domain.set_quantity('ymomentum', ycoord) #
-
-        #----------------------------------------------------------------------------------
-        # Test pmesh_divide_metis
-        #----------------------------------------------------------------------------------
-        nodes, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(domain,numprocs)
-
-        assert_(num.allclose(nodes,points))
-
-        true_vertices = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]]
-
-        true_triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7]]
-
-        assert_(num.allclose(vertices,true_vertices))
-        assert_(num.allclose(triangles,true_triangles))
-
-        assert_(num.allclose(triangles_per_proc,[5,6,5]))
-
-
-        #----------------------------------------------------------------------------------
-        # Test build_submesh
-        #----------------------------------------------------------------------------------
-        submesh = build_submesh(nodes, triangles, boundary, quantities, triangles_per_proc)
-
-
-        assert_(num.allclose(submesh['full_nodes'][0],[[3.0, 0.5, 0.0], [4.0, 0.5, 0.5], [5.0, 0.5, 1.0], [7.0, 1.0, 0.5], [8.0, 1.0, 1.0], [9.0, 0.25, 0.25], [12.0, 0.75, 0.75]]))
-        assert_(num.allclose(submesh['full_nodes'][1],[[0.0, 0.0, 0.0], [1.0, 0.0, 0.5], [2.0, 0.0, 1.0], [4.0, 0.5, 0.5], [5.0, 0.5, 1.0], [9.0, 0.25, 0.25], [10.0, 0.25, 0.75]]))
-        assert_(num.allclose(submesh['full_nodes'][2],[[0.0, 0.0, 0.0], [3.0, 0.5, 0.0], [4.0, 0.5, 0.5], [6.0, 1.0, 0.0], [7.0, 1.0, 0.5], [9.0, 0.25, 0.25], [11.0, 0.75, 0.25]]))
-
-
-        assert_(num.allclose(submesh['ghost_nodes'][0],[[0.0, 0.0, 0.0], [1.0, 0.0, 0.5], [2.0, 0.0, 1.0], [6.0, 1.0, 0.0], [10.0, 0.25, 0.75], [11.0, 0.75, 0.25]]))
-        assert_(num.allclose(submesh['ghost_nodes'][1],[[3.0, 0.5, 0.0], [7.0, 1.0, 0.5], [8.0, 1.0, 1.0], [11.0, 0.75, 0.25], [12.0, 0.75, 0.75]]))
-        assert_(num.allclose(submesh['ghost_nodes'][2],[[1.0, 0.0, 0.5], [5.0, 0.5, 1.0], [8.0, 1.0, 1.0], [12.0, 0.75, 0.75]]))
-
-
-
-        true_full_triangles = [num.array([[ 4,  9,  3],
-                                          [ 4, 12,  5],
-                                          [ 7, 12,  4],
-                                          [ 8, 12,  7],
-                                          [ 5, 12,  8]]),
-                               num.array([[ 0,  9,  1],
-                                          [ 1,  9,  4],
-                                          [ 1, 10,  2],
-                                          [ 4, 10,  1],
-                                          [ 5, 10,  4],
-                                          [ 2, 10,  5]]),
-                               num.array([[ 3,  9,  0],
-                                          [ 3, 11,  4],
-                                          [ 6, 11,  3],
-                                          [ 7, 11,  6],
-                                          [ 4, 11,  7]])]
-
-
-        assert_(num.allclose(submesh['full_triangles'][0],true_full_triangles[0]))
-        assert_(num.allclose(submesh['full_triangles'][1],true_full_triangles[1]))
-        assert_(num.allclose(submesh['full_triangles'][2],true_full_triangles[2]))
-
-        true_ghost_triangles = [num.array([[ 5,  0,  9,  1],
-                                           [ 6,  1,  9,  4],
-                                           [ 8,  4, 10,  1],
-                                           [ 9,  5, 10,  4],
-                                           [10,  2, 10,  5],
-                                           [11,  3,  9,  0],
-                                           [12,  3, 11,  4],
-                                           [13,  6, 11,  3],
-                                           [14,  7, 11,  6],
-                                           [15,  4, 11,  7]]),
-                                num.array([[ 0,  4,  9,  3],
-                                           [ 1,  4, 12,  5],
-                                           [ 2,  7, 12,  4],
-                                           [ 4,  5, 12,  8],
-                                           [11,  3,  9,  0],
-                                           [12,  3, 11,  4]]),
-                                num.array([[ 0,  4,  9,  3],
-                                           [ 1,  4, 12,  5],
-                                           [ 2,  7, 12,  4],
-                                           [ 3,  8, 12,  7],
-                                           [ 5,  0,  9,  1],
-                                           [ 6,  1,  9,  4]])]
-
-
-
-        assert_(num.allclose(submesh['ghost_triangles'][0],true_ghost_triangles[0]))
-        assert_(num.allclose(submesh['ghost_triangles'][1],true_ghost_triangles[1]))
-        assert_(num.allclose(submesh['ghost_triangles'][2],true_ghost_triangles[2]))
-
-        true_full_commun = [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}]
-
-        assert_(true_full_commun == submesh['full_commun'])
-
-
-        true_ghost_commun = [num.array([[ 5,  1],
-                                        [ 6,  1],
-                                        [ 8,  1],
-                                        [ 9,  1],
-                                        [10,  1],
-                                        [11,  2],
-                                        [12,  2],
-                                        [13,  2],
-                                        [14,  2],
-                                        [15,  2]]),
-                             num.array([[ 0,  0],
-                                        [ 1,  0],
-                                        [ 2,  0],
-                                        [ 4,  0],
-                                        [11,  2],
-                                        [12,  2]]),
-                             num.array([[0, 0],
-                                        [1, 0],
-                                        [2, 0],
-                                        [3, 0],
-                                        [5, 1],
-                                        [6, 1]])]
-
-        assert_(num.allclose(submesh['ghost_commun'][0],true_ghost_commun[0]))
-        assert_(num.allclose(submesh['ghost_commun'][1],true_ghost_commun[1]))
-        assert_(num.allclose(submesh['ghost_commun'][2],true_ghost_commun[2]))
-
-
-
-    barrier()
-    #--------------------------------
-    # Now do the comunnication part
-    #--------------------------------
-
-
-    if myid == 0:
-        #----------------------------------------------------------------------------------
-        # Test send_submesh
-        #----------------------------------------------------------------------------------
-        for p in range(1, numprocs):
-            send_submesh(submesh, triangles_per_proc, p, verbose=False)
-
-        #----------------------------------------------------------------------------------
-        # Test extract_submesh
-        #----------------------------------------------------------------------------------
-        points, vertices, boundary, quantities, \
-                ghost_recv_dict, full_send_dict, tri_map, node_map, tri_l2g, node_l2g, \
-                ghost_layer_width =\
-        extract_submesh(submesh, triangles_per_proc)
-
-
-        true_points =  [[0.5, 0.0], [0.5, 0.5], [0.5, 1.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.75, 0.75], [0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [1.0, 0.0], [0.25, 0.75], [0.75, 0.25]]
-
-        true_vertices = [[1, 5, 0], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [7, 5, 8], [8, 5, 1], [1, 11, 8], [2, 11, 1], [9, 11, 2], [0, 5, 7], [0, 12, 1], [10, 12, 0], [3, 12, 10], [1, 12, 3]]
-
-
-        true_ghost_recv = {1: [num.array([5, 6, 7, 8, 9]), num.array([ 5,  6,  8,  9, 10])], 2: [num.array([10, 11, 12, 13, 14]), num.array([11, 12, 13, 14, 15])]}
-
-
-        true_full_send = {1: [num.array([0, 1, 2, 4]), num.array([0, 1, 2, 4])], 2: [num.array([0, 1, 2, 3]), num.array([0, 1, 2, 3])]}
-
-        assert_(num.allclose(ghost_layer_width,  2))
-        assert_(num.allclose(points,   true_points))
-        assert_(num.allclose(vertices, true_vertices))
-        assert_(num.allclose(ghost_recv_dict[1],true_ghost_recv[1]))
-        assert_(num.allclose(ghost_recv_dict[2],true_ghost_recv[2]))
-        assert_(num.allclose(full_send_dict[1],true_full_send[1]))
-        assert_(num.allclose(full_send_dict[2],true_full_send[2]))
-
-        #print triangles_per_proc
-
-    else:
-        #----------------------------------------------------------------------------------
-        # Test rec_submesh
-        #----------------------------------------------------------------------------------
-        points, vertices, boundary, quantities, \
-                ghost_recv_dict, full_send_dict, \
-                no_full_nodes, no_full_trigs, tri_map, node_map, tri_l2g, node_l2g, \
-                ghost_layer_width  = \
-                rec_submesh(0, verbose=False)    
-
-        if myid == 1:
-
-
-            true_points =  [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.5], [0.5, 1.0], [0.25, 0.25], [0.25, 0.75], [0.5, 0.0], [1.0, 0.5], [1.0, 1.0], [0.75, 0.25], [0.75, 0.75]] 
-
-            true_vertices =  [[0, 5, 1], [1, 5, 3], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [3, 5, 7], [3, 11, 4], [8, 11, 3], [4, 11, 9], [7, 5, 0], [7, 10, 3]]
-
-            true_ghost_recv =  {0: [num.array([6, 7, 8, 9]), num.array([0, 1, 2, 4])], 2: [num.array([10, 11]), num.array([11, 12])]}
-
-            true_full_send =  {0: [num.array([0, 1, 3, 4, 5]), num.array([ 5,  6,  8,  9, 10])], 2: [num.array([0, 1]), num.array([5, 6])]}
-
-            true_tri_map  = num.array([ 6,  7,  8, -1,  9,  0, 1,  2,  3,  4,  5, 10, 11])
-
-            true_node_map = num.array([ 0,  1,  2,  7,  3,  4, -1,  8,  9,  5,  6, 10, 11])
-
-            assert_(num.allclose(ghost_layer_width,  2))
-            assert_(num.allclose(tri_map,   true_tri_map))
-            assert_(num.allclose(node_map,   true_node_map))
-            assert_(num.allclose(points,   true_points))
-            assert_(num.allclose(vertices, true_vertices))
-            assert_(num.allclose(ghost_recv_dict[0],true_ghost_recv[0]))
-            assert_(num.allclose(ghost_recv_dict[2],true_ghost_recv[2]))
-            assert_(num.allclose(full_send_dict[0],true_full_send[0]))
-            assert_(num.allclose(full_send_dict[2],true_full_send[2])) 
-
-
-        if myid == 2:
-
-            true_points =   [[0.0, 0.0], [0.5, 0.0], [0.5, 0.5], [1.0, 0.0], [1.0, 0.5], [0.25, 0.25], [0.75, 0.25], [0.0, 0.5], [0.5, 1.0], [1.0, 1.0], [0.75, 0.75]]
-
-            true_vertices =  [[1, 5, 0], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [2, 5, 1], [2, 10, 8], [4, 10, 2], [9, 10, 4], [0, 5, 7], [7, 5, 2]]
-
-            true_ghost_recv =   {0: [num.array([5, 6, 7, 8]), num.array([0, 1, 2, 3])], 1: [num.array([ 9, 10]), num.array([5, 6])]}
-
-            true_full_send =   {0: [num.array([0, 1, 2, 3, 4]), num.array([11, 12, 13, 14, 15])], 1: [num.array([0, 1]), num.array([11, 12])]}
-
-            true_tri_map  = num.array([5, 6, 7, 8, -1, 9, 10, -1, -1, -1, -1, 0, 1, 2, 3, 4, -1])
-
-            true_node_map = num.array([ 0,  7, -1,  1,  2,  8 , 3,  4,  9,  5, -1,  6, 10])
-
-            assert_(num.allclose(ghost_layer_width,  2))
-            assert_(num.allclose(tri_map,   true_tri_map))
-            assert_(num.allclose(node_map,   true_node_map))
-            assert_(num.allclose(points,   true_points))
-            assert_(num.allclose(vertices, true_vertices))
-            assert_(num.allclose(ghost_recv_dict[0],true_ghost_recv[0]))
-            assert_(num.allclose(ghost_recv_dict[1],true_ghost_recv[1]))
-            assert_(num.allclose(full_send_dict[0],true_full_send[0]))
-            assert_(num.allclose(full_send_dict[1],true_full_send[1])) 
-
-
-
-
-
-
-###############################################################
-
-class Test_parallel_distribute_mesh(unittest.TestCase):
-
-    def test_distribute_three_processors(self):
-        # Expect this test to fail if not run from the parallel directory.
-        result = os.system("mpirun -np 3 python test_parallel_distribute_mesh.py")
-        assert_(result == 0)
-
-
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #import pypar
-        #pypar.finalize()
-        raise AssertionError, msg
-        #import sys
-        #sys.exit(1)
-    
-
-
-
-
-
-
-#-------------------------------------------------------------
-if __name__=="__main__":
-    if numprocs == 1: 
-        runner = unittest.TextTestRunner()
-        suite = unittest.makeSuite(Test_parallel_distribute_mesh, 'test')
-        runner.run(suite)
-    else:
-        distibute_three_processors()
-
-    finalize()
-        
-
-        
Index: unk/anuga_core/source/anuga_parallel/test_parallel_file_boundary.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_file_boundary.py	(revision 9423)
+++ 	(revision )
@@ -1,505 +1,0 @@
-
-"""Run parallel shallow water domain.
-
-   run using command like:
-
-   mpirun -np m python run_parallel_sw_merimbula.py
-
-   where m is the number of processors to be used.
-   
-   Will produce sww files with names domain_Pn_m.sww where m is number of processors and
-   n in [0, m-1] refers to specific processor that owned this part of the partitioned mesh.
-"""
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-
-import os
-import sys
-import time
-import pypar
-import numpy as num
-import unittest
-import tempfile
-from struct import pack, unpack
-from anuga.file.netcdf import NetCDFFile
-import copy
-
-#------------------------
-# ANUGA Modules
-#------------------------
-from anuga.utilities.numerical_tools import ensure_numeric
-from anuga.utilities.util_ext        import double_precision
-from anuga.utilities.norms           import l1_norm, l2_norm, linf_norm
-	
-from anuga import Domain
-from anuga import Reflective_boundary
-from anuga import Dirichlet_boundary
-from anuga import Time_boundary
-from anuga import Transmissive_boundary
-from anuga import File_boundary
-
-from mux import WAVEHEIGHT_MUX2_LABEL, EAST_VELOCITY_MUX2_LABEL, \
-                NORTH_VELOCITY_MUX2_LABEL
-from mux import read_mux2_py
-from anuga.file_conversion.urs2sts import urs2sts
-from anuga.file.urs import Read_urs
-from anuga.file.sts import create_sts_boundary
-
-from anuga.utilities.numerical_tools import ensure_numeric
-from anuga.coordinate_transforms.redfearn import redfearn
-from anuga.coordinate_transforms.geo_reference import Geo_reference
-
-from anuga import rectangular_cross_domain
-from anuga.pmesh.mesh_interface import create_mesh_from_regions
-from anuga import create_domain_from_file
-
-from anuga_parallel import distribute, myid, numprocs, send, receive, barrier, finalize
-
-from anuga.file.test_mux import Test_Mux
-
-verbose = False
-
-
-class Test_urs2sts_parallel(Test_Mux):
-    """ A suite of tests to test urs2sts file conversion functions.
-        These tests are quite coarse-grained: converting a file
-        and checking that its headers and some of its contents
-        are correct.
-    """ 
-
-    def sequential_test_time_varying_file_boundary_sts(self):
-        """sequential_ltest_time_varying_file_boundary_sts_sequential(self):
-        Read correct points from ordering file and apply sts to boundary. The boundary is time varying. FIXME add to test_urs2sts.
-        """
-        lat_long_points=[[6.01,97.0],[6.02,97.0],[6.05,96.9],[6.0,97.0]]
-        bounding_polygon=[[6.0,97.0],[6.01,97.0],[6.02,97.0],
-                          [6.02,97.02],[6.00,97.02]]
-        tide = 3.0
-        time_step_count = 65
-        time_step = 2.
-        n=len(lat_long_points)
-        first_tstep=num.ones(n,num.int)
-        last_tstep=(time_step_count)*num.ones(n,num.int)
-        finaltime=num.float(time_step*(time_step_count-1))
-        yieldstep=num.float(time_step)
-        gauge_depth=20*num.ones(n,num.float)
-        ha=2*num.ones((n,time_step_count),num.float)
-        ua=10*num.ones((n,time_step_count),num.float)
-        va=-10*num.ones((n,time_step_count),num.float)
-
-        times=num.arange(0., num.float(time_step_count*time_step), time_step)
-        for i in range(n):
-            #ha[i]+=num.sin(times)
-            ha[i]+=times/finaltime
-
-
-
-        sts_file="test"
-        if myid==0:
-            base_name, files = self.write_mux2(lat_long_points,
-                                               time_step_count,
-                                               time_step,
-                                               first_tstep,
-                                               last_tstep,
-                                               depth=gauge_depth,
-                                               ha=ha,
-                                               ua=ua,
-                                               va=va)
-            # base name will not exist, but 3 other files are created
-
-            # Write order file
-            file_handle, order_base_name = tempfile.mkstemp("")
-            os.close(file_handle)
-            os.remove(order_base_name)
-            d=","
-            order_file=order_base_name+'order.txt'
-            fid=open(order_file,'w')
-        
-            # Write Header
-            header='index, longitude, latitude\n'
-            fid.write(header)
-            indices=[3,0,1]
-            for i in indices:
-                line=str(i)+d+str(lat_long_points[i][1])+d+\
-                    str(lat_long_points[i][0])+"\n"
-                fid.write(line)
-            fid.close()
-
-            urs2sts(base_name,
-                    basename_out=sts_file,
-                    ordering_filename=order_file,
-                    mean_stage=tide,
-                    verbose=verbose)
-            self.delete_mux(files)
-
-            assert(os.access(sts_file+'.sts', os.F_OK))
-
-            os.remove(order_file)
-
-        barrier()
-        boundary_polygon = create_sts_boundary(sts_file)
-
-        # Append the remaining part of the boundary polygon to be defined by
-        # the user
-        bounding_polygon_utm=[]
-        for point in bounding_polygon:
-            zone,easting,northing=redfearn(point[0],point[1])
-            bounding_polygon_utm.append([easting,northing])
-
-        boundary_polygon.append(bounding_polygon_utm[3])
-        boundary_polygon.append(bounding_polygon_utm[4])
-
-        assert num.allclose(bounding_polygon_utm,boundary_polygon)
-
-
-        extent_res=1000000
-        meshname = 'urs_test_mesh' + '.tsh'
-        interior_regions=None
-        boundary_tags={'ocean': [0,1], 'otherocean': [2,3,4]}
-        
-        # have to change boundary tags from last example because now bounding
-        # polygon starts in different place.
-        if myid==0:
-            create_mesh_from_regions(boundary_polygon,
-                                     boundary_tags=boundary_tags,
-                                     maximum_triangle_area=extent_res,
-                                     filename=meshname,
-                                     interior_regions=interior_regions,
-                                     verbose=verbose)
-
-        barrier()
-        
-        domain_fbound = Domain(meshname)
-        domain_fbound.set_quantities_to_be_stored(None)
-        domain_fbound.set_quantity('stage', tide)
-        if verbose: print "Creating file boundary condition"
-        Bf = File_boundary(sts_file+'.sts',
-                           domain_fbound,
-                           boundary_polygon=boundary_polygon)
-        Br = Reflective_boundary(domain_fbound)
-
-        domain_fbound.set_boundary({'ocean': Bf,'otherocean': Br})
-
-        temp_fbound=num.zeros(int(finaltime/yieldstep)+1,num.float)
-        if verbose: print "Evolving domain with file boundary condition"
-        for i, t in enumerate(domain_fbound.evolve(yieldstep=yieldstep,
-                                                   finaltime=finaltime, 
-                                                   skip_initial_step = False)):
-            temp_fbound[i]=domain_fbound.quantities['stage'].centroid_values[2]
-            if verbose: domain_fbound.write_time()
-            
-        
-        domain_drchlt = Domain(meshname)
-        domain_drchlt.set_quantities_to_be_stored(None)
-        domain_drchlt.set_starttime(time_step)
-        domain_drchlt.set_quantity('stage', tide)
-        Br = Reflective_boundary(domain_drchlt)
-        #Bd = Dirichlet_boundary([2.0+tide,220+10*tide,-220-10*tide])
-        Bd = Time_boundary(domain=domain_drchlt, f=lambda t: [2.0+t/finaltime+tide,220.+10.*tide+10.*t/finaltime,-220.-10.*tide-10.*t/finaltime])
-        #Bd = Time_boundary(domain=domain_drchlt,f=lambda t: [2.0+num.sin(t)+tide,10.*(2+20.+num.sin(t)+tide),-10.*(2+20.+num.sin(t)+tide)])
-        domain_drchlt.set_boundary({'ocean': Bd,'otherocean': Br})
-        temp_drchlt=num.zeros(int(finaltime/yieldstep)+1,num.float)
-        
-        for i, t in enumerate(domain_drchlt.evolve(yieldstep=yieldstep,
-                                                   finaltime=finaltime, 
-                                                   skip_initial_step = False)):
-            temp_drchlt[i]=domain_drchlt.quantities['stage'].centroid_values[2]
-            #domain_drchlt.write_time()
-        
-        #print domain_fbound.quantities['stage'].vertex_values
-        #print domain_drchlt.quantities['stage'].vertex_values
-                    
-        assert num.allclose(temp_fbound,temp_drchlt),temp_fbound-temp_drchlt
-
-        
-        assert num.allclose(domain_fbound.quantities['stage'].vertex_values,
-                            domain_drchlt.quantities['stage'].vertex_values)
-                        
-        assert num.allclose(domain_fbound.quantities['xmomentum'].vertex_values,
-                            domain_drchlt.quantities['xmomentum'].vertex_values)                        
-                        
-        assert num.allclose(domain_fbound.quantities['ymomentum'].vertex_values,
-                            domain_drchlt.quantities['ymomentum'].vertex_values)
-        
-        if not sys.platform == 'win32':
-            if myid==0: os.remove(sts_file+'.sts')
-        
-        if myid==0: os.remove(meshname)
-
-    def parallel_test_time_varying_file_boundary_sts(self):
-        """ parallel_test_time_varying_file_boundary_sts_sequential(self):
-            Read correct points from ordering file and apply sts to boundary. 
-            The boundary is time varying. Compares sequential result with 
-            distributed result found using anuga_parallel
-        """
-
-        #------------------------------------------------------------
-        # Define test variables
-        #------------------------------------------------------------
-        lat_long_points=[[6.01,97.0],[6.02,97.0],[6.05,96.9],[6.0,97.0]]
-        bounding_polygon=[[6.0,97.0],[6.01,97.0],[6.02,97.0],
-                          [6.02,97.02],[6.00,97.02]]
-        tide = 3.0
-        time_step_count = 65
-        time_step = 2
-        n=len(lat_long_points)
-        first_tstep=num.ones(n,num.int)
-        last_tstep=(time_step_count)*num.ones(n,num.int)
-        finaltime=num.float(time_step*(time_step_count-1))
-        yieldstep=num.float(time_step)
-        gauge_depth=20*num.ones(n,num.float)
-        ha=2*num.ones((n,time_step_count),num.float)
-        ua=10*num.ones((n,time_step_count),num.float)
-        va=-10*num.ones((n,time_step_count),num.float)
-
-        times=num.arange(0, time_step_count*time_step, time_step)
-        for i in range(n):
-            #ha[i]+=num.sin(times)
-            ha[i]+=times/finaltime
-
-        #------------------------------------------------------------
-        # Write mux data to file then convert to sts format
-        #------------------------------------------------------------
-        sts_file="test"
-        if myid==0:
-            base_name, files = self.write_mux2(lat_long_points,
-                                               time_step_count,
-                                               time_step,
-                                               first_tstep,
-                                               last_tstep,
-                                               depth=gauge_depth,
-                                               ha=ha,
-                                               ua=ua,
-                                               va=va)
-            # base name will not exist, but 3 other files are created
-
-            # Write order file
-            file_handle, order_base_name = tempfile.mkstemp("")
-            os.close(file_handle)
-            os.remove(order_base_name)
-            d=","
-            order_file=order_base_name+'order.txt'
-            fid=open(order_file,'w')
-        
-            # Write Header
-            header='index, longitude, latitude\n'
-            fid.write(header)
-            indices=[3,0,1]
-            for i in indices:
-                line=str(i)+d+str(lat_long_points[i][1])+d+\
-                    str(lat_long_points[i][0])+"\n"
-                fid.write(line)
-            fid.close()
-
-            urs2sts(base_name,
-                    basename_out=sts_file,
-                    ordering_filename=order_file,
-                    mean_stage=tide,
-                    verbose=verbose)
-            self.delete_mux(files)
-
-            assert(os.access(sts_file+'.sts', os.F_OK))
-
-            os.remove(order_file)
-
-        barrier()
-        #------------------------------------------------------------
-        # Define boundary_polygon on each processor. This polygon defines the
-        # urs boundary and lies on a portion of the bounding_polygon
-        #------------------------------------------------------------
-        boundary_polygon = create_sts_boundary(sts_file)
-
-        # Append the remaining part of the boundary polygon to be defined by
-        # the user
-        bounding_polygon_utm=[]
-        for point in bounding_polygon:
-            zone,easting,northing=redfearn(point[0],point[1])
-            bounding_polygon_utm.append([easting,northing])
-
-        boundary_polygon.append(bounding_polygon_utm[3])
-        boundary_polygon.append(bounding_polygon_utm[4])
-
-
-        assert num.allclose(bounding_polygon_utm,boundary_polygon)
-
-        extent_res=10000
-        meshname = 'urs_test_mesh' + '.tsh'
-        interior_regions=None
-        boundary_tags={'ocean': [0,1], 'otherocean': [2,3,4]}
-        
-        #------------------------------------------------------------
-        # Create mesh on the master processor and store in file. This file
-        # is read in by each slave processor when needed
-        #------------------------------------------------------------
-        if myid==0:
-            create_mesh_from_regions(boundary_polygon,
-                                     boundary_tags=boundary_tags,
-                                     maximum_triangle_area=extent_res,
-                                     filename=meshname,
-                                     interior_regions=interior_regions,
-                                     verbose=verbose)
-        
-
-            # barrier()
-            domain_fbound = Domain(meshname)
-            domain_fbound.set_quantities_to_be_stored(None)
-            domain_fbound.set_quantity('stage', tide)
-            # print domain_fbound.mesh.get_boundary_polygon()
-        else:
-            domain_fbound=None
-
-        barrier()
-        if ( verbose and myid == 0 ): 
-            print 'DISTRIBUTING PARALLEL DOMAIN'
-        domain_fbound = distribute(domain_fbound)
-
-        #--------------------------------------------------------------------
-        # Find which sub_domain in which the interpolation points are located 
-        #
-        # Sometimes the interpolation points sit exactly
-        # between two centroids, so in the parallel run we
-        # reset the interpolation points to the centroids
-        # found in the sequential run
-        #--------------------------------------------------------------------
-        interpolation_points = [[279000,664000], [280250,664130], 
-                                    [279280,665400], [280500,665000]]
-
-        interpolation_points=num.array(interpolation_points)
-
-        #if myid==0:
-        #    import pylab as P
-        #    boundary_polygon=num.array(boundary_polygon)
-        #    P.plot(boundary_polygon[:,0],boundary_polygon[:,1])
-        #    P.plot(interpolation_points[:,0],interpolation_points[:,1],'ko')
-        #    P.show()
-
-        fbound_gauge_values = []
-        fbound_proc_tri_ids = []
-        for i, point in enumerate(interpolation_points):
-            fbound_gauge_values.append([]) # Empty list for timeseries
-
-            try:
-                k = domain_fbound.get_triangle_containing_point(point)
-                if domain_fbound.tri_full_flag[k] == 1:
-                    fbound_proc_tri_ids.append(k)
-                else:
-                    fbound_proc_tri_ids.append(-1)            
-            except:
-                fbound_proc_tri_ids.append(-2)
-
-
-        if verbose: print 'P%d has points = %s' %(myid, fbound_proc_tri_ids)
-
-        #------------------------------------------------------------
-        # Set boundary conditions
-        #------------------------------------------------------------
-        Bf = File_boundary(sts_file+'.sts',
-                           domain_fbound,
-                           boundary_polygon=boundary_polygon)
-        Br = Reflective_boundary(domain_fbound)
-    
-        domain_fbound.set_boundary({'ocean': Bf,'otherocean': Br})
-
-        #------------------------------------------------------------
-        # Evolve the domain on each processor
-        #------------------------------------------------------------  
-        for i, t in enumerate(domain_fbound.evolve(yieldstep=yieldstep,
-                                                   finaltime=finaltime, 
-                                                   skip_initial_step = False)):
-
-            stage = domain_fbound.get_quantity('stage')
-            for i in range(4):
-                if fbound_proc_tri_ids[i] > -1:
-                    fbound_gauge_values[i].append(stage.centroid_values[fbound_proc_tri_ids[i]])
-        
-        #------------------------------------------------------------
-        # Create domain to be run sequntially on each processor
-        #------------------------------------------------------------
-        domain_drchlt = Domain(meshname)
-        domain_drchlt.set_quantities_to_be_stored(None)
-        domain_drchlt.set_starttime(time_step)
-        domain_drchlt.set_quantity('stage', tide)
-        Br = Reflective_boundary(domain_drchlt)
-        #Bd = Dirichlet_boundary([2.0+tide,220+10*tide,-220-10*tide])
-        Bd = Time_boundary(domain=domain_drchlt, function=lambda t: [2.0+t/finaltime+tide,220.+10.*tide+10.*t/finaltime,-220.-10.*tide-10.*t/finaltime])
-        #Bd = Time_boundary(domain=domain_drchlt,function=lambda t: [2.0+num.sin(t)+tide,10.*(2+20.+num.sin(t)+tide),-10.*(2+20.+num.sin(t)+tide)])
-        domain_drchlt.set_boundary({'ocean': Bd,'otherocean': Br})
-       
-        drchlt_gauge_values = []
-        drchlt_proc_tri_ids = []
-        for i, point in enumerate(interpolation_points):
-            drchlt_gauge_values.append([]) # Empty list for timeseries
-
-            try:
-                k = domain_drchlt.get_triangle_containing_point(point)
-                if domain_drchlt.tri_full_flag[k] == 1:
-                    drchlt_proc_tri_ids.append(k)
-                else:
-                    drchlt_proc_tri_ids.append(-1)            
-            except:
-                drchlt_proc_tri_ids.append(-2)
-
-
-        if verbose: print 'P%d has points = %s' %(myid, drchlt_proc_tri_ids)
-
-        #------------------------------------------------------------
-        # Evolve entire domain on each processor
-        #------------------------------------------------------------
-        for i, t in enumerate(domain_drchlt.evolve(yieldstep=yieldstep,
-                                                   finaltime=finaltime, 
-                                                   skip_initial_step = False)):
-
-            stage = domain_drchlt.get_quantity('stage')
-            for i in range(4):
-                drchlt_gauge_values[i].append(stage.centroid_values[drchlt_proc_tri_ids[i]])
-
-        #------------------------------------------------------------
-        # Compare sequential values with parallel values
-        #------------------------------------------------------------
-        barrier()
-        success = True
-        for i in range(4):
-            if fbound_proc_tri_ids[i] > -1:
-                fbound_gauge_values[i]=num.array(fbound_gauge_values[i])
-                drchlt_gauge_values[i]=num.array(drchlt_gauge_values[i])
-                #print i,fbound_gauge_values[i][4]
-                #print i,drchlt_gauge_values[i][4]
-                success = success and num.allclose(fbound_gauge_values[i], drchlt_gauge_values[i])
-                assert success#, (fbound_gauge_values[i]-drchlt_gauge_values[i])
-
-        #assert_(success)       
-
-        if not sys.platform == 'win32':
-            if myid==0: os.remove(sts_file+'.sts')
-        
-        if myid==0: os.remove(meshname)
-    
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #pypar.finalize()
-        raise AssertionError, msg
-
-
-# Test an nprocs-way run of the shallow water equations
-# against the sequential code.
-
-if __name__=="__main__":
-    #verbose=False
-    if myid ==0 and verbose: 
-        print 'PARALLEL START'
-    suite = unittest.makeSuite(Test_urs2sts_parallel,'parallel_test')
-    #suite = unittest.makeSuite(Test_urs2sts_parallel,'sequential_test')
-    runner = unittest.TextTestRunner()
-    runner.run(suite)
-
-    #------------------------------------------
-    # Run the code code and compare sequential
-    # results at 4 gauge stations
-    #------------------------------------------
-
-    
-    finalize()
Index: unk/anuga_core/source/anuga_parallel/test_parallel_frac_op.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_frac_op.py	(revision 9423)
+++ 	(revision )
@@ -1,338 +1,0 @@
-import os.path
-import sys
-
-from anuga.utilities.system_tools import get_pathname_from_package
-from anuga.geometry.polygon_function import Polygon_function
-        
-from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
-from anuga.abstract_2d_finite_volumes.quantity import Quantity
-from anuga.abstract_2d_finite_volumes.util import file_function
-
-import anuga
-
-import warnings
-warnings.simplefilter("ignore")
-
-
-#------------------------------------------
-# Import pypar without the initial output
-#------------------------------------------
-class NullStream:
-    def write(self,text):
-        pass
-sys.stdout = NullStream()
-import pypar
-sys.stdout = sys.__stdout__
-
-
-from math import pi, pow, sqrt
-
-import numpy as num
-from parallel_inlet_operator import Parallel_Inlet_operator
-from anuga_parallel import distribute, myid, numprocs, finalize
-from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
-
-from anuga import Inlet_operator, Boyd_box_operator
-
-import random
-import unittest
-
-
-"""
-
-This test exercises the parallel culvert and checks values
-"""
-
-verbose = False
-nprocs = 3
-    
-
-length = 40.
-width = 16.
-
-dx = dy = 2           # Resolution: Length of subdivisions on both axes
-
-#----------------------------------------------------------------------
-# Setup initial conditions
-#----------------------------------------------------------------------
-
-def topography(x, y):
-    """Set up a weir
-    
-    A culvert will connect either side
-    """
-    # General Slope of Topography
-    z=-x/1000
-    
-    N = len(x)
-    for i in range(N):
-
-        # Sloping Embankment Across Channel
-        if 5.0 < x[i] < 10.1:
-            # Cut Out Segment for Culvert face                
-            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
-                z[i]=z[i]
-            else:
-                z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
-        if 10.0 < x[i] < 12.1:
-            z[i] +=  2.5                    # Flat Crest of Embankment
-        if 12.0 < x[i] < 14.5:
-            # Cut Out Segment for Culvert face                
-            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
-                z[i]=z[i]
-            else:
-                z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
-                   
-    return z
-
-#filename=os.path.join(path, 'example_rating_curve.csv')
-
-line0 = [[10.0, 10.0], [30.0, 10.0]]
-#line0 = [[29.0, 10.0], [30.0, 10.0]]
-line1 = [[0.0, 10.0], [0.0, 15.0]]
-Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
-Q1 = 5.0
-
-samples = 50
-
-def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
-    success = True
-
-##-----------------------------------------------------------------------
-## Setup domain
-##-----------------------------------------------------------------------
-
-    points, vertices, boundary = rectangular_cross(int(length/dx),
-                                                   int(width/dy),
-                                                   len1=length, 
-                                                   len2=width)
-
-    domain = anuga.Domain(points, vertices, boundary)   
-    domain.set_name('Test_Parallel_Frac_Op')                 # Output name
-    domain.set_default_order(2)
-
-##-----------------------------------------------------------------------
-## Distribute domain
-##-----------------------------------------------------------------------
-
-    if parallel:
-        domain = distribute(domain)
-        domain.dump_triangulation("frac_op_domain.png")
-    
-
-##-----------------------------------------------------------------------
-## Setup boundary conditions
-##-----------------------------------------------------------------------
-    
-    domain.set_quantity('elevation', topography) 
-    domain.set_quantity('friction', 0.01)         # Constant friction 
-    domain.set_quantity('stage',
-                        expression='elevation')   # Dry initial condition
-
-    
-    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
-    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
-    domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
-
-
-##-----------------------------------------------------------------------
-## Determine triangle index coinciding with test points
-##-----------------------------------------------------------------------
-
-    assert(test_points is not None)
-    assert(len(test_points) == samples)
-
-    tri_ids = []
-
-    for point in test_points:
-        try:
-            k = domain.get_triangle_containing_point(point)
-            if domain.tri_full_flag[k] == 1:
-                tri_ids.append(k)
-            else:
-                tri_ids.append(-1)            
-        except:
-            tri_ids.append(-2)
-
-    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
-
-    if not parallel: control_data = []
-
-    ################ Define Fractional Operators ##########################
-
-    inlet0 = None
-    inlet1 = None
-    boyd_box0 = None
-    
-    inlet0 = Inlet_operator(domain, line0, Q0, logging=True, description='inlet0', verbose = False)
-    inlet1 = Inlet_operator(domain, line1, Q1, logging=True, description='inlet1', verbose = False)
-    
-    # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
-    
-    boyd_box0 = Boyd_box_operator(domain,
-                                  end_points=[[9.0, 2.5],[19.0, 2.5]],
-                                  losses=1.5,
-                                  width=5.0,
-                                  #apron=5.0,
-                                  use_momentum_jet=True,
-                                  use_velocity_head=False,
-                                  manning=0.013,
-                                  logging=True,
-                                  description='boyd_box_0',
-                                  verbose=False)
-        
-    if inlet0 is not None and verbose: inlet0.print_statistics()
-    if inlet1 is not None and verbose: inlet1.print_statistics()
-    if boyd_box0 is not None and verbose: boyd_box0.print_statistics()
-
-
-    ##-----------------------------------------------------------------------
-    ## Evolve system through time
-    ##-----------------------------------------------------------------------
-
-    for t in domain.evolve(yieldstep = 2.0, finaltime = 20.0):
-        if myid == 0 and verbose:
-            domain.write_time()
-
-        #print domain.volumetric_balance_statistics()
-    
-        stage = domain.get_quantity('stage')
-
-
-        if boyd_box0 is not None and verbose : boyd_box0.print_timestepping_statistics()
- 
-        #for i in range(samples):
-        #    if tri_ids[i] >= 0:                
-        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
-                    
-        sys.stdout.flush()
- 
-        pass
-
-    success = True
-
-##-----------------------------------------------------------------------
-## Assign/Test Control data
-##-----------------------------------------------------------------------
-
-    if not parallel:
-        stage = domain.get_quantity('stage')
-
-        for i in range(samples):
-            assert(tri_ids[i] >= 0)
-            control_data.append(stage.centroid_values[tri_ids[i]])
-        
-        if inlet0 is not None:
-            control_data.append(inlet0.inlet.get_average_stage())
-            control_data.append(inlet0.inlet.get_average_xmom())
-            control_data.append(inlet0.inlet.get_average_ymom())
-            control_data.append(inlet0.inlet.get_total_water_volume())
-            control_data.append(inlet0.inlet.get_average_depth())
-
-        if verbose: print 'P%d control_data = %s' %(myid, control_data)
-    else:
-        stage = domain.get_quantity('stage')
-        
-        for i in range(samples):
-            if tri_ids[i] >= 0:
-                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
-                success = success and local_success
-                if verbose: 
-                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
-                if not local_success:
-                    print 'Ouput P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
-
-
-        #assert success        
-                
-        if inlet0 is not None:
-            inlet_master_proc = inlet0.inlet.get_master_proc()
-            average_stage = inlet0.inlet.get_global_average_stage()
-            average_xmom = inlet0.inlet.get_global_average_xmom()
-            average_ymom = inlet0.inlet.get_global_average_ymom()
-            average_volume = inlet0.inlet.get_global_total_water_volume()
-            average_depth = inlet0.inlet.get_global_average_depth()
-
-            if myid == inlet_master_proc:
-                if verbose: 
-                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
-
-                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
-
-                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
-
-                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
-
-                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
-
-
-        #assert(success)
-
-    return control_data, success
-
-
-# Test an nprocs-way run of the shallow water equations
-# against the sequential code.
-
-class Test_parallel_frac_op(unittest.TestCase):
-    def test_parallel_frac_op(self):
-        #print "Expect this test to fail if not run from the parallel directory."
-        result = os.system("mpirun -np %d python test_parallel_frac_op.py" % nprocs)
-        assert_(result == 0)
-
-
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #pypar.finalize()
-        raise AssertionError, msg
-
-if __name__=="__main__":
-
-    if numprocs == 1:
-        runner = unittest.TextTestRunner()
-        suite = unittest.makeSuite(Test_parallel_frac_op, 'test')
-        #print "Running for numproc = 1"
-        runner.run(suite)
-    else:
-        #print "Running for numproc > 1"
-        pypar.barrier()
-        test_points = []
-
-        if myid == 0:
-            if verbose: print 'PARALLEL START'
-            random.seed(2)
-            for i in range(samples):
-                x = random.randrange(0,1000)/1000.0 * length
-                y = random.randrange(0,1000)/1000.0 * width
-                point = [x, y]
-                test_points.append(point)
-
-            for i in range(1,numprocs):
-                pypar.send(test_points, i)
-        else:
-            test_points = pypar.receive(0)
-
-        if myid == 0:
-            control_data, success  = run_test(parallel=False, test_points = test_points, verbose = verbose)
-
-            for proc in range(1,numprocs):
-                pypar.send(control_data, proc)
-        else:
-            control_data = pypar.receive(0)
-
-
-        pypar.barrier()
-        _, success = run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = verbose)
-
-        sys.stdout.flush()
-
-        pypar.barrier()
-
-        msg = 'Discrepency between sequential and parallel runs on P%g' % myid
-        assert success, msg
-
-    finalize()
-    
-
Index: unk/anuga_core/source/anuga_parallel/test_parallel_inlet_operator.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_inlet_operator.py	(revision 9423)
+++ 	(revision )
@@ -1,318 +1,0 @@
-import os.path
-import sys
-
-from anuga.utilities.system_tools import get_pathname_from_package
-from anuga.geometry.polygon_function import Polygon_function
-        
-from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
-from anuga.abstract_2d_finite_volumes.quantity import Quantity
-from anuga.abstract_2d_finite_volumes.util import file_function
-
-import anuga
-
-#from anuga.structures.boyd_box_operator import Boyd_box_operator
-#from anuga.structures.inlet_operator import Inlet_operator
-                            
-#from anuga.culvert_flows.culvert_routines import boyd_generalised_culvert_model
-
-
-import warnings
-warnings.simplefilter('ignore')
-
-#------------------------------------------
-# Import pypar without the initial output
-#------------------------------------------
-class NullStream:
-    def write(self,text):
-        pass
-sys.stdout = NullStream()
-import pypar
-sys.stdout = sys.__stdout__
-
-
-from math import pi, pow, sqrt
-
-import numpy as num
-from parallel_inlet_operator import Parallel_Inlet_operator
-from anuga_parallel import distribute, myid, numprocs, finalize
-from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
-
-from parallel_operator_factory import Inlet_operator
-
-import random
-import unittest
-
-
-"""
-
-This test exercises the parallel culvert and checks values
-"""
-verbose = False
-nprocs = 3
-    
-
-length = 40.
-width = 16.
-
-dx = dy = 2.0          # Resolution: Length of subdivisions on both axes
-
-#----------------------------------------------------------------------
-# Setup initial conditions
-#----------------------------------------------------------------------
-
-def topography(x, y):
-    """Set up a weir
-    
-    A culvert will connect either side
-    """
-    # General Slope of Topography
-    z=-x/1000
-    
-    N = len(x)
-    for i in range(N):
-
-        # Sloping Embankment Across Channel
-        if 5.0 < x[i] < 10.1:
-            # Cut Out Segment for Culvert face                
-            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
-                z[i]=z[i]
-            else:
-                z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
-        if 10.0 < x[i] < 12.1:
-            z[i] +=  2.5                    # Flat Crest of Embankment
-        if 12.0 < x[i] < 14.5:
-            # Cut Out Segment for Culvert face                
-            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
-                z[i]=z[i]
-            else:
-                z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
-                   
-        
-    return z
-
-#filename=os.path.join(path, 'example_rating_curve.csv')
-
-line0 = [[10.0, 10.0], [30.0, 10.0]]
-#line0 = [[29.0, 10.0], [30.0, 10.0]]
-line1 = [[0.0, 10.0], [0.0, 15.0]]
-Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
-Q1 = 5.0
-
-samples = 50
-
-def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
-    success = True
-
-    ##-----------------------------------------------------------------------
-    ## Setup domain
-    ##-----------------------------------------------------------------------
-
-    points, vertices, boundary = rectangular_cross(int(length/dx),
-                                                   int(width/dy),
-                                                   len1=length, 
-                                                   len2=width)
-
-    domain = anuga.Domain(points, vertices, boundary)   
-    domain.set_name('Test_Parallel_Frac_Op')                 # Output name
-    domain.set_default_order(2)
-
-    ##-----------------------------------------------------------------------
-    ## Distribute domain
-    ##-----------------------------------------------------------------------
-
-    if parallel:
-        domain = distribute(domain)
-        domain.dump_triangulation("frac_op_domain.png")
-    
-
-    ##-----------------------------------------------------------------------
-    ## Setup boundary conditions
-    ##-----------------------------------------------------------------------
-    
-    domain.set_quantity('elevation', topography) 
-    domain.set_quantity('friction', 0.01)         # Constant friction 
-    domain.set_quantity('stage',
-                        expression='elevation')   # Dry initial condition
-
-    
-    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
-    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
-    domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
-
-
-    ##-----------------------------------------------------------------------
-    ## Determine triangle index coinciding with test points
-    ##-----------------------------------------------------------------------
-
-    assert(test_points is not None)
-    assert(len(test_points) == samples)
-
-    tri_ids = []
-
-    for point in test_points:
-        try:
-            k = domain.get_triangle_containing_point(point)
-            if domain.tri_full_flag[k] == 1:
-                tri_ids.append(k)
-            else:
-                tri_ids.append(-1)            
-        except:
-            tri_ids.append(-2)
-
-    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
-
-    if not parallel: control_data = []
-
-    ################ Define Fractional Operators ##########################
-
-    inlet0 = None
-    inlet1 = None
-    boyd_box0 = None
-    
-    inlet0 = Inlet_operator(domain, line0, Q0, verbose = False, default=0.0)
-    inlet1 = Inlet_operator(domain, line1, Q1, verbose = False)
-    
-        
-    if inlet0 is not None and verbose: inlet0.print_statistics()
-    if inlet1 is not None and verbose: inlet1.print_statistics()
-    #if boyd_box0 is not None and verbose: boyd_box0.print_statistics()
-
-
-    ##-----------------------------------------------------------------------
-    ## Evolve system through time
-    ##-----------------------------------------------------------------------
-
-    for t in domain.evolve(yieldstep = 2.0, finaltime = 40.0):
-        if myid == 0 and verbose:
-            domain.write_time()
-
-        #print domain.volumetric_balance_statistics()
-    
-        stage = domain.get_quantity('stage')
-
-
-        #if boyd_box0 is not None and verbose : boyd_box0.print_timestepping_statistics()
- 
-        #for i in range(samples):
-        #    if tri_ids[i] >= 0:                
-        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
-                    
-        sys.stdout.flush()
- 
-        pass
-
-    success = True
-
-    ##-----------------------------------------------------------------------
-    ## Assign/Test Control data
-    ##-----------------------------------------------------------------------
-
-    if not parallel:
-        stage = domain.get_quantity('stage')
-
-        for i in range(samples):
-            assert(tri_ids[i] >= 0)
-            control_data.append(stage.centroid_values[tri_ids[i]])
-        
-        if inlet0 is not None:
-            control_data.append(inlet0.inlet.get_average_stage())
-            control_data.append(inlet0.inlet.get_average_xmom())
-            control_data.append(inlet0.inlet.get_average_ymom())
-            control_data.append(inlet0.inlet.get_total_water_volume())
-            control_data.append(inlet0.inlet.get_average_depth())
-
-        if verbose: print 'P%d control_data = %s' %(myid, control_data)
-    else:
-        stage = domain.get_quantity('stage')
-        
-        for i in range(samples):
-            if tri_ids[i] >= 0:
-                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
-                success = success and local_success
-                if verbose: 
-                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
-                
-                
-        if inlet0 is not None:
-            inlet_master_proc = inlet0.inlet.get_master_proc()
-            average_stage = inlet0.inlet.get_global_average_stage()
-            average_xmom = inlet0.inlet.get_global_average_xmom()
-            average_ymom = inlet0.inlet.get_global_average_ymom()
-            average_volume = inlet0.inlet.get_global_total_water_volume()
-            average_depth = inlet0.inlet.get_global_average_depth()
-
-            if myid == inlet_master_proc:
-                if verbose: 
-                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
-
-                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
-
-                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
-
-                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
-
-                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
-
-
-        assert(success)
-
-    return control_data
-
-
-# Test an nprocs-way run of the shallow water equations
-# against the sequential code.
-
-class Test_parallel_frac_op(unittest.TestCase):
-    def test_parallel_frac_op(self):
-        #print "Expect this test to fail if not run from the parallel directory."
-        result = os.system("mpirun -np %d python test_parallel_inlet_operator.py" % nprocs)
-        assert_(result == 0)
-
-
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #pypar.finalize()
-        raise AssertionError, msg
-
-if __name__=="__main__":
-    if numprocs == 1:
-        runner = unittest.TextTestRunner()
-        suite = unittest.makeSuite(Test_parallel_frac_op, 'test')
-        #print "Running for numproc = 1"
-        runner.run(suite)
-    else:
-        #print "Running for numproc > 1"
-        pypar.barrier()
-        test_points = []
-
-        if myid == 0:
-            if verbose: print 'PARALLEL START'
-            for i in range(samples):
-                x = random.randrange(0,1000)/1000.0 * length
-                y = random.randrange(0,1000)/1000.0 * width
-                point = [x, y]
-                test_points.append(point)
-
-            for i in range(1,numprocs):
-                pypar.send(test_points, i)
-        else:
-            test_points = pypar.receive(0)
-
-        if myid == 0:
-            control_data = run_test(parallel=False, test_points = test_points, verbose = verbose)
-
-            for proc in range(1,numprocs):
-                pypar.send(control_data, proc)
-        else:
-            control_data = pypar.receive(0)
-
-
-        pypar.barrier()
-        run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = verbose)
-
-
-    finalize()
-    
-
Index: unk/anuga_core/source/anuga_parallel/test_parallel_riverwall.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_riverwall.py	(revision 9423)
+++ 	(revision )
@@ -1,248 +1,0 @@
-"""
-Test parallel and sequential results of riverwall procedure
-"""
-
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-import unittest
-import os
-import sys
-import anuga
-import numpy as num
-
-from anuga import myid, finalize, distribute, barrier, numprocs
-
-import anuga.utilities.plot_utils as util
-
-from math import exp
-
-
-alg = 'DE0'
-verbose = False
-
-scale_me=1.0
-nprocs = 3
-
-## Set up mesh
-boundaryPolygon=[ [0., 0.], [0., 100.], [100.0, 100.0], [100.0, 0.0]]
-midResPolygon=[ [30., 30.], [30., 70.], [70., 70.], [70., 30.]]
-higherResPolygon=[ [40., 40.], [40., 60.], [60., 60.], [60., 40.]]
-# Riverwall = list of lists, each with a set of x,y,z (and optional QFactor) values
-riverWall={ 'centralWall':
-                [ [50., 0.0, -0.0],
-                  [50., 45., -0.0],
-                  [50., 46., -0.2],
-                  [50., 54., -0.2],
-                  [50., 55., -0.0], 
-                  [50., 100.0, -0.0]] 
-          }
-
-riverWall_Par={'centralWall':{'Qfactor':1.0}}
-# Try to avoid any shallow-water type solution -- becomes unstable
-#riverWall_Par={'centralWall':{'Qfactor':1.0, 's1': 0.999, 's2':0.9999, 'h1':100, 'h2':150}}
-
-# The boundary polygon + riverwall breaks the mesh into multiple regions
-# Must define the resolution in these areas with an xy point + maximum area
-# Otherwise triangle.c gets confused
-regionPtAreas=[ [99., 99., 10.0*10.0*0.5],
-                [1., 1., 10.0*10.0*0.5],
-                [45., 45., 1.0*1.0*0.5],
-                [55., 55., 1.0*1.0*0.5],
-                [65., 65., 3.0*3.0*0.5],
-                [35., 35., 3.0*3.0*0.5] ]
-
-
-
-
-  
-
-###########################################################################
-# Setup Test
-##########################################################################
-def evolution_test(parallel=False, verbose=False):
-
-    #--------------------------------------------------------------------------
-    # Setup computational domain and quantities
-    #--------------------------------------------------------------------------
-    if myid == 0:
-        anuga.create_mesh_from_regions(boundaryPolygon, 
-                                 boundary_tags={'left': [0],
-                                                'top': [1],
-                                                'right': [2],
-                                                'bottom': [3]},
-                                   maximum_triangle_area = 1.0e+20,
-                                   minimum_triangle_angle = 28.0,
-                                   filename = 'runup.msh',
-                                   interior_regions = [ [higherResPolygon, 1.*1.*0.5],
-                                                        [midResPolygon, 3.0*3.0*0.5]],
-                                   breaklines=riverWall.values(),
-                                   use_cache=False,
-                                   verbose=verbose,
-                                   regionPtArea=regionPtAreas)
-        
-        sdomain=anuga.create_domain_from_file('runup.msh')
-        
-        
-        sdomain.set_flow_algorithm(alg)
-        
-        
-        sdomain.set_name('s_riverwall')                         
-        sdomain.set_datadir('.')                         
-        sdomain.set_store_vertices_uniquely()
-        
-        #------------------
-        # Define topography
-        #------------------
-        
-        def topography(x,y):
-            return -x/150.*scale_me 
-        
-        def stagefun(x,y):
-            stg=-0.5*scale_me
-            return stg 
-        
-        sdomain.set_quantity('elevation',topography)     # Use function for elevation
-        sdomain.set_quantity('friction',0.03)             # Constant friction
-        sdomain.set_quantity('stage', stagefun)              # Constant negative initial stage
-    else:
-        sdomain = None
-        
-    #--------------------------------------------------------------------------
-    # Create the parallel domains
-    #--------------------------------------------------------------------------
-    if parallel:
-        
-        if myid == 0 and verbose : print 'DISTRIBUTING TO PARALLEL DOMAIN'
-        pdomain = distribute(sdomain, verbose=verbose)
-        pdomain.set_name('p_riverwall')
-        pdomain.set_store_vertices_uniquely()
-        
-        
-    if myid == 0 and verbose: 
-        print 60*'='
-        print 'EVOLVING pdomain'
-        print 60*'='
-            
-    setup_and_evolve(pdomain, verbose=verbose)
- 
-    barrier()
-   
-    if myid == 0:
-        if verbose: 
-            print 60*'='
-            print 'EVOLVING sdomain'
-            print 60*'='  
-        setup_and_evolve(sdomain, verbose=verbose)
-      
-    barrier()
-    
-    #---------------------------------
-    # Now compare the merged sww files
-    #---------------------------------
-    if myid == 0:
-        if verbose: print 'COMPARING SWW FILES'
-        
-        sdomain_v = util.get_output('s_riverwall.sww')
-        sdomain_c = util.get_centroids(sdomain_v)
-
-        pdomain_v = util.get_output('p_riverwall.sww')
-        pdomain_c = util.get_centroids(pdomain_v)
-        
-
-        # Test some values against the original ordering
-        
-        if verbose:
-            
-            order = 0
-            print 'PDOMAIN CENTROID VALUES'
-            print num.linalg.norm(sdomain_c.x-pdomain_c.x,ord=order)
-            print num.linalg.norm(sdomain_c.y-pdomain_c.y,ord=order)
-            print num.linalg.norm(sdomain_c.stage[-1]-pdomain_c.stage[-1],ord=order)
-            print num.linalg.norm(sdomain_c.xmom[-1]-pdomain_c.xmom[-1],ord=order)
-            print num.linalg.norm(sdomain_c.ymom[-1]-pdomain_c.ymom[-1],ord=order)
-            print num.linalg.norm(sdomain_c.xvel[-1]-pdomain_c.xvel[-1],ord=order)
-            print num.linalg.norm(sdomain_c.yvel[-1]-pdomain_c.yvel[-1],ord=order)        
-            
-        assert num.allclose(sdomain_c.stage,pdomain_c.stage)
-        assert num.allclose(sdomain_c.xmom,pdomain_c.xmom)
-        assert num.allclose(sdomain_c.ymom,pdomain_c.ymom)
-        assert num.allclose(sdomain_c.xvel,pdomain_c.xvel)
-        assert num.allclose(sdomain_c.yvel,pdomain_c.yvel)
-        
-        assert num.allclose(sdomain_v.x,pdomain_v.x)
-        assert num.allclose(sdomain_v.y,pdomain_v.y)
-                  
- 
-        
-        
-def setup_and_evolve(domain, verbose=False):
-    
-
-    domain.riverwallData.create_riverwalls(riverWall, riverWall_Par, verbose=verbose)
-    
-    #--------------------------
-    # Setup boundary conditions
-    #--------------------------
-    Br=anuga.Reflective_boundary(domain)            # Solid reflective wall
-    
-    def boundaryFun(t):
-        output=-0.4*exp(-t/100.)-0.1
-        output=min(output,-0.11)
-        #output=min(output,-0.3)
-        return output  
-    
-
-    Bin_tmss = anuga.Transmissive_momentum_set_stage_boundary(domain=domain, function = boundaryFun) 
-    
-    #----------------------------------------------
-    # Associate boundary tags with boundary objects
-    #----------------------------------------------
-    domain.set_boundary({'left': Br, 'right': Bin_tmss, 'top': Br, 'bottom':Br})
-    
-
-    
-    #------------------------------
-    #Evolve the system through time
-    #------------------------------
-    if verbose: print 'Evolve'
-    for t in domain.evolve(yieldstep=10.0,finaltime=150.0):
-        if myid == 0 and verbose: print domain.timestepping_statistics()
-    
-    domain.sww_merge(delete_old=True)
-
-    
-
-# Test an nprocs-way run of the shallow water equations
-# against the sequential code.
-
-class Test_parallel_riverwall(unittest.TestCase):
-    def test_parallel_riverwall(self):
-        if verbose : print "Expect this test to fail if not run from the parallel directory."
-        result = os.system("mpirun -np %d python test_parallel_riverwall.py" % nprocs)
-        assert_(result == 0)
-
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #pypar.finalize()
-        raise AssertionError, msg
-
-if __name__=="__main__":
-    if numprocs == 1: 
-        runner = unittest.TextTestRunner()
-        suite = unittest.makeSuite(Test_parallel_riverwall, 'test')
-        runner.run(suite)
-    else:
-
-        #------------------------------------------
-        # Run the code code and compare sequential
-        # and parallel values
-        #------------------------------------------
-        if myid ==0 and verbose: print 'PARALLEL START'
-        evolution_test(parallel=True, verbose=verbose)
-        finalize()
-
-
Index: unk/anuga_core/source/anuga_parallel/test_parallel_shallow_domain.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_shallow_domain.py	(revision 9423)
+++ 	(revision )
@@ -1,156 +1,0 @@
-
-"""Test a run of the sequential shallow water domain against
-a run of the parallel shallow water domain.
-
-WARNING: This assumes that the command to run jobs is mpirun.
-Tested with MPICH and LAM (Ole)
-"""
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-
-import unittest
-import os
-import sys
-
-
-
-import numpy as num
-
-
-
-#------------------------------------------
-# Import pypar without the initial output
-#------------------------------------------
-class NullStream:
-    def write(self,text):
-        pass
-sys.stdout = NullStream()
-import pypar
-sys.stdout = sys.__stdout__
-
-
-#------------------------------------------
-# anuga imports
-#------------------------------------------
-
-from anuga.utilities.numerical_tools import ensure_numeric
-from anuga.utilities.util_ext        import double_precision
-from anuga.utilities.norms           import l1_norm, l2_norm, linf_norm
-
-from anuga import Domain
-from anuga import Reflective_boundary
-from anuga import Dirichlet_boundary
-from anuga import Time_boundary
-from anuga import Transmissive_boundary
-
-from anuga import rectangular_cross
-from anuga import create_domain_from_file
-
-
-from anuga_parallel import distribute, myid, numprocs, finalize
-
-
-#--------------------------------------------------------------------------
-# Setup parameters
-#--------------------------------------------------------------------------
-
-mesh_filename = "merimbula_10785_1.tsh"
-#mesh_filename = "test-100.tsh"
-yieldstep = 1
-finaltime = 1
-quantity = 'stage'
-nprocs = 2
-verbose = False
-
-#--------------------------------------------------------------------------
-# Setup procedures
-#--------------------------------------------------------------------------
-class Set_Stage:
-    """Set an initial condition with constant water height, for x<x0
-    """
-
-    def __init__(self, x0=0.25, x1=0.5, h=1.0):
-        self.x0 = x0
-        self.x1 = x1
-        self.h  = h
-
-    def __call__(self, x, y):
-        return self.h*((x>self.x0)&(x<self.x1))
-
-#--------------------------------------------------------------------------
-# Setup test
-#--------------------------------------------------------------------------
-def evolution_test(parallel=False):
-
-
-    domain = create_domain_from_file(mesh_filename)
-    domain.set_quantity('stage', Set_Stage(756000.0, 756500.0, 2.0))
-
-    #--------------------------------------------------------------------------
-    # Create parallel domain if requested
-    #--------------------------------------------------------------------------
-
-    if parallel:
-        if myid == 0 and verbose: print 'DISTRIBUTING PARALLEL DOMAIN'
-        domain = distribute(domain)
-
-    #------------------------------------------------------------------------------
-    # Setup boundary conditions
-    # This must currently happen *after* domain has been distributed
-    #------------------------------------------------------------------------------
-    domain.store = False
-    Br = Reflective_boundary(domain)      # Solid reflective wall
-
-    domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Br})
-
-    #------------------------------------------------------------------------------
-    # Evolution
-    #------------------------------------------------------------------------------
-    if parallel:
-        if myid == 0 and verbose: print 'PARALLEL EVOLVE'
-    else:
-        if verbose: print 'SEQUENTIAL EVOLVE'
-
-    for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-        pass
-
-    domain.dump_triangulation()
-
-# Test an nprocs-way run of the shallow water equations
-# against the sequential code.
-
-class Test_parallel_shallow_domain(unittest.TestCase):
-    def test_parallel_shallow_domain(self):
-        #print "Expect this test to fail if not run from the parallel directory."
-        result = os.system("mpirun -np %d python test_parallel_shallow_domain.py" % nprocs)
-        assert_(result == 0)
-
-
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #pypar.finalize()
-        raise AssertionError, msg
-
-if __name__=="__main__":
-    if numprocs == 1: 
-        runner = unittest.TextTestRunner()
-        suite = unittest.makeSuite(Test_parallel_shallow_domain, 'test')
-        runner.run(suite)
-    else:
-
-        pypar.barrier()
-        if myid ==0:
-            if verbose: print 'PARALLEL START'
-
-        evolution_test(parallel=True)
-        
-        if myid == 0:     
-            if verbose: print 'Parallel test OK'
-
-
-
-    finalize()
Index: unk/anuga_core/source/anuga_parallel/test_parallel_sw_flow.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_sw_flow.py	(revision 9423)
+++ 	(revision )
@@ -1,218 +1,0 @@
-"""
-Simple water flow example using ANUGA
-
-Water driven up a linear slope and time varying boundary,
-similar to a beach environment
-
-This is a very simple test of the parallel algorithm using the simplified parallel API
-"""
-
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-import unittest
-import os
-import sys
-#import pypar
-import numpy as num
-
-
-
-from anuga import Domain
-from anuga import Reflective_boundary
-from anuga import Dirichlet_boundary
-from anuga import Time_boundary
-from anuga import Transmissive_boundary
-
-from anuga import rectangular_cross_domain
-
-
-from anuga_parallel import distribute, myid, numprocs, send, receive, barrier, finalize
-
-#--------------------------------------------------------------------------
-# Setup parameters
-#--------------------------------------------------------------------------
-yieldstep = 0.25
-finaltime = 6.0
-nprocs = 4
-N = 29
-M = 29 
-verbose = False
-
-#---------------------------------
-# Setup Functions
-#---------------------------------
-def topography(x,y): 
-    return -x/2    
-
-###########################################################################
-# Setup Test
-##########################################################################
-def evolution_test(parallel=False, G = None, seq_interpolation_points=None, verbose=False):
-
-    #--------------------------------------------------------------------------
-    # Setup computational domain and quantities
-    #--------------------------------------------------------------------------
-    domain = rectangular_cross_domain(M, N)
-    domain.set_quantity('elevation', topography) # Use function for elevation
-    domain.set_quantity('friction', 0.0)         # Constant friction 
-    domain.set_quantity('stage', expression='elevation') # Dry initial stage
-
-    #--------------------------------------------------------------------------
-    # Create the parallel domain
-    #--------------------------------------------------------------------------
-    if parallel:
-        if myid == 0 and verbose : print 'DISTRIBUTING PARALLEL DOMAIN'
-        domain = distribute(domain, verbose=False)
-
-    #--------------------------------------------------------------------------
-    # Setup domain parameters
-    #--------------------------------------------------------------------------
-    domain.set_name('runup')                    # Set sww filename
-    domain.set_datadir('.')                     # Set output dir
-
-    domain.set_flow_algorithm('2_0')
-    domain.set_quantities_to_be_stored(None)
-
-
-    #------------------------------------------------------------------------------
-    # Setup boundary conditions
-    # This must currently happen *AFTER* domain has been distributed
-    #------------------------------------------------------------------------------
-
-    Br = Reflective_boundary(domain)      # Solid reflective wall
-    Bd = Dirichlet_boundary([-0.2,0.,0.]) # Constant boundary values
-
-    # Associate boundary tags with boundary objects
-    domain.set_boundary({'left': Br, 'right': Bd, 'top': Br, 'bottom': Br})
-
-    #------------------------------------------------------------------------------
-    # Find which sub_domain in which the interpolation points are located 
-    #
-    # Sometimes the interpolation points sit exactly
-    # between two centroids, so in the parallel run we
-    # reset the interpolation points to the centroids
-    # found in the sequential run
-    #------------------------------------------------------------------------------
-    interpolation_points = [[0.4,0.5], [0.6,0.5], [0.8,0.5], [0.9,0.5]]
-
-
-    gauge_values = []
-    tri_ids = []
-    for i, point in enumerate(interpolation_points):
-        gauge_values.append([]) # Empty list for timeseries
-
-        #if is_inside_polygon(point, domain.get_boundary_polygon()):
-        #print "Point ", myid, i, point
-        try:
-            k = domain.get_triangle_containing_point(point)
-            if domain.tri_full_flag[k] == 1:
-                tri_ids.append(k)
-            else:
-                tri_ids.append(-1)            
-        except:
-            tri_ids.append(-2)
-
-        #print "  tri_ids ",myid, i, tri_ids[-1]
-		
-    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
-
-
-    c_coord = domain.get_centroid_coordinates()
-    interpolation_points = []
-    for id in tri_ids:
-        if id<1:
-            if verbose: print 'WARNING: Interpolation point not within the domain!'
-        interpolation_points.append(c_coord[id,:])
-            
-    #------------------------------------------------------------------------------
-    # Evolve system through time
-    #------------------------------------------------------------------------------
-    time = []
-
-    if parallel:
-        if myid == 0 and verbose: print 'PARALLEL EVOLVE'
-    else:
-        if myid == 0 and verbose: print 'SEQUENTIAL EVOLVE'
-    
-
-    for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-        if myid == 0 and verbose : domain.write_time()
-
-        # Record time series at known points
-        time.append(domain.get_time())
-
-        stage = domain.get_quantity('stage')
-
-        for i in range(4):
-            if tri_ids[i] > -1:
-                gauge_values[i].append(stage.centroid_values[tri_ids[i]])
-
-
-    #----------------------------------------
-    # Setup test arrays during sequential run
-    #----------------------------------------
-    if not parallel:
-        G = []
-        for i in range(4):
-            G.append(gauge_values[i])
-
-    success = True
-
-    for i in range(4):
-        if tri_ids[i] > -1:
-            #print num.max(num.array(gauge_values[i])- num.array(G[i]))
-            success = success and num.allclose(gauge_values[i], G[i])
-
-    assert_(success)
-    
-    return G, interpolation_points
-
-# Test an nprocs-way run of the shallow water equations
-# against the sequential code.
-
-class Test_parallel_sw_flow(unittest.TestCase):
-    def test_parallel_sw_flow(self):
-        if verbose : print "Expect this test to fail if not run from the parallel directory."
-        result = os.system("mpirun -np %d python test_parallel_sw_flow.py" % nprocs)
-        assert_(result == 0)
-
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #pypar.finalize()
-        raise AssertionError, msg
-
-if __name__=="__main__":
-    if numprocs == 1: 
-        runner = unittest.TextTestRunner()
-        suite = unittest.makeSuite(Test_parallel_sw_flow, 'test')
-        runner.run(suite)
-    else:
-
-        #------------------------------------------
-        # Run the sequential code on each processor
-        # and save results at 4 gauge stations to
-        # array G
-        #------------------------------------------
-        barrier()
-        if myid == 0 and verbose: print 'SEQUENTIAL START'
-
-        G , interpolation_points = evolution_test(parallel=False,verbose=verbose)
-        G = num.array(G,num.float)
-
-        barrier()
-        
-        #------------------------------------------
-        # Run the code code and compare sequential
-        # results at 4 gauge stations
-        #------------------------------------------
-        if myid ==0 and verbose: print 'PARALLEL START'
-
-        evolution_test(parallel=True, G=G, seq_interpolation_points = interpolation_points, verbose= verbose)
-        
-        finalize()
-
-
Index: unk/anuga_core/source/anuga_parallel/test_sequential_dist_sw_flow.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_sequential_dist_sw_flow.py	(revision 9423)
+++ 	(revision )
@@ -1,312 +1,0 @@
-"""
-Simple water flow example using ANUGA
-
-Water driven up a linear slope and time varying boundary,
-similar to a beach environment
-
-This is a very simple test of the parallel algorithm using the simplified parallel API
-"""
-
-
-#------------------------------------------------------------------------------
-# Import necessary modules
-#------------------------------------------------------------------------------
-import unittest
-import os
-import sys
-#import pypar
-import numpy as num
-
-
-
-from anuga import Domain
-from anuga import Reflective_boundary
-from anuga import Dirichlet_boundary
-from anuga import Time_boundary
-from anuga import Transmissive_boundary
-
-from anuga import rectangular_cross_domain
-
-
-from anuga_parallel import distribute, myid, numprocs, send, receive, barrier, finalize
-
-from anuga_parallel.sequential_distribute import sequential_distribute_dump
-from anuga_parallel.sequential_distribute import sequential_distribute_load
-
-import anuga.utilities.plot_utils as util
-
-
-#--------------------------------------------------------------------------
-# Setup parameters
-#--------------------------------------------------------------------------
-yieldstep = 0.25
-finaltime = 3.0
-nprocs = 4
-N = 29
-M = 29 
-verbose = False
-
-
-new_parameters = {}
-new_parameters['ghost_layer_width'] = 2
-
-#---------------------------------
-# Setup Functions
-#---------------------------------
-def topography(x,y): 
-    return -x/2    
-
-###########################################################################
-# Setup Test
-##########################################################################
-def evolution_test(parallel=False, verbose=False):
-
-    #--------------------------------------------------------------------------
-    # Setup computational domain and quantities
-    #--------------------------------------------------------------------------
-    if myid == 0:
-        domain = rectangular_cross_domain(M, N)
-        domain.set_name('odomain')                    # Set sww filename
-        domain.set_datadir('.')   
-        domain.set_quantity('elevation', topography) # Use function for elevation
-        domain.set_quantity('friction', 0.0)         # Constant friction 
-        domain.set_quantity('stage', expression='elevation') # Dry initial stage
-    else:
-        domain = None
-        
-    #--------------------------------------------------------------------------
-    # Create pickled partition
-    #--------------------------------------------------------------------------
-    if myid == 0:
-        if verbose: print 'DUMPING PARTITION DATA'
-        sequential_distribute_dump(domain, numprocs, verbose=verbose, parameters=new_parameters)    
-
-    #--------------------------------------------------------------------------
-    # Create the parallel domains
-    #--------------------------------------------------------------------------
-    if parallel:
-        
-        if myid == 0 and verbose : print 'DISTRIBUTING TO PARALLEL DOMAIN'
-        pdomain = distribute(domain, verbose=verbose, parameters=new_parameters)
-        pdomain.set_name('pdomain')
-        
-        if myid == 0 and verbose : print 'LOADING IN PARALLEL DOMAIN'
-        sdomain = sequential_distribute_load(filename='odomain', verbose = verbose)
-        sdomain.set_name('sdomain')
-        
-    if myid == 0 and verbose: print 'EVOLVING pdomain'    
-    setup_and_evolve(pdomain, verbose=verbose)
- 
-    if myid == 0 and verbose: print 'EVOLVING sdomain'   
-    setup_and_evolve(sdomain, verbose=verbose)
-    
-    if myid == 0:
-        if verbose: print 'EVOLVING odomain'   
-        setup_and_evolve(domain, verbose=verbose)
-    
-
-    if myid == 0:
-        parameter_file=open('odomain.txt', 'w')
-        from pprint import pprint
-        pprint(domain.get_algorithm_parameters(),parameter_file,indent=4)
-        parameter_file.close()
-
-        parameter_file=open('sdomain.txt', 'w')
-        from pprint import pprint
-        pprint(sdomain.get_algorithm_parameters(),parameter_file,indent=4)
-        parameter_file.close()
-
-        parameter_file=open('pdomain.txt', 'w')
-        from pprint import pprint
-        pprint(pdomain.get_algorithm_parameters(),parameter_file,indent=4)
-        parameter_file.close()        
-    
-    assert num.allclose(pdomain.quantities['stage'].centroid_values, sdomain.quantities['stage'].centroid_values)
-    assert num.allclose(pdomain.quantities['stage'].vertex_values, sdomain.quantities['stage'].vertex_values)
-    
-    assert num.allclose(pdomain.vertex_coordinates, sdomain.vertex_coordinates)
-    assert num.allclose(pdomain.centroid_coordinates, sdomain.centroid_coordinates)
-    
-    
-
-    #---------------------------------
-    # Now compare the merged sww files
-    #---------------------------------
-    if myid == 0:
-        if verbose: print 'COMPARING SWW FILES'
-        
-        odomain_v = util.get_output('odomain.sww')
-        odomain_c = util.get_centroids(odomain_v)
-
-        pdomain_v = util.get_output('pdomain.sww')
-        pdomain_c = util.get_centroids(pdomain_v)
-        
-        sdomain_v = util.get_output('sdomain.sww')
-        sdomain_c = util.get_centroids(sdomain_v)
-
-        # Test some values against the original ordering
-        
-        if verbose:
-            
-            order = 0
-            print 'PDOMAIN CENTROID VALUES'
-            print num.linalg.norm(odomain_c.x-pdomain_c.x,ord=order)
-            print num.linalg.norm(odomain_c.y-pdomain_c.y,ord=order)
-            print num.linalg.norm(odomain_c.stage[-1]-pdomain_c.stage[-1],ord=order)
-            print num.linalg.norm(odomain_c.xmom[-1]-pdomain_c.xmom[-1],ord=order)
-            print num.linalg.norm(odomain_c.ymom[-1]-pdomain_c.ymom[-1],ord=order)
-            print num.linalg.norm(odomain_c.xvel[-1]-pdomain_c.xvel[-1],ord=order)
-            print num.linalg.norm(odomain_c.yvel[-1]-pdomain_c.yvel[-1],ord=order)        
-            
-             
-            print 'SDOMAIN CENTROID VALUES'        
-            print num.linalg.norm(odomain_c.x-sdomain_c.x,ord=order)
-            print num.linalg.norm(odomain_c.y-sdomain_c.y,ord=order)
-            print num.linalg.norm(odomain_c.stage[-1]-sdomain_c.stage[-1],ord=order)
-            print num.linalg.norm(odomain_c.xmom[-1]-sdomain_c.xmom[-1],ord=order)
-            print num.linalg.norm(odomain_c.ymom[-1]-sdomain_c.ymom[-1],ord=order)
-            print num.linalg.norm(odomain_c.xvel[-1]-sdomain_c.xvel[-1],ord=order)
-            print num.linalg.norm(odomain_c.yvel[-1]-sdomain_c.yvel[-1],ord=order)
-            
-            print 'PDOMAIN VERTEX VALUES'        
-            print num.linalg.norm(odomain_v.stage[-1]-pdomain_v.stage[-1],ord=order)
-            print num.linalg.norm(odomain_v.xmom[-1]-pdomain_v.xmom[-1],ord=order)
-            print num.linalg.norm(odomain_v.ymom[-1]-pdomain_v.ymom[-1],ord=order)
-            print num.linalg.norm(odomain_v.xvel[-1]-pdomain_v.xvel[-1],ord=order)
-            print num.linalg.norm(odomain_v.yvel[-1]-pdomain_v.yvel[-1],ord=order)
-            
-            print 'SDOMAIN VERTEX VALUES'     
-            print num.linalg.norm(odomain_v.stage[-1]-sdomain_v.stage[-1],ord=order)
-            print num.linalg.norm(odomain_v.xmom[-1]-sdomain_v.xmom[-1],ord=order)
-            print num.linalg.norm(odomain_v.ymom[-1]-sdomain_v.ymom[-1],ord=order)
-            print num.linalg.norm(odomain_v.xvel[-1]-sdomain_v.xvel[-1],ord=order)
-            print num.linalg.norm(odomain_v.yvel[-1]-sdomain_v.yvel[-1],ord=order)
-            
-            
-            
-
-        assert num.allclose(odomain_c.stage,pdomain_c.stage)
-        assert num.allclose(odomain_c.xmom,pdomain_c.xmom)
-        assert num.allclose(odomain_c.ymom,pdomain_c.ymom)
-        assert num.allclose(odomain_c.xvel,pdomain_c.xvel)
-        assert num.allclose(odomain_c.yvel,pdomain_c.yvel)
-        
-        assert num.allclose(odomain_v.x,pdomain_v.x)
-        assert num.allclose(odomain_v.y,pdomain_v.y)
-                
-        assert num.linalg.norm(odomain_v.x-pdomain_v.x,ord=0) == 0
-        assert num.linalg.norm(odomain_v.y-pdomain_v.y,ord=0) == 0
-        assert num.linalg.norm(odomain_v.stage[-1]-pdomain_v.stage[-1],ord=0) < 100
-        assert num.linalg.norm(odomain_v.xmom[-1]-pdomain_v.xmom[-1],ord=0) < 100 
-        assert num.linalg.norm(odomain_v.ymom[-1]-pdomain_v.ymom[-1],ord=0) < 100
-        assert num.linalg.norm(odomain_v.xvel[-1]-pdomain_v.xvel[-1],ord=0) < 100
-        assert num.linalg.norm(odomain_v.yvel[-1]-pdomain_v.yvel[-1],ord=0) < 100     
-        
-        assert num.allclose(odomain_c.x,sdomain_c.x)
-        assert num.allclose(odomain_c.y,sdomain_c.y)
-        assert num.allclose(odomain_c.stage,sdomain_c.stage)
-        assert num.allclose(odomain_c.xmom,sdomain_c.xmom)
-        assert num.allclose(odomain_c.ymom,sdomain_c.ymom)
-        assert num.allclose(odomain_c.xvel,sdomain_c.xvel)
-        assert num.allclose(odomain_c.yvel,sdomain_c.yvel)
-        
-        assert num.allclose(odomain_v.x,sdomain_v.x)
-        assert num.allclose(odomain_v.y,sdomain_v.y)
-        
-        order = 0
-        assert num.linalg.norm(odomain_v.x-sdomain_v.x,ord=order) == 0
-        assert num.linalg.norm(odomain_v.y-sdomain_v.y,ord=order) == 0
-        assert num.linalg.norm(odomain_v.stage[-1]-sdomain_v.stage[-1],ord=order) < 100
-        assert num.linalg.norm(odomain_v.xmom[-1]-sdomain_v.xmom[-1],ord=order) < 100
-        assert num.linalg.norm(odomain_v.ymom[-1]-sdomain_v.ymom[-1],ord=order) < 100
-        assert num.linalg.norm(odomain_v.xvel[-1]-sdomain_v.xvel[-1],ord=order) < 100
-        assert num.linalg.norm(odomain_v.yvel[-1]-sdomain_v.yvel[-1],ord=order) < 100        
-                
-        # COMPARE CENTROID PDOMAIN SDOMAIN  
-        assert num.allclose(pdomain_c.x,sdomain_c.x)
-        assert num.allclose(pdomain_c.y,sdomain_c.y)
-        assert num.allclose(pdomain_c.stage[-1],sdomain_c.stage[-1])
-        assert num.allclose(pdomain_c.xmom[-1],sdomain_c.xmom[-1])
-        assert num.allclose(pdomain_c.ymom[-1],sdomain_c.ymom[-1])
-        assert num.allclose(pdomain_c.xvel[-1],sdomain_c.xvel[-1])
-        assert num.allclose(pdomain_c.yvel[-1],sdomain_c.yvel[-1])
-            
-            
-        # COMPARE VERTEX PDOMAIN SDOMAIN
-        assert num.allclose(pdomain_v.x,sdomain_v.x)
-        assert num.allclose(pdomain_v.y,sdomain_v.y)
-        assert num.allclose(pdomain_v.stage[-1],sdomain_v.stage[-1])
-        assert num.allclose(pdomain_v.xmom[-1],sdomain_v.xmom[-1])
-        assert num.allclose(pdomain_v.ymom[-1],sdomain_v.ymom[-1])
-        assert num.allclose(pdomain_v.xvel[-1],sdomain_v.xvel[-1])
-        assert num.allclose(pdomain_v.yvel[-1],sdomain_v.yvel[-1])   
-            
-
-        
-        
-def setup_and_evolve(domain, verbose=False):
-
-    #--------------------------------------------------------------------------
-    # Setup domain parameters
-    #--------------------------------------------------------------------------
-    domain.set_flow_algorithm('DE0')
-    #domain.set_store_vertices_uniquely()
-
-    #------------------------------------------------------------------------------
-    # Setup boundary conditions
-    # This must currently happen *AFTER* domain has been distributed
-    #------------------------------------------------------------------------------
-
-    Br = Reflective_boundary(domain)      # Solid reflective wall
-    Bd = Dirichlet_boundary([-0.2,0.,0.]) # Constant boundary values
-
-    # Associate boundary tags with boundary objects
-    domain.set_boundary({'left': Br, 'right': Bd, 'top': Br, 'bottom': Br})
-
-    #------------------------------------------------------------------------------
-    # Evolve
-    #------------------------------------------------------------------------------
-    for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
-        if myid == 0 and verbose : domain.write_time()
-        #if myid == 0 and verbose : print domain.quantities['stage'].get_maximum_value()
-
-
-    domain.sww_merge()
-    
-
-# Test an nprocs-way run of the shallow water equations
-# against the sequential code.
-
-class Test_parallel_sw_flow(unittest.TestCase):
-    def test_parallel_sw_flow(self):
-        if verbose : print "Expect this test to fail if not run from the parallel directory."
-        result = os.system("mpirun -np %d python test_sequential_dist_sw_flow.py" % nprocs)
-        assert_(result == 0)
-
-# Because we are doing assertions outside of the TestCase class
-# the PyUnit defined assert_ function can't be used.
-def assert_(condition, msg="Assertion Failed"):
-    if condition == False:
-        #pypar.finalize()
-        raise AssertionError, msg
-
-if __name__=="__main__":
-    if numprocs == 1: 
-        runner = unittest.TextTestRunner()
-        suite = unittest.makeSuite(Test_parallel_sw_flow, 'test')
-        runner.run(suite)
-    else:
-
-
-        
-        #------------------------------------------
-        # Run the code code and compare sequential
-        # results at 4 gauge stations
-        #------------------------------------------
-        if myid ==0 and verbose: print 'PARALLEL START'
-
-        evolution_test(parallel=True, verbose=verbose)
-        
-        finalize()
-
-
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_all.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_all.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_all.py	(revision 9424)
@@ -0,0 +1,223 @@
+"""Regression testing framework
+This module will search for scripts in the same directory named
+test_*.py.  Each such script should be a test suite that tests a
+module through PyUnit. This script will aggregate all
+found test suites into one big test suite and run them all at once.
+"""
+
+# Author: Mark Pilgrim
+# Modified by Ole Nielsen
+
+import unittest
+import os
+import sys
+import tempfile
+import time
+import anuga.utilities.system_tools as aust
+from anuga.utilities.terminal_width import terminal_width
+
+
+
+
+
+#List files that should be excluded from the testing process.
+#E.g. if they are known to fail and under development
+exclude_files = ['test_failure.py', 'test_parallel_boyd_pipe_operator.py', 'test_parallel_boyd_box_op_apron.py']
+
+# Directories that should not be searched for test files.
+exclude_dirs = ['documentation',               # Special requirements
+                '.svn',                              # subversion
+                'props', 'wcprops', 'prop-base', 'text-base', 'tmp']
+
+
+##
+# @brief List a string sequence on the screen in columns.
+# @param names Sequence of strings to list.
+# @param func Function to apply to each string in sequence.
+# @param col_width Force columns to this width (default calculated).
+# @param page_width Set displayable page width to this (default 132).
+def list_names(names, func=None, col_width=None, page_width=None):
+    # set defaults
+    p_width = page_width - 1            # set page width
+    if p_width is None:
+        p_width = 132                   # default page width
+
+    c_width = col_width                 # set column width
+    if c_width is None:
+        c_width = 0
+        for name in names:
+            if func:
+                name = func(name)
+            c_width = max(c_width, len(name))
+    c_width += 2                        # 2 column padding
+
+    # calculate number of columns allowed
+    max_columns = int(p_width / c_width)
+
+    # print columns
+    column = 0
+    for name in names:
+        if func:
+            name = func(name)
+        print '%-*s' % (c_width-1, name),
+        column += 1
+        if column >= max_columns:
+            column = 0
+            print
+
+    # if last line not finished, end it here
+    if column > 0:
+        print
+
+
+##
+# @brief Get 'test_*.py' files and paths to directories.
+# @param path Path to directory to start walk in.
+# @return A tuple (<files>, <dirs>).
+# @note Don't include any files in and below forbidden directories.
+def get_test_files(path):
+    walk = os.walk(path)
+
+    test_files = []
+    path_files = []
+
+    for (dirpath, dirnames, filenames) in walk:
+        # exclude forbidden directories
+        for e_dir in exclude_dirs:
+            try:
+                dirnames.remove(e_dir)
+            except ValueError:
+                pass
+
+        # check for test_*.py files
+        for filename in filenames:
+            if filename.startswith('test_') and filename.endswith('.py'):
+                test_files.append(filename)
+                if dirpath not in path_files:
+                    path_files.append(dirpath)
+
+    return test_files, path_files
+
+
+def regressionTest(test_verbose=False):
+    # start off with where we are
+    path = os.getcwd()
+    print
+    print 'Testing path: %s' % path
+
+    # get the terminal width
+    term_width = terminal_width()
+
+    # explain what we are doing
+    print
+    print "The following directories will be skipped over:"
+    exclude_dirs.sort()
+    list_names(exclude_dirs, page_width=term_width)
+
+    # get all test_*.py and enclosing directories
+    test_files, path_files = get_test_files(path)
+    path_files.sort()
+
+    files = [x for x in test_files if not x == 'test_all.py']
+    files.sort()        # Ensure same order on all platforms
+
+    print
+    print 'Paths searched:'
+    list_names(path_files, os.path.basename, page_width=term_width)
+
+    print    
+    print 'Files tested:'
+    list_names(files, page_width=term_width)
+    print
+
+    # update system path with found paths
+    for path in path_files:
+        sys.path.append(path)
+   
+    # exclude files that we can't handle 
+    for file in exclude_files:
+        print 'WARNING: File '+ file + ' to be excluded from testing'
+        try:
+            files.remove(file)
+        except ValueError, e:
+            msg = 'File "%s" was not found in test suite.\n' % file
+            msg += 'Original error is "%s"\n' % e
+            msg += 'Perhaps it should be removed from exclude list?'
+            raise Exception, msg
+
+    # import all test_*.py files
+    # NOTE: This implies that test_*.py files MUST HAVE UNIQUE NAMES!
+    filenameToModuleName = lambda f: os.path.splitext(f)[0]
+    moduleNames = map(filenameToModuleName, files)
+    modules = map(__import__, moduleNames)
+
+    # Fix up the system path
+    for file in path_files:
+        sys.path.remove(file)
+
+    # bundle up all the tests
+    load = unittest.defaultTestLoader.loadTestsFromModule
+    testCaseClasses = map(load, modules)
+
+    if test_verbose is True:
+        # Test the code by setting verbose to True.
+        # The test cases have to be set up for this to work.
+        # See test data manager for an example.
+        for test_suite in testCaseClasses:
+            for tests in test_suite._tests:
+                # tests is of class TestSuite
+                if len(tests._tests) > 1:
+                    # these are the test functions
+                    try:
+                        # Calls class method set_verbose in test case classes
+                        tests._tests[0].set_verbose()
+                    except:
+                        pass                # No all classes have set_verbose
+
+    return unittest.TestSuite(testCaseClasses)
+
+
+##
+# @brief Check that the environment is sane.
+# @note Stops here if there is an error.
+def check_anuga_import():
+    try:
+        # importing something that loads quickly
+        import anuga.anuga_exceptions
+    except ImportError:
+        print "Python cannot import ANUGA module."
+        print "Check you have followed all steps of its installation."
+        import sys
+        sys.exit()
+
+
+if __name__ == '__main__':
+    check_anuga_import()
+
+    if len(sys.argv) > 1 and sys.argv[1][0].upper() == 'V':
+        test_verbose = True
+        saveout = sys.stdout
+        filename = ".temp"
+        fid = open(filename, 'w')
+        sys.stdout = fid
+    else:
+        test_verbose = False
+    suite = regressionTest(test_verbose)
+    runner = unittest.TextTestRunner(verbosity=1)
+    runner.run(suite)
+
+    # timestamp at the end
+    timestamp = time.asctime()
+    version = aust.get_revision_number()
+    print
+    print 'Finished at %s, version %s' % (timestamp, version)
+
+    # Cleaning up
+    if len(sys.argv) > 1 and sys.argv[1][0].upper() == 'V':
+        sys.stdout = saveout
+        #fid.close() # This was causing an error in windows
+        #os.remove(filename)
+
+    
+    if sys.platform == 'win32':
+        raw_input('Press the RETURN key')
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_distribute_mesh.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_distribute_mesh.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_distribute_mesh.py	(revision 9424)
@@ -0,0 +1,1518 @@
+#!/usr/bin/env python
+
+import unittest
+import sys
+from math import sqrt
+
+
+from anuga import Domain
+from anuga import rectangular_cross
+
+from anuga_parallel.distribute_mesh import pmesh_divide_metis
+from anuga_parallel.distribute_mesh import build_submesh
+from anuga_parallel.distribute_mesh import submesh_full, submesh_ghost, submesh_quantities
+from anuga_parallel.distribute_mesh import extract_submesh, rec_submesh, send_submesh
+
+import numpy as num
+
+
+def topography(x,y): 
+    return -x/2
+
+
+def xcoord(x,y):
+    return x
+
+def ycoord(x,y):
+    return y
+
+
+
+class Test_Distribute_Mesh(unittest.TestCase):
+    def setUp(self):
+        pass
+
+
+    def tearDown(self):
+        pass
+
+
+
+    def test_pmesh_1(self):
+        """
+        test distributing with just one processor
+        """
+        
+        points, vertices, boundary = rectangular_cross(2,2)
+
+
+        true_points = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], \
+        [0.5, 0.5], [0.5, 1.0], [1.0, 0.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], \
+        [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
+
+        true_vertices = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [1, 10, 2], \
+        [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7],
+        [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]]
+
+
+        assert num.allclose(points,true_points)
+        assert num.allclose(vertices,true_vertices)
+
+        domain = Domain(points, vertices, boundary)
+
+
+        domain.set_quantity('elevation', topography) # Use function for elevation
+        domain.set_quantity('friction', 0.0)         # Constant friction 
+        domain.set_quantity('stage', expression='elevation') # Dry initial stage
+        domain.set_quantity('xmomentum', expression='friction + 2.0') # 
+        domain.set_quantity('ymomentum', ycoord) #
+
+
+        #print domain.quantities['ymomentum'].centroid_values
+ 
+        nodes, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(domain,1)
+
+
+        true_nodes = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], \
+        [1.0, 0.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
+
+        true_triangles = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [1, 10, 2], [4, 10, 1], \
+        [5, 10, 4], [2, 10, 5], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7], [4, 12, 5], \
+        [7, 12, 4], [8, 12, 7], [5, 12, 8]]
+
+
+        assert num.allclose(nodes,true_nodes)
+        assert num.allclose(triangles,true_triangles)
+
+
+        assert num.allclose(triangles_per_proc,[16])
+        
+        
+
+    def test_pmesh_2(self):
+        """
+        Test 2 way pmesh
+        """
+        points, vertices, boundary = rectangular_cross(2,2)
+
+
+        true_points = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], \
+        [1.0, 0.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
+
+        true_vertices = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [1, 10, 2], [4, 10, 1], \
+        [5, 10, 4], [2, 10, 5], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7], [4, 12, 5], \
+        [7, 12, 4], [8, 12, 7], [5, 12, 8]]
+
+
+        assert num.allclose(points,true_points)
+        assert num.allclose(vertices,true_vertices)
+
+        domain = Domain(points, vertices, boundary)
+
+
+        domain.set_quantity('elevation', topography) # Use function for elevation
+        domain.set_quantity('friction', 0.0)         # Constant friction 
+        domain.set_quantity('stage', expression='elevation') # Dry initial stage
+        domain.set_quantity('xmomentum', expression='friction + 2.0') # 
+        domain.set_quantity('ymomentum', ycoord) #
+
+
+        #print domain.quantities['ymomentum'].centroid_values
+ 
+        nodes, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(domain,2)
+
+
+        true_nodes = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], \
+        [1.0, 0.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
+
+
+        true_triangles = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [4, 10, 1], [3, 11, 4], \
+        [4, 11, 7], [4, 12, 5], [1, 10, 2], [5, 10, 4], [2, 10, 5], [6, 11, 3], [7, 11, 6], \
+        [7, 12, 4], [8, 12, 7], [5, 12, 8]]
+
+
+        assert num.allclose(nodes,true_nodes)
+        assert num.allclose(triangles,true_triangles)
+
+
+        assert num.allclose(triangles_per_proc,[8,8])
+
+
+            
+    def test_build_submesh_3(self):
+        """
+        Test 3 way build_submesh
+        """
+
+        nodes = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], [1.0, 0.0], \
+        [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
+
+
+        triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], \
+        [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], \
+        [6, 11, 3], [7, 11, 6], [4, 11, 7]]
+
+
+        boundary = {(13, 1): 'bottom', (7, 1): 'left', (3, 1): 'right', (14, 1): 'right', \
+        (11, 1): 'bottom', (10, 1): 'top', (5, 1): 'left', (4, 1): 'top'}
+
+        triangles_per_proc = [5, 6, 5]
+
+
+        quantities = {'stage': num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]),  'elevation': num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]),  'ymomentum': num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]]),  'friction': num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]),  'xmomentum': num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])}
+
+
+        
+        true_submesh = {'full_boundary': [{(3, 1): 'right', (4, 1): 'top'},\
+        {(5, 1): 'left', (10, 1): 'top', (7, 1): 'left'}, \
+        {(13, 1): 'bottom', (14, 1): 'right', (11, 1): 'bottom'}],
+                        'ghost_nodes': [num.array([[  0.  ,   0.  ,   0.  ],
+       [  1.  ,   0.  ,   0.5 ],
+       [  2.  ,   0.  ,   1.  ],
+       [  6.  ,   1.  ,   0.  ],
+       [ 10.  ,   0.25,   0.75],
+       [ 11.  ,   0.75,   0.25]]), num.array([[  3.  ,   0.5 ,   0.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  8.  ,   1.  ,   1.  ],
+       [ 11.  ,   0.75,   0.25],
+       [ 12.  ,   0.75,   0.75]]), num.array([[  1.  ,   0.  ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  8.  ,   1.  ,   1.  ],
+       [ 12.  ,   0.75,   0.75]])],
+                        'full_nodes': [num.array([[  3.  ,   0.5 ,   0.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  8.  ,   1.  ,   1.  ],
+       [  9.  ,   0.25,   0.25],
+       [ 12.  ,   0.75,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
+       [  1.  ,   0.  ,   0.5 ],
+       [  2.  ,   0.  ,   1.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  9.  ,   0.25,   0.25],
+       [ 10.  ,   0.25,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
+       [  3.  ,   0.5 ,   0.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  6.  ,   1.  ,   0.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  9.  ,   0.25,   0.25],
+       [ 11.  ,   0.75,   0.25]])],
+                        'ghost_triangles': [num.array([[ 5,  0,  9,  1],
+       [ 6,  1,  9,  4],
+       [ 8,  4, 10,  1],
+       [ 9,  5, 10,  4],
+       [10,  2, 10,  5],
+       [11,  3,  9,  0],
+       [12,  3, 11,  4],
+       [13,  6, 11,  3],
+       [14,  7, 11,  6],
+       [15,  4, 11,  7]]), num.array([[ 0,  4,  9,  3],
+       [ 1,  4, 12,  5],
+       [ 2,  7, 12,  4],
+       [ 4,  5, 12,  8],
+       [11,  3,  9,  0],
+       [12,  3, 11,  4]]), num.array([[ 0,  4,  9,  3],
+       [ 1,  4, 12,  5],
+       [ 2,  7, 12,  4],
+       [ 3,  8, 12,  7],
+       [ 5,  0,  9,  1],
+       [ 6,  1,  9,  4]])],
+                        'ghost_boundary': [{(13, 1): 'ghost', (8, 0): 'ghost', (14, 1): 'ghost', \
+                        (11, 1): 'ghost', (10, 1): 'ghost', (5, 1): 'ghost', (10, 2): 'ghost'}, \
+                        {(12, 2): 'ghost', (12, 0): 'ghost', (2, 1): 'ghost', (11, 1): 'ghost',\
+                        (2, 2): 'ghost', (4, 1): 'ghost', (4, 0): 'ghost'}, {(3, 2): 'ghost', \
+                        (6, 1): 'ghost', (3, 1): 'ghost', (5, 1): 'ghost', (1, 0): 'ghost', (1, 1): 'ghost'}],
+                        'full_triangles': [[[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]], \
+                        [[0, 9, 1], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5]], \
+                        [[3, 9, 0], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7]]],
+                        'full_commun': [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, \
+                        {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, \
+                        {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}],
+                        'ghost_commun': [num.array([[ 5,  1],
+       [ 6,  1],
+       [ 8,  1],
+       [ 9,  1],
+       [10,  1],
+       [11,  2],
+       [12,  2],
+       [13,  2],
+       [14,  2],
+       [15,  2]]), num.array([[ 0,  0],
+       [ 1,  0],
+       [ 2,  0],
+       [ 4,  0],
+       [11,  2],
+       [12,  2]]), num.array([[0, 0],
+       [1, 0],
+       [2, 0],
+       [3, 0],
+       [5, 1],
+       [6, 1]])],  'ghost_quan': {'stage': [num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ]])],  'elevation': [num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ]])],  'ymomentum': [num.array([[ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]])], 'xmomentum': [num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])]},  'full_quan': {'stage': [num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]])],  'elevation': [num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]])],  'ymomentum': [num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]])],  'xmomentum': [num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])]}}
+                                  
+
+        from anuga.abstract_2d_finite_volumes.neighbour_mesh import Mesh
+        
+        mesh = Mesh(nodes, triangles, boundary)
+        boundary_polygon = mesh.get_boundary_polygon()
+
+
+        # Subdivide into non-overlapping partitions
+
+        submesh = submesh_full(mesh, triangles_per_proc)
+
+        #print submesh
+
+
+        for i in range(3):
+            assert num.allclose(true_submesh['full_triangles'][i],submesh['full_triangles'][i])
+            assert num.allclose(true_submesh['full_nodes'][i],submesh['full_nodes'][i])
+        assert true_submesh['full_boundary'] == submesh['full_boundary']
+
+        # Add any extra ghost boundary layer information
+
+        submesh = submesh_ghost(submesh, mesh, triangles_per_proc)
+
+        for i in range(3):
+            assert num.allclose(true_submesh['ghost_triangles'][i],submesh['ghost_triangles'][i])
+            assert num.allclose(true_submesh['ghost_nodes'][i],submesh['ghost_nodes'][i])
+            assert num.allclose(true_submesh['ghost_commun'][i],submesh['ghost_commun'][i])
+
+        assert true_submesh['full_commun'] == submesh['full_commun']
+
+
+        # Order the quantities information to be the same as the triangle
+        # information
+
+
+        submesh = submesh_quantities(submesh, quantities, \
+                                 triangles_per_proc)
+
+
+
+        for key, value in true_submesh['ghost_quan'].iteritems():
+            for i in range(3):
+                assert num.allclose(true_submesh['ghost_quan'][key][i],submesh['ghost_quan'][key][i])
+                assert num.allclose(true_submesh['full_quan'][key][i],submesh['full_quan'][key][i])
+        
+
+        submesh["boundary_polygon"] = boundary_polygon
+
+
+
+    def test_build_submesh_3_layer_4(self):
+        """
+        Test 3 Layer 4 way build_submesh with ghost_layer_width = 4
+        """
+
+        nodes = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], \
+        [1.0, 0.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
+
+
+        triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], \
+        [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], \
+        [6, 11, 3], [7, 11, 6], [4, 11, 7]]
+
+
+        boundary = {(13, 1): 'bottom', (7, 1): 'left', (3, 1): 'right', (14, 1): 'right', \
+        (11, 1): 'bottom', (10, 1): 'top', (5, 1): 'left', (4, 1): 'top'}
+
+        """
+                     top
+            (10,1)       (4,1)
+        2 ---------- 5 ---------- 8
+        | \   10   / | \   04   / |
+        |   \    /   |   \    /   |
+ (7,1)  | 07  10  09 | 01  12  03 | (3,1)
+        |   /    \   |   /    \   |
+        | /   08   \ | /   02   \ |
+ left   1 ---------- 4 ---------- 7  right
+        | \   06   / | \   15   / |
+        |   \    /   |   \    /   |
+ (5,1)  | 05  09  00 | 12  11  14 | (14,1)
+        |   /    \   |   /    \   |
+        | /   11   \ | /   13   \ |
+        0 ---------- 3 ---------- 6
+            (11,1)        (13,1)
+                   bottom
+
+
+        Processor 0 Full Triangles 00,01,02,03,04
+        Processor 1 Full Triangles 05,06,07,08,09,10
+        Processor 2 Full Triangles 11,12,13,14,15
+        """
+
+
+        triangles_per_proc = [5, 6, 5]
+
+
+        quantities = {'stage': num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]),  'elevation': num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]),  'ymomentum': num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]]),  'friction': num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]),  'xmomentum': num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])}
+
+
+
+        true_submesh = {'full_boundary': [{(3, 1): 'right', (4, 1): 'top'}, {(5, 1): 'left', \
+        (10, 1): 'top', (7, 1): 'left'}, {(13, 1): 'bottom', (14, 1): 'right', (11, 1): 'bottom'}],
+                        'ghost_nodes': [num.array([[  0.  ,   0.  ,   0.  ],
+       [  1.  ,   0.  ,   0.5 ],
+       [  2.  ,   0.  ,   1.  ],
+       [  6.  ,   1.  ,   0.  ],
+       [ 10.  ,   0.25,   0.75],
+       [ 11.  ,   0.75,   0.25]]), num.array([[  3.  ,   0.5 ,   0.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  8.  ,   1.  ,   1.  ],
+       [ 11.  ,   0.75,   0.25],
+       [ 12.  ,   0.75,   0.75]]), num.array([[  1.  ,   0.  ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  8.  ,   1.  ,   1.  ],
+       [ 12.  ,   0.75,   0.75]])],
+                        'full_nodes': [num.array([[  3.  ,   0.5 ,   0.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  8.  ,   1.  ,   1.  ],
+       [  9.  ,   0.25,   0.25],
+       [ 12.  ,   0.75,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
+       [  1.  ,   0.  ,   0.5 ],
+       [  2.  ,   0.  ,   1.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  9.  ,   0.25,   0.25],
+       [ 10.  ,   0.25,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
+       [  3.  ,   0.5 ,   0.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  6.  ,   1.  ,   0.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  9.  ,   0.25,   0.25],
+       [ 11.  ,   0.75,   0.25]])],
+                        'ghost_triangles': [num.array([[ 5,  0,  9,  1],
+       [ 6,  1,  9,  4],
+       [ 8,  4, 10,  1],
+       [ 9,  5, 10,  4],
+       [10,  2, 10,  5],
+       [11,  3,  9,  0],
+       [12,  3, 11,  4],
+       [13,  6, 11,  3],
+       [14,  7, 11,  6],
+       [15,  4, 11,  7]]), num.array([[ 0,  4,  9,  3],
+       [ 1,  4, 12,  5],
+       [ 2,  7, 12,  4],
+       [ 4,  5, 12,  8],
+       [11,  3,  9,  0],
+       [12,  3, 11,  4]]), num.array([[ 0,  4,  9,  3],
+       [ 1,  4, 12,  5],
+       [ 2,  7, 12,  4],
+       [ 3,  8, 12,  7],
+       [ 5,  0,  9,  1],
+       [ 6,  1,  9,  4]])],
+                        'ghost_boundary': [{(13, 1): 'ghost', (8, 0): 'ghost', \
+                        (14, 1): 'ghost', (11, 1): 'ghost', (10, 1): 'ghost', \
+                        (5, 1): 'ghost', (10, 2): 'ghost'}, {(12, 2): 'ghost', \
+                        (12, 0): 'ghost', (2, 1): 'ghost', (11, 1): 'ghost', \
+                        (2, 2): 'ghost', (4, 1): 'ghost', (4, 0): 'ghost'}, \
+                        {(3, 2): 'ghost', (6, 1): 'ghost', (3, 1): 'ghost', \
+                        (5, 1): 'ghost', (1, 0): 'ghost', (1, 1): 'ghost'}],
+                        'full_triangles': [[[4, 9, 3], [4, 12, 5], [7, 12, 4], \
+                        [8, 12, 7], [5, 12, 8]], [[0, 9, 1], [1, 9, 4], [1, 10, 2], \
+                        [4, 10, 1], [5, 10, 4], [2, 10, 5]], [[3, 9, 0], [3, 11, 4], \
+                        [6, 11, 3], [7, 11, 6], [4, 11, 7]]],
+                        'full_commun': [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, \
+                        {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, \
+                        {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}],
+                        'ghost_commun': [num.array([[ 5,  1],
+       [ 6,  1],
+       [ 8,  1],
+       [ 9,  1],
+       [10,  1],
+       [11,  2],
+       [12,  2],
+       [13,  2],
+       [14,  2],
+       [15,  2]]), num.array([[ 0,  0],
+       [ 1,  0],
+       [ 2,  0],
+       [ 4,  0],
+       [11,  2],
+       [12,  2]]), num.array([[0, 0],
+       [1, 0],
+       [2, 0],
+       [3, 0],
+       [5, 1],
+       [6, 1]])],  'ghost_quan': {'stage': [num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ]])],  'elevation': [num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ]])],  'ymomentum': [num.array([[ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]])], 'xmomentum': [num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])]},  'full_quan': {'stage': [num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]])],  'elevation': [num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]])],  'ymomentum': [num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]])],  'xmomentum': [num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])]}}
+
+
+        # setup parameters to test using different ghost_layer_widths
+        parameters = dict(ghost_layer_width = 1)
+
+        from anuga.abstract_2d_finite_volumes.neighbour_mesh import Mesh
+
+        mesh = Mesh(nodes, triangles, boundary)
+        boundary_polygon = mesh.get_boundary_polygon()
+
+
+        # Subdivide into non-overlapping partitions
+
+        submesh = submesh_full(mesh, triangles_per_proc)
+
+
+
+        for i in range(3):
+            assert num.allclose(true_submesh['full_triangles'][i],submesh['full_triangles'][i])
+            assert num.allclose(true_submesh['full_nodes'][i],submesh['full_nodes'][i])
+        assert true_submesh['full_boundary'] == submesh['full_boundary']
+
+        # Add any extra ghost boundary layer information
+
+        submesh = submesh_ghost(submesh, mesh, triangles_per_proc, parameters)
+
+        #print submesh
+
+        #print 'ghost_triangles', submesh['ghost_triangles']
+        #print 'ghost_boundary', submesh['ghost_boundary']
+        #print 'ghost_nodes', submesh['ghost_nodes']
+        #print 'ghost_commun', submesh['ghost_commun']
+
+
+        true_submesh['ghost_boundary'] = [{(12, 2): 'ghost', (9, 0): 'ghost', (9, 2): 'ghost', \
+        (6, 1): 'ghost', (15, 0): 'ghost', (11, 1): 'bottom', (11, 0): 'ghost', (6, 2): 'ghost'}, \
+        {(0, 1): 'ghost', (1, 2): 'ghost', (1, 0): 'ghost', (11, 1): 'bottom'},\
+        {(5, 1): 'left', (2, 0): 'ghost', (0, 2): 'ghost', (5, 0): 'ghost', (2, 2): 'ghost'}]
+
+
+        true_submesh['ghost_triangles'] = [num.array([[ 6,  1,  9,  4],\
+                                                      [ 9,  5, 10,  4],\
+                                                      [11,  3,  9,  0],\
+                                                      [12,  3, 11,  4],\
+                                                      [15,  4, 11,  7]]), \
+                                           num.array([[ 0,  4,  9,  3],\
+                                                      [ 1,  4, 12,  5],\
+                                                     [11,  3,  9,  0]]),\
+                                           num.array([[ 0,  4,  9,  3],\
+                                                      [ 2,  7, 12,  4],\
+                                                      [ 5,  0,  9,  1]])]
+
+        true_submesh['ghost_nodes'] =  [num.array([[  0.  ,   0.  ,   0.  ],\
+                                                   [  1.  ,   0.  ,   0.5 ],\
+                                                   [ 10.  ,   0.25,   0.75],\
+                                                   [ 11.  ,   0.75,   0.25]]),\
+                                        num.array([[  3.  ,   0.5 ,   0.  ],\
+                                                   [ 12.  ,   0.75,   0.75]]),\
+                                        num.array([[  1.  ,   0.  ,   0.5 ],\
+                                                   [ 12.  ,   0.75,   0.75]])]
+
+        true_submesh['ghost_commun'] = [num.array([[ 6,  1],\
+                                                   [ 9,  1],\
+                                                   [11,  2],\
+                                                   [12,  2],\
+                                                   [15,  2]]),\
+                                        num.array([[ 0,  0],\
+                                                   [ 1,  0],\
+                                                   [11,  2]]),\
+                                        num.array([[0, 0],\
+                                                   [2, 0],\
+                                                   [5, 1]])]
+
+
+
+
+        for i in range(3):
+            assert true_submesh['ghost_boundary'][i] == submesh['ghost_boundary'][i]
+            assert num.allclose(true_submesh['ghost_triangles'][i],submesh['ghost_triangles'][i])
+            assert num.allclose(true_submesh['ghost_nodes'][i],submesh['ghost_nodes'][i])
+            assert num.allclose(true_submesh['ghost_commun'][i],submesh['ghost_commun'][i])
+
+
+
+        true_submesh['full_commun'] = [{0: [1, 2], 1: [1], 2: [2], 3: [], 4: []},\
+                                       {5: [2], 6: [0], 7: [], 8: [], 9: [0], 10: []},\
+                                       {11: [0, 1], 12: [0], 13: [], 14: [], 15: [0]}]
+
+        assert true_submesh['full_commun'] == submesh['full_commun']
+
+
+        # Order the quantities information to be the same as the triangle
+        # information
+
+
+        submesh = submesh_quantities(submesh, quantities, \
+                                 triangles_per_proc)
+
+
+        #print submesh['full_quan']
+
+
+
+        true_submesh['full_quan'] = {'stage': [num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]])], 'elevation': [num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]])], 'ymomentum': [num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]])], 'friction': [num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]])], 'xmomentum': [num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])]}
+
+
+        true_submesh['ghost_quan'] = {'stage': [num.array([[-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.25 , -0.125, -0.   ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.   , -0.125, -0.   ]])], 'elevation': [num.array([[-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.25 , -0.125, -0.   ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.   , -0.125, -0.   ]])], 'ymomentum': [num.array([[ 0.5 ,  0.25,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ]]), num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 0.  ,  0.25,  0.5 ]])], 'friction': [num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]])], 'xmomentum': [num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])]}
+
+
+        for key, value in true_submesh['ghost_quan'].iteritems():
+            for i in range(3):
+                assert num.allclose(true_submesh['ghost_quan'][key][i],submesh['ghost_quan'][key][i])
+                assert num.allclose(true_submesh['full_quan'][key][i],submesh['full_quan'][key][i])
+
+
+        submesh["boundary_polygon"] = boundary_polygon
+
+
+    def test_build_extract_submesh_3(self):
+        """
+        Test 3 way extract_submesh
+        """
+
+        nodes = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], [1.0, 0.0], \
+        [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
+
+
+        triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], \
+        [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], \
+        [6, 11, 3], [7, 11, 6], [4, 11, 7]]
+
+
+        boundary = {(13, 1): 'bottom', (7, 1): 'left', (3, 1): 'right', (14, 1): 'right', \
+        (11, 1): 'bottom', (10, 1): 'top', (5, 1): 'left', (4, 1): 'top'}
+
+        triangles_per_proc = [5, 6, 5]
+
+
+        quantities = {'stage': num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]),  'elevation': num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]),  'ymomentum': num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]]),  'friction': num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]),  'xmomentum': num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])}
+
+
+
+        true_submesh = {'full_boundary': [{(3, 1): 'right', (4, 1): 'top'},\
+        {(5, 1): 'left', (10, 1): 'top', (7, 1): 'left'}, \
+        {(13, 1): 'bottom', (14, 1): 'right', (11, 1): 'bottom'}],
+                        'ghost_nodes': [num.array([[  0.  ,   0.  ,   0.  ],
+       [  1.  ,   0.  ,   0.5 ],
+       [  2.  ,   0.  ,   1.  ],
+       [  6.  ,   1.  ,   0.  ],
+       [ 10.  ,   0.25,   0.75],
+       [ 11.  ,   0.75,   0.25]]), num.array([[  3.  ,   0.5 ,   0.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  8.  ,   1.  ,   1.  ],
+       [ 11.  ,   0.75,   0.25],
+       [ 12.  ,   0.75,   0.75]]), num.array([[  1.  ,   0.  ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  8.  ,   1.  ,   1.  ],
+       [ 12.  ,   0.75,   0.75]])],
+                        'full_nodes': [num.array([[  3.  ,   0.5 ,   0.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  8.  ,   1.  ,   1.  ],
+       [  9.  ,   0.25,   0.25],
+       [ 12.  ,   0.75,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
+       [  1.  ,   0.  ,   0.5 ],
+       [  2.  ,   0.  ,   1.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  9.  ,   0.25,   0.25],
+       [ 10.  ,   0.25,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
+       [  3.  ,   0.5 ,   0.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  6.  ,   1.  ,   0.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  9.  ,   0.25,   0.25],
+       [ 11.  ,   0.75,   0.25]])],
+                        'ghost_triangles': [num.array([[ 5,  0,  9,  1],
+       [ 6,  1,  9,  4],
+       [ 8,  4, 10,  1],
+       [ 9,  5, 10,  4],
+       [10,  2, 10,  5],
+       [11,  3,  9,  0],
+       [12,  3, 11,  4],
+       [13,  6, 11,  3],
+       [14,  7, 11,  6],
+       [15,  4, 11,  7]]), num.array([[ 0,  4,  9,  3],
+       [ 1,  4, 12,  5],
+       [ 2,  7, 12,  4],
+       [ 4,  5, 12,  8],
+       [11,  3,  9,  0],
+       [12,  3, 11,  4]]), num.array([[ 0,  4,  9,  3],
+       [ 1,  4, 12,  5],
+       [ 2,  7, 12,  4],
+       [ 3,  8, 12,  7],
+       [ 5,  0,  9,  1],
+       [ 6,  1,  9,  4]])],
+                        'ghost_boundary': [{(13, 1): 'ghost', (8, 0): 'ghost', (14, 1): 'ghost', \
+                        (11, 1): 'ghost', (10, 1): 'ghost', (5, 1): 'ghost', (10, 2): 'ghost'}, \
+                        {(12, 2): 'ghost', (12, 0): 'ghost', (2, 1): 'ghost', (11, 1): 'ghost',\
+                        (2, 2): 'ghost', (4, 1): 'ghost', (4, 0): 'ghost'}, {(3, 2): 'ghost', \
+                        (6, 1): 'ghost', (3, 1): 'ghost', (5, 1): 'ghost', (1, 0): 'ghost', (1, 1): 'ghost'}],
+                        'full_triangles': [[[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]], \
+                        [[0, 9, 1], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5]], \
+                        [[3, 9, 0], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7]]],
+                        'full_commun': [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, \
+                        {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, \
+                        {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}],
+                        'ghost_commun': [num.array([[ 5,  1],
+       [ 6,  1],
+       [ 8,  1],
+       [ 9,  1],
+       [10,  1],
+       [11,  2],
+       [12,  2],
+       [13,  2],
+       [14,  2],
+       [15,  2]]), num.array([[ 0,  0],
+       [ 1,  0],
+       [ 2,  0],
+       [ 4,  0],
+       [11,  2],
+       [12,  2]]), num.array([[0, 0],
+       [1, 0],
+       [2, 0],
+       [3, 0],
+       [5, 1],
+       [6, 1]])],  'ghost_quan': {'stage': [num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ]])],  'elevation': [num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ]])],  'ymomentum': [num.array([[ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]])], 'xmomentum': [num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])]},  'full_quan': {'stage': [num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]])],  'elevation': [num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]])],  'ymomentum': [num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]])],  'xmomentum': [num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])]}}
+
+
+        # Subdivide into non-overlapping partitions
+        submesh = build_submesh(nodes, triangles, boundary, quantities,
+                  triangles_per_proc, parameters = None)
+
+
+        for i in range(3):
+            assert num.allclose(true_submesh['full_triangles'][i],submesh['full_triangles'][i])
+            assert num.allclose(true_submesh['full_nodes'][i],submesh['full_nodes'][i])
+            assert num.allclose(true_submesh['ghost_triangles'][i],submesh['ghost_triangles'][i])
+            assert num.allclose(true_submesh['ghost_nodes'][i],submesh['ghost_nodes'][i])
+            assert num.allclose(true_submesh['ghost_commun'][i],submesh['ghost_commun'][i])
+
+        assert true_submesh['full_boundary'] == submesh['full_boundary']
+        assert true_submesh['full_commun'] == submesh['full_commun']
+
+        for key, value in true_submesh['ghost_quan'].iteritems():
+            for i in range(3):
+                assert num.allclose(true_submesh['ghost_quan'][key][i],submesh['ghost_quan'][key][i])
+                assert num.allclose(true_submesh['full_quan'][key][i],submesh['full_quan'][key][i])
+
+
+        # Now test the extract_submesh for the 3 processors
+
+        submesh_cell_0 = extract_submesh(submesh,triangles_per_proc,p=0)
+        submesh_cell_1 = extract_submesh(submesh,triangles_per_proc,p=1)
+        submesh_cell_2 = extract_submesh(submesh,triangles_per_proc,p=2)
+
+
+        from pprint import pprint
+
+        #pprint(submesh_cell_1)
+
+#-------------------------------------------------------------
+
+if __name__ == "__main__":
+    suite = unittest.makeSuite(Test_Distribute_Mesh,'test')
+    runner = unittest.TextTestRunner()
+    runner.run(suite)
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_failure.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_failure.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_failure.py	(revision 9424)
@@ -0,0 +1,324 @@
+import os.path
+import sys
+
+from anuga.utilities.system_tools import get_pathname_from_package
+from anuga.geometry.polygon_function import Polygon_function
+        
+from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
+from anuga.abstract_2d_finite_volumes.quantity import Quantity
+from anuga.abstract_2d_finite_volumes.util import file_function
+
+import anuga
+
+from anuga.structures.boyd_box_operator import Boyd_box_operator
+from anuga.structures.inlet_operator import Inlet_operator
+                            
+#from anuga.culvert_flows.culvert_routines import boyd_generalised_culvert_model
+     
+from math import pi, pow, sqrt
+
+import numpy as num
+#from parallel_inlet_operator import Parallel_Inlet_operator
+from anuga_parallel import distribute, myid, numprocs, finalize
+from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
+
+#from parallel_operator_factory import Inlet_operator, Boyd_box_operator
+import pypar
+import random
+
+
+"""test_that_culvert_runs_rating
+
+This test exercises the culvert and checks values outside rating curve
+are dealt with       
+"""
+verbose = True
+path = get_pathname_from_package('anuga.culvert_flows')    
+
+length = 40.
+width = 15.
+
+dx = dy = 0.5          # Resolution: Length of subdivisions on both axes
+
+#----------------------------------------------------------------------
+# Setup initial conditions
+#----------------------------------------------------------------------
+
+def topography(x, y):
+    """Set up a weir
+    
+    A culvert will connect either side
+    """
+    # General Slope of Topography
+    z=-x/1000
+    
+    N = len(x)
+    for i in range(N):
+
+       # Sloping Embankment Across Channel
+        if 5.0 < x[i] < 10.1:
+            # Cut Out Segment for Culvert face                
+            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
+               z[i]=z[i]
+            else:
+               z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
+        if 10.0 < x[i] < 12.1:
+           z[i] +=  2.5                    # Flat Crest of Embankment
+        if 12.0 < x[i] < 14.5:
+            # Cut Out Segment for Culvert face                
+            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
+               z[i]=z[i]
+            else:
+               z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
+                   
+        
+    return z
+
+filename=os.path.join(path, 'example_rating_curve.csv')
+
+line0 = [[10.0, 10.0], [30.0, 10.0]]
+#line0 = [[29.0, 10.0], [30.0, 10.0]]
+line1 = [[0.0, 10.0], [0.0, 15.0]]
+Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
+Q1 = 5.0
+
+samples = 50
+
+def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
+    success = True
+
+##-----------------------------------------------------------------------
+## Setup domain
+##-----------------------------------------------------------------------
+
+    points, vertices, boundary = rectangular_cross(int(length/dx),
+                                                   int(width/dy),
+                                                   len1=length, 
+                                                   len2=width)
+
+    domain = anuga.Domain(points, vertices, boundary)   
+    domain.set_name('Test_Parallel_Frac_Op')                 # Output name
+    domain.set_default_order(2)
+
+##-----------------------------------------------------------------------
+## Distribute domain
+##-----------------------------------------------------------------------
+
+    if parallel: domain = distribute(domain)
+    
+
+##-----------------------------------------------------------------------
+## Setup boundary conditions
+##-----------------------------------------------------------------------
+    
+    domain.set_quantity('elevation', topography) 
+    domain.set_quantity('friction', 0.01)         # Constant friction 
+    domain.set_quantity('stage',
+                        expression='elevation')   # Dry initial condition
+
+    
+    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
+    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
+    domain.set_boundary({'left': Bi, 'right': Br, 'top': Br, 'bottom': Br})
+
+
+##-----------------------------------------------------------------------
+## Determine triangle index coinciding with test points
+##-----------------------------------------------------------------------
+
+    assert(test_points is not None)
+    assert(len(test_points) == samples)
+
+    tri_ids = []
+
+    for point in test_points:
+        try:
+            k = domain.get_triangle_containing_point(point)
+            if domain.tri_full_flag[k] == 1:
+                tri_ids.append(k)
+            else:
+                tri_ids.append(-1)            
+        except:
+            tri_ids.append(-2)
+
+    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
+
+    if not parallel: control_data = []
+
+    ################ Define Fractional Operators ##########################
+
+    inlet0 = None
+    inlet1 = None
+    boyd_box0 = None
+    
+    #inlet0 = Inlet_operator(domain, line0, Q0, debug = True)
+    #inlet1 = Inlet_operator(domain, line1, Q1, debug = True)
+    
+    ## # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
+    ## if myid == 5 and parallel:
+    ##     boyd_box0 = Boyd_box_operator(domain,
+    ##                                   end_points=[[9.0, 2.5],[13.0, 2.5]],
+    ##                                   losses=1.5,
+    ##                                   width=1.0,
+    ##                                   apron=0.5,
+    ##                                   use_momentum_jet=True,
+    ##                                   use_velocity_head=False,
+    ##                                   manning=0.013,
+    ##                                   verbose=False)
+    ## elif not parallel:
+    ##     boyd_box0 = Boyd_box_operator(domain,
+    ##                                   end_points=[[9.0, 2.5],[13.0, 2.5]],
+    ##                                   losses=1.5,
+    ##                                   width=1.0,
+    ##                                   apron=0.5,
+    ##                                   use_momentum_jet=True,
+    ##                                   use_velocity_head=False,
+    ##                                   manning=0.013,
+    ##                                   verbose=False)
+
+#    if parallel:
+#        factory = Parallel_operator_factory(domain, debug = True)
+#
+#        inlet0 = factory.inlet_operator_factory(line0, Q0)
+#        inlet1 = factory.inlet_operator_factory(line1, Q1)
+#        
+#        boyd_box0 = factory.boyd_box_operator_factory(end_points=[[9.0, 2.5],[19.0, 2.5]],
+#                                          losses=1.5,
+#                                          width=1.5,
+#                                          apron=5.0,
+#                                          use_momentum_jet=True,
+#                                          use_velocity_head=False,
+#                                          manning=0.013,
+#                                          verbose=False)
+#
+#    else:
+#        inlet0 = Inlet_operator(domain, line0, Q0)
+#        inlet1 = Inlet_operator(domain, line1, Q1)
+#
+#        # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
+#        boyd_box0 = Boyd_box_operator(domain,
+#                          end_points=[[9.0, 2.5],[19.0, 2.5]],
+#                          losses=1.5,
+#                          width=1.5,
+#                          apron=5.0,
+#                          use_momentum_jet=True,
+#                          use_velocity_head=False,
+#                          manning=0.013,
+#                          verbose=False)
+    
+    #######################################################################
+
+    ##-----------------------------------------------------------------------
+    ## Evolve system through time
+    ##-----------------------------------------------------------------------
+
+    for t in domain.evolve(yieldstep = 1.0, finaltime = 4):
+        domain.write_time()
+
+        #print domain.volumetric_balance_statistics()
+    
+        stage = domain.get_quantity('stage')
+
+        #for i in range(samples):
+        #    if tri_ids[i] >= 0:                
+        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
+                    
+        sys.stdout.flush()
+ 
+        pass
+
+ 
+
+    success = True
+
+##-----------------------------------------------------------------------
+## Assign/Test Control data
+##-----------------------------------------------------------------------
+
+    if not parallel:
+        stage = domain.get_quantity('stage')
+
+        for i in range(samples):
+            assert(tri_ids[i] >= 0)
+            control_data.append(stage.centroid_values[tri_ids[i]])
+
+        if inlet0 is not None:
+            control_data.append(inlet0.inlet.get_average_stage())
+            control_data.append(inlet0.inlet.get_average_xmom())
+            control_data.append(inlet0.inlet.get_average_ymom())
+            control_data.append(inlet0.inlet.get_total_water_volume())
+            control_data.append(inlet0.inlet.get_average_depth())
+
+        if verbose: print 'P%d control_data = %s' %(myid, control_data)
+    else:
+        stage = domain.get_quantity('stage')
+        
+        for i in range(samples):
+            if tri_ids[i] >= 0:
+                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
+                success = success and local_success
+                if verbose and not local_success: 
+                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success)
+                    sys.stdout.flush()
+                    
+                
+                
+        if inlet0 is not None:
+            inlet_master_proc = inlet0.inlet.get_master_proc()
+            average_stage = inlet0.inlet.get_global_average_stage()
+            average_xmom = inlet0.inlet.get_global_average_xmom()
+            average_ymom = inlet0.inlet.get_global_average_ymom()
+            average_volume = inlet0.inlet.get_global_total_water_volume()
+            average_depth = inlet0.inlet.get_global_average_depth()
+
+            if myid == inlet_master_proc:
+                if verbose: 
+                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
+
+                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
+
+                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
+
+                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
+
+                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
+
+
+        #assert(success)
+
+    return control_data
+
+
+if __name__=="__main__":
+    
+    test_points = []
+
+    if myid == 0:
+
+        for i in range(samples):
+            x = random.randrange(0,1000)/1000.0 * length
+            y = random.randrange(0,1000)/1000.0 * width
+            point = [x, y]
+            test_points.append(point)
+        
+        for i in range(1,numprocs):
+            pypar.send(test_points, i)
+    else:
+        test_points = pypar.receive(0)
+
+    print "Test Points::"
+    print test_points
+
+    if myid == 0:
+        control_data = run_test(parallel=False, test_points = test_points, verbose = True)
+        
+        for proc in range(1,numprocs):
+            pypar.send(control_data, proc)
+    else:
+        control_data = pypar.receive(0)
+
+    pypar.barrier()
+    run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = True)
+
+
+finalize()
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_boyd_box_op_apron.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_boyd_box_op_apron.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_boyd_box_op_apron.py	(revision 9424)
@@ -0,0 +1,365 @@
+import os.path
+import sys
+
+from anuga.utilities.system_tools import get_pathname_from_package
+from anuga.geometry.polygon_function import Polygon_function
+        
+from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
+from anuga.abstract_2d_finite_volumes.quantity import Quantity
+from anuga.abstract_2d_finite_volumes.util import file_function
+
+import anuga
+
+import warnings
+warnings.simplefilter("ignore")
+
+#------------------------------------------
+# Import pypar without the initial output
+#------------------------------------------
+class NullStream:
+    def write(self,text):
+        pass
+sys.stdout = NullStream()
+import pypar
+sys.stdout = sys.__stdout__
+
+
+from math import pi, pow, sqrt
+
+import numpy as num
+from parallel_inlet_operator import Parallel_Inlet_operator
+from anuga_parallel import distribute, myid, numprocs, finalize
+from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
+
+from parallel_operator_factory import Inlet_operator, Boyd_box_operator
+
+import random
+import unittest
+
+
+"""
+
+This test exercises the parallel culvert and checks values
+"""
+verbose = False
+nprocs = 3
+    
+
+length = 40.
+width = 16.
+
+dx = dy = 4           # Resolution: Length of subdivisions on both axes
+
+#----------------------------------------------------------------------
+# Setup initial conditions
+#----------------------------------------------------------------------
+
+def topography(x, y):
+    """Set up a weir
+    
+    A culvert will connect either side
+    """
+    # General Slope of Topography
+    z=-x/1000
+    
+    N = len(x)
+    for i in range(N):
+
+       # Sloping Embankment Across Channel
+        if 5.0 < x[i] < 10.1:
+            # Cut Out Segment for Culvert face                
+            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
+               z[i]=z[i]
+            else:
+               z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
+        if 10.0 < x[i] < 12.1:
+           z[i] +=  2.5                    # Flat Crest of Embankment
+        if 12.0 < x[i] < 14.5:
+            # Cut Out Segment for Culvert face                
+            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
+               z[i]=z[i]
+            else:
+               z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
+                   
+        
+    return z
+
+#filename=os.path.join(path, 'example_rating_curve.csv')
+
+line0 = [[10.0, 10.0], [30.0, 10.0]]
+#line0 = [[29.0, 10.0], [30.0, 10.0]]
+line1 = [[0.0, 10.0], [0.0, 15.0]]
+Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
+Q1 = 5.0
+
+samples = 50
+
+def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
+    success = True
+
+##-----------------------------------------------------------------------
+## Setup domain
+##-----------------------------------------------------------------------
+
+    points, vertices, boundary = rectangular_cross(int(length/dx),
+                                                   int(width/dy),
+                                                   len1=length, 
+                                                   len2=width)
+
+    domain = anuga.Domain(points, vertices, boundary)   
+    domain.set_name('Test_Parallel_Frac_Op')                 # Output name
+    domain.set_default_order(2)
+
+##-----------------------------------------------------------------------
+## Distribute domain
+##-----------------------------------------------------------------------
+
+    if parallel:
+        domain = distribute(domain)
+        domain.dump_triangulation("frac_op_domain.png")
+    
+
+##-----------------------------------------------------------------------
+## Setup boundary conditions
+##-----------------------------------------------------------------------
+    
+    domain.set_quantity('elevation', topography) 
+    domain.set_quantity('friction', 0.01)         # Constant friction 
+    domain.set_quantity('stage',
+                        expression='elevation')   # Dry initial condition
+
+    
+    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
+    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
+    domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
+
+
+##-----------------------------------------------------------------------
+## Determine triangle index coinciding with test points
+##-----------------------------------------------------------------------
+
+    assert(test_points is not None)
+    assert(len(test_points) == samples)
+
+    tri_ids = []
+
+    for point in test_points:
+        try:
+            k = domain.get_triangle_containing_point(point)
+            if domain.tri_full_flag[k] == 1:
+                tri_ids.append(k)
+            else:
+                tri_ids.append(-1)            
+        except:
+            tri_ids.append(-2)
+
+    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
+
+    if not parallel: control_data = []
+
+    ################ Define Fractional Operators ##########################
+
+    inlet0 = None
+    inlet1 = None
+    boyd_box0 = None
+    
+    inlet0 = Inlet_operator(domain, line0, Q0, verbose = False, label = 'Inlet_0')
+    inlet1 = Inlet_operator(domain, line1, Q1, verbose = False, label = 'Inlet_1')
+    
+    # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
+    
+    boyd_box0 = Boyd_box_operator(domain,
+                                  end_points=[[9.0, 2.5],[19.0, 2.5]],
+                                  losses=1.5,
+                                  width=5.0,
+                                  apron=5.0,
+                                  use_momentum_jet=True,
+                                  use_velocity_head=False,
+                                  manning=0.013,
+                                  label='Boyd_Box_0',
+                                  verbose=False)
+        
+    #if inlet0 is not None and verbose: inlet0.print_statistics()
+    #if inlet1 is not None and verbose: inlet1.print_statistics()
+    
+    if boyd_box0 is not None and verbose:
+        print "++++", myid
+        boyd_box0.print_statistics()
+
+#    if parallel:
+#        factory = Parallel_operator_factory(domain, verbose = True)
+#
+#        inlet0 = factory.inlet_operator_factory(line0, Q0)
+#        inlet1 = factory.inlet_operator_factory(line1, Q1)
+#        
+#        boyd_box0 = factory.boyd_box_operator_factory(end_points=[[9.0, 2.5],[19.0, 2.5]],
+#                                          losses=1.5,
+#                                          width=1.5,
+#                                          apron=5.0,
+#                                          use_momentum_jet=True,
+#                                          use_velocity_head=False,
+#                                          manning=0.013,
+#                                          verbose=False)
+#
+#    else:
+#        inlet0 = Inlet_operator(domain, line0, Q0)
+#        inlet1 = Inlet_operator(domain, line1, Q1)
+#
+#        # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
+#        boyd_box0 = Boyd_box_operator(domain,
+#                          end_points=[[9.0, 2.5],[19.0, 2.5]],
+#                          losses=1.5,
+#                          width=1.5,
+#                          apron=5.0,
+#                          use_momentum_jet=True,
+#                          use_velocity_head=False,
+#                          manning=0.013,
+#                          verbose=False)
+    
+    #######################################################################
+
+    ##-----------------------------------------------------------------------
+    ## Evolve system through time
+    ##-----------------------------------------------------------------------
+
+    for t in domain.evolve(yieldstep = 2.0, finaltime = 20.0):
+        if myid == 0 and verbose:
+            domain.write_time()
+
+        #print domain.volumetric_balance_statistics()
+    
+        stage = domain.get_quantity('stage')
+
+
+        if boyd_box0 is not None and verbose :
+            if myid == boyd_box0.master_proc:
+                print 'master_proc ',myid
+                boyd_box0.print_timestepping_statistics()
+ 
+        #for i in range(samples):
+        #    if tri_ids[i] >= 0:                
+        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
+                    
+        sys.stdout.flush()
+ 
+        pass
+
+    success = True
+
+##-----------------------------------------------------------------------
+## Assign/Test Control data
+##-----------------------------------------------------------------------
+
+    if not parallel:
+        stage = domain.get_quantity('stage')
+
+        for i in range(samples):
+            assert(tri_ids[i] >= 0)
+            control_data.append(stage.centroid_values[tri_ids[i]])
+        
+        if inlet0 is not None:
+            control_data.append(inlet0.inlet.get_average_stage())
+            control_data.append(inlet0.inlet.get_average_xmom())
+            control_data.append(inlet0.inlet.get_average_ymom())
+            control_data.append(inlet0.inlet.get_total_water_volume())
+            control_data.append(inlet0.inlet.get_average_depth())
+
+        if verbose: print 'P%d control_data = %s' %(myid, control_data)
+    else:
+        stage = domain.get_quantity('stage')
+        
+        for i in range(samples):
+            if tri_ids[i] >= 0:
+                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
+                success = success and local_success
+                if verbose: 
+                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
+                
+                
+        if inlet0 is not None:
+            inlet_master_proc = inlet0.inlet.get_master_proc()
+            average_stage = inlet0.inlet.get_global_average_stage()
+            average_xmom = inlet0.inlet.get_global_average_xmom()
+            average_ymom = inlet0.inlet.get_global_average_ymom()
+            average_volume = inlet0.inlet.get_global_total_water_volume()
+            average_depth = inlet0.inlet.get_global_average_depth()
+
+            if myid == inlet_master_proc:
+                if verbose: 
+                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
+
+                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
+
+                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
+
+                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
+
+                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
+
+
+        assert(success)
+
+    return control_data
+
+
+# Test an nprocs-way run of the shallow water equations
+# against the sequential code.
+
+
+
+class Test_parallel_boyd_box_apron(unittest.TestCase):
+    def test_parallel_operator(self):
+        
+
+        #print "Expect this test to fail if not run from the parallel directory."
+        result = os.system("mpirun -np %d python test_parallel_boyd_box_op_apron.py" % nprocs)
+        assert_(result == 0)
+
+
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #pypar.finalize()
+        raise AssertionError, msg
+
+if __name__=="__main__":
+    if numprocs == 1:
+        runner = unittest.TextTestRunner()
+        suite = unittest.makeSuite(Test_parallel_boyd_box_apron, 'test')
+        #print "Running for numproc = 1"
+        runner.run(suite)
+    else:
+        #print "Running for numproc > 1"
+        pypar.barrier()
+        test_points = []
+
+        if myid == 0:
+            if verbose: print 'PARALLEL START'
+            for i in range(samples):
+                x = random.randrange(0,1000)/1000.0 * length
+                y = random.randrange(0,1000)/1000.0 * width
+                point = [x, y]
+                test_points.append(point)
+
+            for i in range(1,numprocs):
+                pypar.send(test_points, i)
+        else:
+            test_points = pypar.receive(0)
+
+        if myid == 0:
+            control_data = run_test(parallel=False, test_points = test_points, verbose = verbose)
+
+            for proc in range(1,numprocs):
+                pypar.send(control_data, proc)
+        else:
+            control_data = pypar.receive(0)
+
+
+        pypar.barrier()
+        run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = verbose)
+
+
+    finalize()
+    
+
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_boyd_box_operator.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_boyd_box_operator.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_boyd_box_operator.py	(revision 9424)
@@ -0,0 +1,367 @@
+import os.path
+import sys
+
+from anuga.utilities.system_tools import get_pathname_from_package
+from anuga.geometry.polygon_function import Polygon_function
+        
+from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
+from anuga.abstract_2d_finite_volumes.quantity import Quantity
+from anuga.abstract_2d_finite_volumes.util import file_function
+
+import anuga
+
+import warnings
+warnings.simplefilter("ignore")
+
+#from anuga.structures.boyd_box_operator import Boyd_box_operator
+#from anuga.structures.inlet_operator import Inlet_operator
+                            
+#from anuga.culvert_flows.culvert_routines import boyd_generalised_culvert_model
+
+
+#------------------------------------------
+# Import pypar without the initial output
+#------------------------------------------
+class NullStream:
+    def write(self,text):
+        pass
+sys.stdout = NullStream()
+import pypar
+sys.stdout = sys.__stdout__
+
+
+from math import pi, pow, sqrt
+
+import numpy as num
+from parallel_inlet_operator import Parallel_Inlet_operator
+from anuga_parallel import distribute, myid, numprocs, finalize
+from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
+
+from parallel_operator_factory import Inlet_operator, Boyd_box_operator
+
+import random
+import unittest
+
+
+"""
+
+This test exercises the parallel culvert and checks values
+"""
+verbose = False
+nprocs = 3
+    
+
+length = 40.
+width = 16.
+
+dx = dy = 4           # Resolution: Length of subdivisions on both axes
+
+#----------------------------------------------------------------------
+# Setup initial conditions
+#----------------------------------------------------------------------
+
+def topography(x, y):
+    """Set up a weir
+    
+    A culvert will connect either side
+    """
+    # General Slope of Topography
+    z=-x/1000
+    
+    N = len(x)
+    for i in range(N):
+
+       # Sloping Embankment Across Channel
+        if 5.0 < x[i] < 10.1:
+            # Cut Out Segment for Culvert face                
+            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
+               z[i]=z[i]
+            else:
+               z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
+        if 10.0 < x[i] < 12.1:
+           z[i] +=  2.5                    # Flat Crest of Embankment
+        if 12.0 < x[i] < 14.5:
+            # Cut Out Segment for Culvert face                
+            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
+               z[i]=z[i]
+            else:
+               z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
+                   
+        
+    return z
+
+#filename=os.path.join(path, 'example_rating_curve.csv')
+
+line0 = [[10.0, 10.0], [30.0, 10.0]]
+#line0 = [[29.0, 10.0], [30.0, 10.0]]
+line1 = [[0.0, 10.0], [0.0, 15.0]]
+Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
+Q1 = 5.0
+
+samples = 50
+
+def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
+    success = True
+
+##-----------------------------------------------------------------------
+## Setup domain
+##-----------------------------------------------------------------------
+
+    points, vertices, boundary = rectangular_cross(int(length/dx),
+                                                   int(width/dy),
+                                                   len1=length, 
+                                                   len2=width)
+
+    domain = anuga.Domain(points, vertices, boundary)   
+    domain.set_name('Test_Parallel_Frac_Op')                 # Output name
+    domain.set_default_order(2)
+
+##-----------------------------------------------------------------------
+## Distribute domain
+##-----------------------------------------------------------------------
+
+    if parallel:
+        domain = distribute(domain)
+        domain.dump_triangulation("frac_op_domain.png")
+    
+
+##-----------------------------------------------------------------------
+## Setup boundary conditions
+##-----------------------------------------------------------------------
+    
+    domain.set_quantity('elevation', topography) 
+    domain.set_quantity('friction', 0.01)         # Constant friction 
+    domain.set_quantity('stage',
+                        expression='elevation')   # Dry initial condition
+
+    
+    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
+    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
+    domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
+
+
+##-----------------------------------------------------------------------
+## Determine triangle index coinciding with test points
+##-----------------------------------------------------------------------
+
+    assert(test_points is not None)
+    assert(len(test_points) == samples)
+
+    tri_ids = []
+
+    for point in test_points:
+        try:
+            k = domain.get_triangle_containing_point(point)
+            if domain.tri_full_flag[k] == 1:
+                tri_ids.append(k)
+            else:
+                tri_ids.append(-1)            
+        except:
+            tri_ids.append(-2)
+
+    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
+
+    if not parallel: control_data = []
+
+    ################ Define Fractional Operators ##########################
+
+    inlet0 = None
+    inlet1 = None
+    boyd_box0 = None
+    
+    inlet0 = Inlet_operator(domain, line0, Q0, verbose = False, label = 'Inlet_0')
+    inlet1 = Inlet_operator(domain, line1, Q1, verbose = False, label = 'Inlet_1')
+    
+    # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
+    
+    boyd_box0 = Boyd_box_operator(domain,
+                                  end_points=[[9.0, 2.5],[19.0, 2.5]],
+                                  losses=1.5,
+                                  width=5.0,
+                                  #apron=5.,
+                                  use_momentum_jet=True,
+                                  use_velocity_head=False,
+                                  manning=0.013,
+                                  label='Boyd_Box_0',
+                                  verbose=False)
+        
+    #if inlet0 is not None and verbose: inlet0.print_statistics()
+    #if inlet1 is not None and verbose: inlet1.print_statistics()
+    
+    if boyd_box0 is not None and verbose:
+        print "++++", myid
+        boyd_box0.print_statistics()
+
+#    if parallel:
+#        factory = Parallel_operator_factory(domain, verbose = True)
+#
+#        inlet0 = factory.inlet_operator_factory(line0, Q0)
+#        inlet1 = factory.inlet_operator_factory(line1, Q1)
+#        
+#        boyd_box0 = factory.boyd_box_operator_factory(end_points=[[9.0, 2.5],[19.0, 2.5]],
+#                                          losses=1.5,
+#                                          width=1.5,
+#                                          apron=5.0,
+#                                          use_momentum_jet=True,
+#                                          use_velocity_head=False,
+#                                          manning=0.013,
+#                                          verbose=False)
+#
+#    else:
+#        inlet0 = Inlet_operator(domain, line0, Q0)
+#        inlet1 = Inlet_operator(domain, line1, Q1)
+#
+#        # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
+#        boyd_box0 = Boyd_box_operator(domain,
+#                          end_points=[[9.0, 2.5],[19.0, 2.5]],
+#                          losses=1.5,
+#                          width=1.5,
+#                          apron=5.0,
+#                          use_momentum_jet=True,
+#                          use_velocity_head=False,
+#                          manning=0.013,
+#                          verbose=False)
+    
+    #######################################################################
+
+    ##-----------------------------------------------------------------------
+    ## Evolve system through time
+    ##-----------------------------------------------------------------------
+
+    for t in domain.evolve(yieldstep = 2.0, finaltime = 20.0):
+        if myid == 0 and verbose:
+            domain.write_time()
+
+        #print domain.volumetric_balance_statistics()
+    
+        stage = domain.get_quantity('stage')
+
+
+        if boyd_box0 is not None and verbose :
+            if myid == boyd_box0.master_proc:
+                print 'master_proc ',myid
+                boyd_box0.print_timestepping_statistics()
+ 
+        #for i in range(samples):
+        #    if tri_ids[i] >= 0:                
+        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
+                    
+        sys.stdout.flush()
+ 
+        pass
+
+    success = True
+
+##-----------------------------------------------------------------------
+## Assign/Test Control data
+##-----------------------------------------------------------------------
+
+    if not parallel:
+        stage = domain.get_quantity('stage')
+
+        for i in range(samples):
+            assert(tri_ids[i] >= 0)
+            control_data.append(stage.centroid_values[tri_ids[i]])
+        
+        if inlet0 is not None:
+            control_data.append(inlet0.inlet.get_average_stage())
+            control_data.append(inlet0.inlet.get_average_xmom())
+            control_data.append(inlet0.inlet.get_average_ymom())
+            control_data.append(inlet0.inlet.get_total_water_volume())
+            control_data.append(inlet0.inlet.get_average_depth())
+
+        if verbose: print 'P%d control_data = %s' %(myid, control_data)
+    else:
+        stage = domain.get_quantity('stage')
+        
+        for i in range(samples):
+            if tri_ids[i] >= 0:
+                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
+                success = success and local_success
+                if verbose: 
+                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
+                
+                
+        if inlet0 is not None:
+            inlet_master_proc = inlet0.inlet.get_master_proc()
+            average_stage = inlet0.inlet.get_global_average_stage()
+            average_xmom = inlet0.inlet.get_global_average_xmom()
+            average_ymom = inlet0.inlet.get_global_average_ymom()
+            average_volume = inlet0.inlet.get_global_total_water_volume()
+            average_depth = inlet0.inlet.get_global_average_depth()
+
+            if myid == inlet_master_proc:
+                if verbose: 
+                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
+
+                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
+
+                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
+
+                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
+
+                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
+
+
+        assert(success)
+
+    return control_data
+
+
+# Test an nprocs-way run of the shallow water equations
+# against the sequential code.
+
+class Test_parallel_boyd_box_operator(unittest.TestCase):
+    def test_parallel_operator(self):
+        #print "Expect this test to fail if not run from the parallel directory."
+        result = os.system("mpirun -np %d python test_parallel_boyd_box_operator.py" % nprocs)
+        assert_(result == 0)
+
+
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #pypar.finalize()
+        raise AssertionError, msg
+
+if __name__=="__main__":
+    if numprocs == 1:
+        runner = unittest.TextTestRunner()
+        suite = unittest.makeSuite(Test_parallel_boyd_box_operator, 'test')
+        #print "Running for numproc = 1"
+        runner.run(suite)
+    else:
+        #print "Running for numproc > 1"
+        pypar.barrier()
+        test_points = []
+
+        if myid == 0:
+            if verbose: print 'PARALLEL START'
+            for i in range(samples):
+                x = random.randrange(0,1000)/1000.0 * length
+                y = random.randrange(0,1000)/1000.0 * width
+                point = [x, y]
+                test_points.append(point)
+
+            for i in range(1,numprocs):
+                pypar.send(test_points, i)
+        else:
+            test_points = pypar.receive(0)
+
+        if myid == 0:
+            control_data = run_test(parallel=False, test_points = test_points, verbose = verbose)
+
+            for proc in range(1,numprocs):
+                pypar.send(control_data, proc)
+        else:
+            control_data = pypar.receive(0)
+
+
+        pypar.barrier()
+        run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = verbose)
+
+
+    finalize()
+    
+
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_boyd_pipe_operator.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_boyd_pipe_operator.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_boyd_pipe_operator.py	(revision 9424)
@@ -0,0 +1,356 @@
+import os.path
+import sys
+
+from anuga.utilities.system_tools import get_pathname_from_package
+from anuga.geometry.polygon_function import Polygon_function
+        
+from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
+from anuga.abstract_2d_finite_volumes.quantity import Quantity
+from anuga.abstract_2d_finite_volumes.util import file_function
+
+import anuga
+
+#------------------------------------------
+# Import pypar without the initial output
+#------------------------------------------
+class NullStream:
+    def write(self,text):
+        pass
+sys.stdout = NullStream()
+import pypar
+sys.stdout = sys.__stdout__
+
+
+from math import pi, pow, sqrt
+
+import numpy as num
+
+#from parallel_inlet_operator import Parallel_Inlet_operator
+
+from anuga_parallel import distribute, myid, numprocs, finalize
+from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
+
+from parallel_operator_factory import Inlet_operator
+from parallel_operator_factory import Boyd_pipe_operator
+
+import random
+import unittest
+
+
+"""
+
+This test exercises the parallel culvert and checks values
+"""
+verbose = False
+nprocs = 3
+    
+
+length = 40.
+width = 16.
+
+dx = dy = 2           # Resolution: Length of subdivisions on both axes
+
+#----------------------------------------------------------------------
+# Setup initial conditions
+#----------------------------------------------------------------------
+
+def topography(x, y):
+    """Set up a weir
+    
+    A culvert will connect either side
+    """
+    # General Slope of Topography
+    z=-x/1000
+    
+    N = len(x)
+    for i in range(N):
+
+       # Sloping Embankment Across Channel
+        if 5.0 < x[i] < 10.1:
+            # Cut Out Segment for Culvert face                
+            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
+               z[i]=z[i]
+            else:
+               z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
+        if 10.0 < x[i] < 12.1:
+           z[i] +=  2.5                    # Flat Crest of Embankment
+        if 12.0 < x[i] < 14.5:
+            # Cut Out Segment for Culvert face                
+            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
+               z[i]=z[i]
+            else:
+               z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
+                   
+        
+    return z
+
+#filename=os.path.join(path, 'example_rating_curve.csv')
+
+line0 = [[10.0, 10.0], [30.0, 10.0]]
+#line0 = [[29.0, 10.0], [30.0, 10.0]]
+line1 = [[0.0, 10.0], [0.0, 15.0]]
+Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
+Q1 = 5.0
+
+samples = 50
+
+def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
+    success = True
+
+##-----------------------------------------------------------------------
+## Setup domain
+##-----------------------------------------------------------------------
+
+    points, vertices, boundary = rectangular_cross(int(length/dx),
+                                                   int(width/dy),
+                                                   len1=length, 
+                                                   len2=width)
+
+    domain = anuga.Domain(points, vertices, boundary)   
+    domain.set_name('Test_Parallel_Boyd_Pipe_Op')                 # Output name
+    domain.set_default_order(2)
+
+##-----------------------------------------------------------------------
+## Distribute domain
+##-----------------------------------------------------------------------
+
+    if parallel:
+        domain = distribute(domain)
+        domain.dump_triangulation("boyd_pipe_domain.png")
+    
+
+##-----------------------------------------------------------------------
+## Setup boundary conditions
+##-----------------------------------------------------------------------
+    
+    domain.set_quantity('elevation', topography) 
+    domain.set_quantity('friction', 0.01)         # Constant friction 
+    domain.set_quantity('stage',
+                        expression='elevation')   # Dry initial condition
+
+    
+    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
+    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
+    domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
+
+
+##-----------------------------------------------------------------------
+## Determine triangle index coinciding with test points
+##-----------------------------------------------------------------------
+
+    assert(test_points is not None)
+    assert(len(test_points) == samples)
+
+    tri_ids = []
+
+    for point in test_points:
+        try:
+            k = domain.get_triangle_containing_point(point)
+            if domain.tri_full_flag[k] == 1:
+                tri_ids.append(k)
+            else:
+                tri_ids.append(-1)            
+        except:
+            tri_ids.append(-2)
+
+    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
+
+    if not parallel: control_data = []
+
+    ################ Define Fractional Operators ##########################
+
+    inlet0 = None
+    inlet1 = None
+    boyd_pipe0 = None
+    
+    inlet0 = Inlet_operator(domain, line0, Q0, verbose = False)
+    inlet1 = Inlet_operator(domain, line1, Q1, verbose = False)
+    
+    # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
+    
+    boyd_pipe0 = Boyd_pipe_operator(domain,
+                                  end_points=[[9.0, 2.5],[19.0, 2.5]],
+                                  losses=1.5,
+                                  diameter=5.0,
+                                  apron=5.0,
+                                  use_momentum_jet=True,
+                                  use_velocity_head=False,
+                                  manning=0.013,
+                                  verbose=False)
+        
+    if inlet0 is not None and verbose: inlet0.print_statistics()
+    if inlet1 is not None and verbose: inlet1.print_statistics()
+    if boyd_pipe0 is not None and verbose: boyd_pipe0.print_statistics()
+
+#    if parallel:
+#        factory = Parallel_operator_factory(domain, verbose = True)
+#
+#        inlet0 = factory.inlet_operator_factory(line0, Q0)
+#        inlet1 = factory.inlet_operator_factory(line1, Q1)
+#        
+#        boyd_box0 = factory.boyd_box_operator_factory(end_points=[[9.0, 2.5],[19.0, 2.5]],
+#                                          losses=1.5,
+#                                          width=1.5,
+#                                          apron=5.0,
+#                                          use_momentum_jet=True,
+#                                          use_velocity_head=False,
+#                                          manning=0.013,
+#                                          verbose=False)
+#
+#    else:
+#        inlet0 = Inlet_operator(domain, line0, Q0)
+#        inlet1 = Inlet_operator(domain, line1, Q1)
+#
+#        # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
+#        boyd_box0 = Boyd_box_operator(domain,
+#                          end_points=[[9.0, 2.5],[19.0, 2.5]],
+#                          losses=1.5,
+#                          width=1.5,
+#                          apron=5.0,
+#                          use_momentum_jet=True,
+#                          use_velocity_head=False,
+#                          manning=0.013,
+#                          verbose=False)
+    
+    #######################################################################
+
+    ##-----------------------------------------------------------------------
+    ## Evolve system through time
+    ##-----------------------------------------------------------------------
+
+    for t in domain.evolve(yieldstep = 2.0, finaltime = 20.0):
+        if myid == 0 and verbose:
+            domain.write_time()
+
+        #print domain.volumetric_balance_statistics()
+    
+        stage = domain.get_quantity('stage')
+
+
+        if boyd_pipe0 is not None and verbose : boyd_pipe0.print_timestepping_statistics()
+ 
+        #for i in range(samples):
+        #    if tri_ids[i] >= 0:                
+        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
+                    
+        sys.stdout.flush()
+ 
+        pass
+
+    success = True
+
+##-----------------------------------------------------------------------
+## Assign/Test Control data
+##-----------------------------------------------------------------------
+
+    if not parallel:
+        stage = domain.get_quantity('stage')
+
+        for i in range(samples):
+            assert(tri_ids[i] >= 0)
+            control_data.append(stage.centroid_values[tri_ids[i]])
+        
+        if inlet0 is not None:
+            control_data.append(inlet0.inlet.get_average_stage())
+            control_data.append(inlet0.inlet.get_average_xmom())
+            control_data.append(inlet0.inlet.get_average_ymom())
+            control_data.append(inlet0.inlet.get_total_water_volume())
+            control_data.append(inlet0.inlet.get_average_depth())
+
+        if verbose: print 'P%d control_data = %s' %(myid, control_data)
+    else:
+        stage = domain.get_quantity('stage')
+        
+        for i in range(samples):
+            if tri_ids[i] >= 0:
+                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
+                success = success and local_success
+                if verbose: 
+                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
+                
+                
+        if inlet0 is not None:
+            inlet_master_proc = inlet0.inlet.get_master_proc()
+            average_stage = inlet0.inlet.get_global_average_stage()
+            average_xmom = inlet0.inlet.get_global_average_xmom()
+            average_ymom = inlet0.inlet.get_global_average_ymom()
+            average_volume = inlet0.inlet.get_global_total_water_volume()
+            average_depth = inlet0.inlet.get_global_average_depth()
+
+            if myid == inlet_master_proc:
+                if verbose: 
+                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
+
+                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
+
+                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
+
+                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
+
+                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
+
+
+        assert(success)
+
+    if parallel:
+        finalize()
+
+    return control_data
+
+
+# Test an nprocs-way run of the shallow water equations
+# against the sequential code.
+
+class Test_parallel_boyd_pipe_op(unittest.TestCase):
+    def test_parallel_operator(self):
+        #print "Expect this test to fail if not run from the parallel directory."
+        result = os.system("mpirun -np %d python test_parallel_boyd_pipe_operator.py" % nprocs)
+        assert_(result == 0)
+
+
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #pypar.finalize()
+        raise AssertionError, msg
+
+if __name__=="__main__":
+    if numprocs == 1:
+        runner = unittest.TextTestRunner()
+        suite = unittest.makeSuite(Test_parallel_boyd_pipe_op, 'test')
+        #print "Running for numproc = 1"
+        runner.run(suite)
+    else:
+        #print "Running for numproc > 1"
+        pypar.barrier()
+        test_points = []
+
+        if myid == 0:
+            if verbose: print 'PARALLEL START'
+            for i in range(samples):
+                x = random.randrange(0,1000)/1000.0 * length
+                y = random.randrange(0,1000)/1000.0 * width
+                point = [x, y]
+                test_points.append(point)
+
+            for i in range(1,numprocs):
+                pypar.send(test_points, i)
+        else:
+            test_points = pypar.receive(0)
+
+        if myid == 0:
+            control_data = run_test(parallel=False, test_points = test_points, verbose = verbose)
+
+            for proc in range(1,numprocs):
+                pypar.send(control_data, proc)
+        else:
+            control_data = pypar.receive(0)
+
+
+        pypar.barrier()
+        run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = verbose)
+
+
+    
+
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_distribute_domain.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_distribute_domain.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_distribute_domain.py	(revision 9424)
@@ -0,0 +1,238 @@
+
+"""Test a run of the sequential shallow water domain against
+a run of the parallel shallow water domain.
+
+WARNING: This assumes that the command to run jobs is mpirun.
+Tested with MPICH and LAM (Ole)
+"""
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+
+import unittest
+import os
+import sys
+
+
+
+import numpy as num
+
+#------------------------------------------
+# Import pypar without the initial output
+#------------------------------------------
+class NullStream:
+    def write(self,text):
+        pass
+sys.stdout = NullStream()
+import pypar
+sys.stdout = sys.__stdout__
+
+
+#------------------------------------------
+# anuga imports
+#------------------------------------------
+
+from anuga.utilities.numerical_tools import ensure_numeric
+from anuga.utilities.util_ext        import double_precision
+from anuga.utilities.norms           import l1_norm, l2_norm, linf_norm
+
+from anuga import Domain
+from anuga import Reflective_boundary
+from anuga import Dirichlet_boundary
+from anuga import Time_boundary
+from anuga import Transmissive_boundary
+
+from anuga import rectangular_cross
+from anuga import create_domain_from_file
+
+
+from anuga_parallel import distribute, myid, numprocs, finalize
+
+
+#--------------------------------------------------------------------------
+# Setup parameters
+#--------------------------------------------------------------------------
+
+mesh_filename = "merimbula_10785_1.tsh"
+#mesh_filename = "test-100.tsh"
+yieldstep = 1
+finaltime = 20
+quantity = 'stage'
+nprocs = 4
+verbose = False
+
+#--------------------------------------------------------------------------
+# Setup procedures
+#--------------------------------------------------------------------------
+class Set_Stage:
+    """Set an initial condition with constant water height, for x<x0
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))
+
+#--------------------------------------------------------------------------
+# Setup test
+#--------------------------------------------------------------------------
+def evolution_test(parallel=False):
+
+
+    domain = create_domain_from_file(mesh_filename)
+    domain.set_quantity('stage', Set_Stage(756000.0, 756500.0, 2.0))
+
+    #--------------------------------------------------------------------------
+    # Create parallel domain if requested
+    #--------------------------------------------------------------------------
+
+    if parallel:
+        if myid == 0 and verbose: print 'DISTRIBUTING PARALLEL DOMAIN'
+        domain = distribute(domain)
+
+    #------------------------------------------------------------------------------
+    # Setup boundary conditions
+    # This must currently happen *after* domain has been distributed
+    #------------------------------------------------------------------------------
+    domain.store = False
+    Br = Reflective_boundary(domain)      # Solid reflective wall
+
+    domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Br})
+
+    #------------------------------------------------------------------------------
+    # Setup diagnostic arrays
+    #------------------------------------------------------------------------------
+    l1list = []
+    l2list = []
+    linflist = []
+    l1norm = num.zeros(3, num.float)
+    l2norm = num.zeros(3, num.float)
+    linfnorm = num.zeros(3, num.float)
+    recv_norm = num.zeros(3, num.float)
+
+    #------------------------------------------------------------------------------
+    # Evolution
+    #------------------------------------------------------------------------------
+    if parallel:
+        if myid == 0 and verbose: print 'PARALLEL EVOLVE'
+    else:
+        if verbose: print 'SEQUENTIAL EVOLVE'
+        
+    for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+        edges = domain.quantities[quantity].edge_values.take(num.flatnonzero(domain.tri_full_flag),axis=0)
+        l1norm[0] = l1_norm(edges[:,0])
+        l1norm[1] = l1_norm(edges[:,1])
+        l1norm[2] = l1_norm(edges[:,2])
+        l2norm[0] = l2_norm(edges[:,0])
+        l2norm[1] = l2_norm(edges[:,1])
+        l2norm[2] = l2_norm(edges[:,2])
+        linfnorm[0] = linf_norm(edges[:,0])
+        linfnorm[1] = linf_norm(edges[:,1])
+        linfnorm[2] = linf_norm(edges[:,2])
+        if parallel:
+            l2norm[0] = pow(l2norm[0], 2)
+            l2norm[1] = pow(l2norm[1], 2)
+            l2norm[2] = pow(l2norm[2], 2)
+            if myid == 0:
+                #domain.write_time()
+
+                #print edges[:,1]            
+                for p in range(1, numprocs):
+                    recv_norm = pypar.receive(p)
+                    l1norm += recv_norm
+                    recv_norm = pypar.receive(p)
+                    l2norm += recv_norm
+                    recv_norm = pypar.receive(p)
+                    linfnorm[0] = max(linfnorm[0], recv_norm[0])
+                    linfnorm[1] = max(linfnorm[1], recv_norm[1])
+                    linfnorm[2] = max(linfnorm[2], recv_norm[2])
+
+                l2norm[0] = pow(l2norm[0], 0.5)
+                l2norm[1] = pow(l2norm[1], 0.5)
+                l2norm[2] = pow(l2norm[2], 0.5)
+
+                l1list.append(l1norm)                
+                l2list.append(l2norm)
+                linflist.append(linfnorm)                
+            else:
+                pypar.send(l1norm, 0)
+                pypar.send(l2norm, 0)
+                pypar.send(linfnorm, 0)
+        else:
+            #domain.write_time()
+            l1list.append(l1norm)                
+            l2list.append(l2norm)
+            linflist.append(linfnorm)
+            
+
+    return (l1list, l2list, linflist)
+
+# Test an nprocs-way run of the shallow water equations
+# against the sequential code.
+
+class Test_parallel_distribute_domain(unittest.TestCase):
+    def test_parallel_distribute_domain(self):
+        #print "Expect this test to fail if not run from the parallel directory."
+        result = os.system("mpirun -np %d python test_parallel_distribute_domain.py" % nprocs)
+        assert_(result == 0)
+
+
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #pypar.finalize()
+        raise AssertionError, msg
+
+if __name__=="__main__":
+    if numprocs == 1: 
+        runner = unittest.TextTestRunner()
+        suite = unittest.makeSuite(Test_parallel_distribute_domain, 'test')
+        runner.run(suite)
+    else:
+
+        pypar.barrier()
+        if myid == 0:
+            if verbose: print 'SEQUENTIAL START'
+            l1norm_seq, l2norm_seq, linfnorm_seq = evolution_test(parallel=False)
+
+        pypar.barrier()
+        if myid ==0:
+            if verbose: print 'PARALLEL START'
+        
+        l1norm_par, l2norm_par, linfnorm_par = evolution_test(parallel=True)
+        
+        if myid == 0:
+            assert_(len(l1norm_seq) == len(l1norm_par))
+            assert_(len(l2norm_seq) == len(l2norm_par))
+            assert_(len(linfnorm_seq) == len(linfnorm_par))
+            assert_(len(l1norm_seq) == len(l2norm_seq))
+            assert_(len(l2norm_seq) == len(linfnorm_seq))
+            # Anything smaller than tol we consider to be 0.
+            # This usualy comes out to be 10^-14 (DBL_DIG = 10^-15)
+            # * 10 to cater for rounding error in computation.
+            tol = pow(10, -1 * (double_precision() - 1))
+            for x in range(len(l1norm_seq)):
+                for y in range(3):
+                    # Calculate relative difference in the norms
+                    assert_(abs(l1norm_seq[x][y] - l1norm_par[x][y])/l1norm_seq[x][y] < tol)
+                    assert_(abs(l2norm_seq[x][y] - l2norm_par[x][y])/l2norm_seq[x][y] < tol)
+                    assert_(abs(linfnorm_seq[x][y] - linfnorm_par[x][y])/linfnorm_seq[x][y] < tol)
+                    if x > 0:
+                        # Verify that the quantity is being conserved across iterations.
+                        assert_(abs(l1norm_seq[x][y] - l1norm_seq[x-1][y]) < tol)
+                        assert_(abs(l2norm_seq[x][y] - l2norm_seq[x-1][y]) < tol)
+                        assert_(abs(linfnorm_seq[x][y] - linfnorm_seq[x-1][y]) < tol)
+                        assert_(abs(l1norm_par[x][y] - l1norm_par[x-1][y]) < tol)
+                        assert_(abs(l2norm_par[x][y] - l2norm_par[x-1][y]) < tol)
+                        assert_(abs(linfnorm_par[x][y] - linfnorm_par[x-1][y]) < tol)
+                
+            if verbose: print 'Parallel test OK'
+
+
+
+    finalize()
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_distribute_mesh.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_distribute_mesh.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_distribute_mesh.py	(revision 9424)
@@ -0,0 +1,326 @@
+#!/usr/bin/env python
+
+import unittest
+import sys
+import os
+from math import sqrt
+
+	
+from anuga import Domain
+from anuga import rectangular_cross
+
+from anuga_parallel.distribute_mesh import pmesh_divide_metis
+from anuga_parallel.distribute_mesh import build_submesh
+from anuga_parallel.distribute_mesh import submesh_full, submesh_ghost, submesh_quantities
+from anuga_parallel.distribute_mesh import extract_submesh, rec_submesh, send_submesh
+
+from anuga_parallel import myid, numprocs, barrier, finalize
+
+import numpy as num
+
+
+def topography(x,y): 
+    return -x/2
+
+
+def xcoord(x,y):
+    return x
+
+def ycoord(x,y):
+    return y
+
+
+def distibute_three_processors():
+    """
+    Do a parallel test of distributing a rectangle onto 3 processors
+    """
+
+    import pypar
+
+    myid = pypar.rank()
+    numprocs = pypar.size()
+
+    if not numprocs == 3: return
+
+    #print numprocs
+
+    barrier()
+
+    if myid == 0:
+
+        points, vertices, boundary = rectangular_cross(2,2)
+
+        domain = Domain(points, vertices, boundary)
+
+
+        domain.set_quantity('elevation', topography) # Use function for elevation
+        domain.set_quantity('friction', 0.0)         # Constant friction 
+        domain.set_quantity('stage', expression='elevation') # Dry initial stage
+        domain.set_quantity('xmomentum', expression='friction + 2.0') # 
+        domain.set_quantity('ymomentum', ycoord) #
+
+        #----------------------------------------------------------------------------------
+        # Test pmesh_divide_metis
+        #----------------------------------------------------------------------------------
+        nodes, triangles, boundary, triangles_per_proc, quantities = pmesh_divide_metis(domain,numprocs)
+
+        assert_(num.allclose(nodes,points))
+
+        true_vertices = [[0, 9, 1], [3, 9, 0], [4, 9, 3], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]]
+
+        true_triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7]]
+
+        assert_(num.allclose(vertices,true_vertices))
+        assert_(num.allclose(triangles,true_triangles))
+
+        assert_(num.allclose(triangles_per_proc,[5,6,5]))
+
+
+        #----------------------------------------------------------------------------------
+        # Test build_submesh
+        #----------------------------------------------------------------------------------
+        submesh = build_submesh(nodes, triangles, boundary, quantities, triangles_per_proc)
+
+
+        assert_(num.allclose(submesh['full_nodes'][0],[[3.0, 0.5, 0.0], [4.0, 0.5, 0.5], [5.0, 0.5, 1.0], [7.0, 1.0, 0.5], [8.0, 1.0, 1.0], [9.0, 0.25, 0.25], [12.0, 0.75, 0.75]]))
+        assert_(num.allclose(submesh['full_nodes'][1],[[0.0, 0.0, 0.0], [1.0, 0.0, 0.5], [2.0, 0.0, 1.0], [4.0, 0.5, 0.5], [5.0, 0.5, 1.0], [9.0, 0.25, 0.25], [10.0, 0.25, 0.75]]))
+        assert_(num.allclose(submesh['full_nodes'][2],[[0.0, 0.0, 0.0], [3.0, 0.5, 0.0], [4.0, 0.5, 0.5], [6.0, 1.0, 0.0], [7.0, 1.0, 0.5], [9.0, 0.25, 0.25], [11.0, 0.75, 0.25]]))
+
+
+        assert_(num.allclose(submesh['ghost_nodes'][0],[[0.0, 0.0, 0.0], [1.0, 0.0, 0.5], [2.0, 0.0, 1.0], [6.0, 1.0, 0.0], [10.0, 0.25, 0.75], [11.0, 0.75, 0.25]]))
+        assert_(num.allclose(submesh['ghost_nodes'][1],[[3.0, 0.5, 0.0], [7.0, 1.0, 0.5], [8.0, 1.0, 1.0], [11.0, 0.75, 0.25], [12.0, 0.75, 0.75]]))
+        assert_(num.allclose(submesh['ghost_nodes'][2],[[1.0, 0.0, 0.5], [5.0, 0.5, 1.0], [8.0, 1.0, 1.0], [12.0, 0.75, 0.75]]))
+
+
+
+        true_full_triangles = [num.array([[ 4,  9,  3],
+                                          [ 4, 12,  5],
+                                          [ 7, 12,  4],
+                                          [ 8, 12,  7],
+                                          [ 5, 12,  8]]),
+                               num.array([[ 0,  9,  1],
+                                          [ 1,  9,  4],
+                                          [ 1, 10,  2],
+                                          [ 4, 10,  1],
+                                          [ 5, 10,  4],
+                                          [ 2, 10,  5]]),
+                               num.array([[ 3,  9,  0],
+                                          [ 3, 11,  4],
+                                          [ 6, 11,  3],
+                                          [ 7, 11,  6],
+                                          [ 4, 11,  7]])]
+
+
+        assert_(num.allclose(submesh['full_triangles'][0],true_full_triangles[0]))
+        assert_(num.allclose(submesh['full_triangles'][1],true_full_triangles[1]))
+        assert_(num.allclose(submesh['full_triangles'][2],true_full_triangles[2]))
+
+        true_ghost_triangles = [num.array([[ 5,  0,  9,  1],
+                                           [ 6,  1,  9,  4],
+                                           [ 8,  4, 10,  1],
+                                           [ 9,  5, 10,  4],
+                                           [10,  2, 10,  5],
+                                           [11,  3,  9,  0],
+                                           [12,  3, 11,  4],
+                                           [13,  6, 11,  3],
+                                           [14,  7, 11,  6],
+                                           [15,  4, 11,  7]]),
+                                num.array([[ 0,  4,  9,  3],
+                                           [ 1,  4, 12,  5],
+                                           [ 2,  7, 12,  4],
+                                           [ 4,  5, 12,  8],
+                                           [11,  3,  9,  0],
+                                           [12,  3, 11,  4]]),
+                                num.array([[ 0,  4,  9,  3],
+                                           [ 1,  4, 12,  5],
+                                           [ 2,  7, 12,  4],
+                                           [ 3,  8, 12,  7],
+                                           [ 5,  0,  9,  1],
+                                           [ 6,  1,  9,  4]])]
+
+
+
+        assert_(num.allclose(submesh['ghost_triangles'][0],true_ghost_triangles[0]))
+        assert_(num.allclose(submesh['ghost_triangles'][1],true_ghost_triangles[1]))
+        assert_(num.allclose(submesh['ghost_triangles'][2],true_ghost_triangles[2]))
+
+        true_full_commun = [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}]
+
+        assert_(true_full_commun == submesh['full_commun'])
+
+
+        true_ghost_commun = [num.array([[ 5,  1],
+                                        [ 6,  1],
+                                        [ 8,  1],
+                                        [ 9,  1],
+                                        [10,  1],
+                                        [11,  2],
+                                        [12,  2],
+                                        [13,  2],
+                                        [14,  2],
+                                        [15,  2]]),
+                             num.array([[ 0,  0],
+                                        [ 1,  0],
+                                        [ 2,  0],
+                                        [ 4,  0],
+                                        [11,  2],
+                                        [12,  2]]),
+                             num.array([[0, 0],
+                                        [1, 0],
+                                        [2, 0],
+                                        [3, 0],
+                                        [5, 1],
+                                        [6, 1]])]
+
+        assert_(num.allclose(submesh['ghost_commun'][0],true_ghost_commun[0]))
+        assert_(num.allclose(submesh['ghost_commun'][1],true_ghost_commun[1]))
+        assert_(num.allclose(submesh['ghost_commun'][2],true_ghost_commun[2]))
+
+
+
+    barrier()
+    #--------------------------------
+    # Now do the comunnication part
+    #--------------------------------
+
+
+    if myid == 0:
+        #----------------------------------------------------------------------------------
+        # Test send_submesh
+        #----------------------------------------------------------------------------------
+        for p in range(1, numprocs):
+            send_submesh(submesh, triangles_per_proc, p, verbose=False)
+
+        #----------------------------------------------------------------------------------
+        # Test extract_submesh
+        #----------------------------------------------------------------------------------
+        points, vertices, boundary, quantities, \
+                ghost_recv_dict, full_send_dict, tri_map, node_map, tri_l2g, node_l2g, \
+                ghost_layer_width =\
+        extract_submesh(submesh, triangles_per_proc)
+
+
+        true_points =  [[0.5, 0.0], [0.5, 0.5], [0.5, 1.0], [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.75, 0.75], [0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [1.0, 0.0], [0.25, 0.75], [0.75, 0.25]]
+
+        true_vertices = [[1, 5, 0], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [7, 5, 8], [8, 5, 1], [1, 11, 8], [2, 11, 1], [9, 11, 2], [0, 5, 7], [0, 12, 1], [10, 12, 0], [3, 12, 10], [1, 12, 3]]
+
+
+        true_ghost_recv = {1: [num.array([5, 6, 7, 8, 9]), num.array([ 5,  6,  8,  9, 10])], 2: [num.array([10, 11, 12, 13, 14]), num.array([11, 12, 13, 14, 15])]}
+
+
+        true_full_send = {1: [num.array([0, 1, 2, 4]), num.array([0, 1, 2, 4])], 2: [num.array([0, 1, 2, 3]), num.array([0, 1, 2, 3])]}
+
+        assert_(num.allclose(ghost_layer_width,  2))
+        assert_(num.allclose(points,   true_points))
+        assert_(num.allclose(vertices, true_vertices))
+        assert_(num.allclose(ghost_recv_dict[1],true_ghost_recv[1]))
+        assert_(num.allclose(ghost_recv_dict[2],true_ghost_recv[2]))
+        assert_(num.allclose(full_send_dict[1],true_full_send[1]))
+        assert_(num.allclose(full_send_dict[2],true_full_send[2]))
+
+        #print triangles_per_proc
+
+    else:
+        #----------------------------------------------------------------------------------
+        # Test rec_submesh
+        #----------------------------------------------------------------------------------
+        points, vertices, boundary, quantities, \
+                ghost_recv_dict, full_send_dict, \
+                no_full_nodes, no_full_trigs, tri_map, node_map, tri_l2g, node_l2g, \
+                ghost_layer_width  = \
+                rec_submesh(0, verbose=False)    
+
+        if myid == 1:
+
+
+            true_points =  [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.5], [0.5, 1.0], [0.25, 0.25], [0.25, 0.75], [0.5, 0.0], [1.0, 0.5], [1.0, 1.0], [0.75, 0.25], [0.75, 0.75]] 
+
+            true_vertices =  [[0, 5, 1], [1, 5, 3], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [3, 5, 7], [3, 11, 4], [8, 11, 3], [4, 11, 9], [7, 5, 0], [7, 10, 3]]
+
+            true_ghost_recv =  {0: [num.array([6, 7, 8, 9]), num.array([0, 1, 2, 4])], 2: [num.array([10, 11]), num.array([11, 12])]}
+
+            true_full_send =  {0: [num.array([0, 1, 3, 4, 5]), num.array([ 5,  6,  8,  9, 10])], 2: [num.array([0, 1]), num.array([5, 6])]}
+
+            true_tri_map  = num.array([ 6,  7,  8, -1,  9,  0, 1,  2,  3,  4,  5, 10, 11])
+
+            true_node_map = num.array([ 0,  1,  2,  7,  3,  4, -1,  8,  9,  5,  6, 10, 11])
+
+            assert_(num.allclose(ghost_layer_width,  2))
+            assert_(num.allclose(tri_map,   true_tri_map))
+            assert_(num.allclose(node_map,   true_node_map))
+            assert_(num.allclose(points,   true_points))
+            assert_(num.allclose(vertices, true_vertices))
+            assert_(num.allclose(ghost_recv_dict[0],true_ghost_recv[0]))
+            assert_(num.allclose(ghost_recv_dict[2],true_ghost_recv[2]))
+            assert_(num.allclose(full_send_dict[0],true_full_send[0]))
+            assert_(num.allclose(full_send_dict[2],true_full_send[2])) 
+
+
+        if myid == 2:
+
+            true_points =   [[0.0, 0.0], [0.5, 0.0], [0.5, 0.5], [1.0, 0.0], [1.0, 0.5], [0.25, 0.25], [0.75, 0.25], [0.0, 0.5], [0.5, 1.0], [1.0, 1.0], [0.75, 0.75]]
+
+            true_vertices =  [[1, 5, 0], [1, 6, 2], [3, 6, 1], [4, 6, 3], [2, 6, 4], [2, 5, 1], [2, 10, 8], [4, 10, 2], [9, 10, 4], [0, 5, 7], [7, 5, 2]]
+
+            true_ghost_recv =   {0: [num.array([5, 6, 7, 8]), num.array([0, 1, 2, 3])], 1: [num.array([ 9, 10]), num.array([5, 6])]}
+
+            true_full_send =   {0: [num.array([0, 1, 2, 3, 4]), num.array([11, 12, 13, 14, 15])], 1: [num.array([0, 1]), num.array([11, 12])]}
+
+            true_tri_map  = num.array([5, 6, 7, 8, -1, 9, 10, -1, -1, -1, -1, 0, 1, 2, 3, 4, -1])
+
+            true_node_map = num.array([ 0,  7, -1,  1,  2,  8 , 3,  4,  9,  5, -1,  6, 10])
+
+            assert_(num.allclose(ghost_layer_width,  2))
+            assert_(num.allclose(tri_map,   true_tri_map))
+            assert_(num.allclose(node_map,   true_node_map))
+            assert_(num.allclose(points,   true_points))
+            assert_(num.allclose(vertices, true_vertices))
+            assert_(num.allclose(ghost_recv_dict[0],true_ghost_recv[0]))
+            assert_(num.allclose(ghost_recv_dict[1],true_ghost_recv[1]))
+            assert_(num.allclose(full_send_dict[0],true_full_send[0]))
+            assert_(num.allclose(full_send_dict[1],true_full_send[1])) 
+
+
+
+
+
+
+###############################################################
+
+class Test_parallel_distribute_mesh(unittest.TestCase):
+
+    def test_distribute_three_processors(self):
+        # Expect this test to fail if not run from the parallel directory.
+        result = os.system("mpirun -np 3 python test_parallel_distribute_mesh.py")
+        assert_(result == 0)
+
+
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #import pypar
+        #pypar.finalize()
+        raise AssertionError, msg
+        #import sys
+        #sys.exit(1)
+    
+
+
+
+
+
+
+#-------------------------------------------------------------
+if __name__=="__main__":
+    if numprocs == 1: 
+        runner = unittest.TextTestRunner()
+        suite = unittest.makeSuite(Test_parallel_distribute_mesh, 'test')
+        runner.run(suite)
+    else:
+        distibute_three_processors()
+
+    finalize()
+        
+
+        
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_file_boundary.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_file_boundary.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_file_boundary.py	(revision 9424)
@@ -0,0 +1,505 @@
+
+"""Run parallel shallow water domain.
+
+   run using command like:
+
+   mpirun -np m python run_parallel_sw_merimbula.py
+
+   where m is the number of processors to be used.
+   
+   Will produce sww files with names domain_Pn_m.sww where m is number of processors and
+   n in [0, m-1] refers to specific processor that owned this part of the partitioned mesh.
+"""
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+
+import os
+import sys
+import time
+import pypar
+import numpy as num
+import unittest
+import tempfile
+from struct import pack, unpack
+from anuga.file.netcdf import NetCDFFile
+import copy
+
+#------------------------
+# ANUGA Modules
+#------------------------
+from anuga.utilities.numerical_tools import ensure_numeric
+from anuga.utilities.util_ext        import double_precision
+from anuga.utilities.norms           import l1_norm, l2_norm, linf_norm
+	
+from anuga import Domain
+from anuga import Reflective_boundary
+from anuga import Dirichlet_boundary
+from anuga import Time_boundary
+from anuga import Transmissive_boundary
+from anuga import File_boundary
+
+from mux import WAVEHEIGHT_MUX2_LABEL, EAST_VELOCITY_MUX2_LABEL, \
+                NORTH_VELOCITY_MUX2_LABEL
+from mux import read_mux2_py
+from anuga.file_conversion.urs2sts import urs2sts
+from anuga.file.urs import Read_urs
+from anuga.file.sts import create_sts_boundary
+
+from anuga.utilities.numerical_tools import ensure_numeric
+from anuga.coordinate_transforms.redfearn import redfearn
+from anuga.coordinate_transforms.geo_reference import Geo_reference
+
+from anuga import rectangular_cross_domain
+from anuga.pmesh.mesh_interface import create_mesh_from_regions
+from anuga import create_domain_from_file
+
+from anuga_parallel import distribute, myid, numprocs, send, receive, barrier, finalize
+
+from anuga.file.test_mux import Test_Mux
+
+verbose = False
+
+
+class Test_urs2sts_parallel(Test_Mux):
+    """ A suite of tests to test urs2sts file conversion functions.
+        These tests are quite coarse-grained: converting a file
+        and checking that its headers and some of its contents
+        are correct.
+    """ 
+
+    def sequential_test_time_varying_file_boundary_sts(self):
+        """sequential_ltest_time_varying_file_boundary_sts_sequential(self):
+        Read correct points from ordering file and apply sts to boundary. The boundary is time varying. FIXME add to test_urs2sts.
+        """
+        lat_long_points=[[6.01,97.0],[6.02,97.0],[6.05,96.9],[6.0,97.0]]
+        bounding_polygon=[[6.0,97.0],[6.01,97.0],[6.02,97.0],
+                          [6.02,97.02],[6.00,97.02]]
+        tide = 3.0
+        time_step_count = 65
+        time_step = 2.
+        n=len(lat_long_points)
+        first_tstep=num.ones(n,num.int)
+        last_tstep=(time_step_count)*num.ones(n,num.int)
+        finaltime=num.float(time_step*(time_step_count-1))
+        yieldstep=num.float(time_step)
+        gauge_depth=20*num.ones(n,num.float)
+        ha=2*num.ones((n,time_step_count),num.float)
+        ua=10*num.ones((n,time_step_count),num.float)
+        va=-10*num.ones((n,time_step_count),num.float)
+
+        times=num.arange(0., num.float(time_step_count*time_step), time_step)
+        for i in range(n):
+            #ha[i]+=num.sin(times)
+            ha[i]+=times/finaltime
+
+
+
+        sts_file="test"
+        if myid==0:
+            base_name, files = self.write_mux2(lat_long_points,
+                                               time_step_count,
+                                               time_step,
+                                               first_tstep,
+                                               last_tstep,
+                                               depth=gauge_depth,
+                                               ha=ha,
+                                               ua=ua,
+                                               va=va)
+            # base name will not exist, but 3 other files are created
+
+            # Write order file
+            file_handle, order_base_name = tempfile.mkstemp("")
+            os.close(file_handle)
+            os.remove(order_base_name)
+            d=","
+            order_file=order_base_name+'order.txt'
+            fid=open(order_file,'w')
+        
+            # Write Header
+            header='index, longitude, latitude\n'
+            fid.write(header)
+            indices=[3,0,1]
+            for i in indices:
+                line=str(i)+d+str(lat_long_points[i][1])+d+\
+                    str(lat_long_points[i][0])+"\n"
+                fid.write(line)
+            fid.close()
+
+            urs2sts(base_name,
+                    basename_out=sts_file,
+                    ordering_filename=order_file,
+                    mean_stage=tide,
+                    verbose=verbose)
+            self.delete_mux(files)
+
+            assert(os.access(sts_file+'.sts', os.F_OK))
+
+            os.remove(order_file)
+
+        barrier()
+        boundary_polygon = create_sts_boundary(sts_file)
+
+        # Append the remaining part of the boundary polygon to be defined by
+        # the user
+        bounding_polygon_utm=[]
+        for point in bounding_polygon:
+            zone,easting,northing=redfearn(point[0],point[1])
+            bounding_polygon_utm.append([easting,northing])
+
+        boundary_polygon.append(bounding_polygon_utm[3])
+        boundary_polygon.append(bounding_polygon_utm[4])
+
+        assert num.allclose(bounding_polygon_utm,boundary_polygon)
+
+
+        extent_res=1000000
+        meshname = 'urs_test_mesh' + '.tsh'
+        interior_regions=None
+        boundary_tags={'ocean': [0,1], 'otherocean': [2,3,4]}
+        
+        # have to change boundary tags from last example because now bounding
+        # polygon starts in different place.
+        if myid==0:
+            create_mesh_from_regions(boundary_polygon,
+                                     boundary_tags=boundary_tags,
+                                     maximum_triangle_area=extent_res,
+                                     filename=meshname,
+                                     interior_regions=interior_regions,
+                                     verbose=verbose)
+
+        barrier()
+        
+        domain_fbound = Domain(meshname)
+        domain_fbound.set_quantities_to_be_stored(None)
+        domain_fbound.set_quantity('stage', tide)
+        if verbose: print "Creating file boundary condition"
+        Bf = File_boundary(sts_file+'.sts',
+                           domain_fbound,
+                           boundary_polygon=boundary_polygon)
+        Br = Reflective_boundary(domain_fbound)
+
+        domain_fbound.set_boundary({'ocean': Bf,'otherocean': Br})
+
+        temp_fbound=num.zeros(int(finaltime/yieldstep)+1,num.float)
+        if verbose: print "Evolving domain with file boundary condition"
+        for i, t in enumerate(domain_fbound.evolve(yieldstep=yieldstep,
+                                                   finaltime=finaltime, 
+                                                   skip_initial_step = False)):
+            temp_fbound[i]=domain_fbound.quantities['stage'].centroid_values[2]
+            if verbose: domain_fbound.write_time()
+            
+        
+        domain_drchlt = Domain(meshname)
+        domain_drchlt.set_quantities_to_be_stored(None)
+        domain_drchlt.set_starttime(time_step)
+        domain_drchlt.set_quantity('stage', tide)
+        Br = Reflective_boundary(domain_drchlt)
+        #Bd = Dirichlet_boundary([2.0+tide,220+10*tide,-220-10*tide])
+        Bd = Time_boundary(domain=domain_drchlt, f=lambda t: [2.0+t/finaltime+tide,220.+10.*tide+10.*t/finaltime,-220.-10.*tide-10.*t/finaltime])
+        #Bd = Time_boundary(domain=domain_drchlt,f=lambda t: [2.0+num.sin(t)+tide,10.*(2+20.+num.sin(t)+tide),-10.*(2+20.+num.sin(t)+tide)])
+        domain_drchlt.set_boundary({'ocean': Bd,'otherocean': Br})
+        temp_drchlt=num.zeros(int(finaltime/yieldstep)+1,num.float)
+        
+        for i, t in enumerate(domain_drchlt.evolve(yieldstep=yieldstep,
+                                                   finaltime=finaltime, 
+                                                   skip_initial_step = False)):
+            temp_drchlt[i]=domain_drchlt.quantities['stage'].centroid_values[2]
+            #domain_drchlt.write_time()
+        
+        #print domain_fbound.quantities['stage'].vertex_values
+        #print domain_drchlt.quantities['stage'].vertex_values
+                    
+        assert num.allclose(temp_fbound,temp_drchlt),temp_fbound-temp_drchlt
+
+        
+        assert num.allclose(domain_fbound.quantities['stage'].vertex_values,
+                            domain_drchlt.quantities['stage'].vertex_values)
+                        
+        assert num.allclose(domain_fbound.quantities['xmomentum'].vertex_values,
+                            domain_drchlt.quantities['xmomentum'].vertex_values)                        
+                        
+        assert num.allclose(domain_fbound.quantities['ymomentum'].vertex_values,
+                            domain_drchlt.quantities['ymomentum'].vertex_values)
+        
+        if not sys.platform == 'win32':
+            if myid==0: os.remove(sts_file+'.sts')
+        
+        if myid==0: os.remove(meshname)
+
+    def parallel_test_time_varying_file_boundary_sts(self):
+        """ parallel_test_time_varying_file_boundary_sts_sequential(self):
+            Read correct points from ordering file and apply sts to boundary. 
+            The boundary is time varying. Compares sequential result with 
+            distributed result found using anuga_parallel
+        """
+
+        #------------------------------------------------------------
+        # Define test variables
+        #------------------------------------------------------------
+        lat_long_points=[[6.01,97.0],[6.02,97.0],[6.05,96.9],[6.0,97.0]]
+        bounding_polygon=[[6.0,97.0],[6.01,97.0],[6.02,97.0],
+                          [6.02,97.02],[6.00,97.02]]
+        tide = 3.0
+        time_step_count = 65
+        time_step = 2
+        n=len(lat_long_points)
+        first_tstep=num.ones(n,num.int)
+        last_tstep=(time_step_count)*num.ones(n,num.int)
+        finaltime=num.float(time_step*(time_step_count-1))
+        yieldstep=num.float(time_step)
+        gauge_depth=20*num.ones(n,num.float)
+        ha=2*num.ones((n,time_step_count),num.float)
+        ua=10*num.ones((n,time_step_count),num.float)
+        va=-10*num.ones((n,time_step_count),num.float)
+
+        times=num.arange(0, time_step_count*time_step, time_step)
+        for i in range(n):
+            #ha[i]+=num.sin(times)
+            ha[i]+=times/finaltime
+
+        #------------------------------------------------------------
+        # Write mux data to file then convert to sts format
+        #------------------------------------------------------------
+        sts_file="test"
+        if myid==0:
+            base_name, files = self.write_mux2(lat_long_points,
+                                               time_step_count,
+                                               time_step,
+                                               first_tstep,
+                                               last_tstep,
+                                               depth=gauge_depth,
+                                               ha=ha,
+                                               ua=ua,
+                                               va=va)
+            # base name will not exist, but 3 other files are created
+
+            # Write order file
+            file_handle, order_base_name = tempfile.mkstemp("")
+            os.close(file_handle)
+            os.remove(order_base_name)
+            d=","
+            order_file=order_base_name+'order.txt'
+            fid=open(order_file,'w')
+        
+            # Write Header
+            header='index, longitude, latitude\n'
+            fid.write(header)
+            indices=[3,0,1]
+            for i in indices:
+                line=str(i)+d+str(lat_long_points[i][1])+d+\
+                    str(lat_long_points[i][0])+"\n"
+                fid.write(line)
+            fid.close()
+
+            urs2sts(base_name,
+                    basename_out=sts_file,
+                    ordering_filename=order_file,
+                    mean_stage=tide,
+                    verbose=verbose)
+            self.delete_mux(files)
+
+            assert(os.access(sts_file+'.sts', os.F_OK))
+
+            os.remove(order_file)
+
+        barrier()
+        #------------------------------------------------------------
+        # Define boundary_polygon on each processor. This polygon defines the
+        # urs boundary and lies on a portion of the bounding_polygon
+        #------------------------------------------------------------
+        boundary_polygon = create_sts_boundary(sts_file)
+
+        # Append the remaining part of the boundary polygon to be defined by
+        # the user
+        bounding_polygon_utm=[]
+        for point in bounding_polygon:
+            zone,easting,northing=redfearn(point[0],point[1])
+            bounding_polygon_utm.append([easting,northing])
+
+        boundary_polygon.append(bounding_polygon_utm[3])
+        boundary_polygon.append(bounding_polygon_utm[4])
+
+
+        assert num.allclose(bounding_polygon_utm,boundary_polygon)
+
+        extent_res=10000
+        meshname = 'urs_test_mesh' + '.tsh'
+        interior_regions=None
+        boundary_tags={'ocean': [0,1], 'otherocean': [2,3,4]}
+        
+        #------------------------------------------------------------
+        # Create mesh on the master processor and store in file. This file
+        # is read in by each slave processor when needed
+        #------------------------------------------------------------
+        if myid==0:
+            create_mesh_from_regions(boundary_polygon,
+                                     boundary_tags=boundary_tags,
+                                     maximum_triangle_area=extent_res,
+                                     filename=meshname,
+                                     interior_regions=interior_regions,
+                                     verbose=verbose)
+        
+
+            # barrier()
+            domain_fbound = Domain(meshname)
+            domain_fbound.set_quantities_to_be_stored(None)
+            domain_fbound.set_quantity('stage', tide)
+            # print domain_fbound.mesh.get_boundary_polygon()
+        else:
+            domain_fbound=None
+
+        barrier()
+        if ( verbose and myid == 0 ): 
+            print 'DISTRIBUTING PARALLEL DOMAIN'
+        domain_fbound = distribute(domain_fbound)
+
+        #--------------------------------------------------------------------
+        # Find which sub_domain in which the interpolation points are located 
+        #
+        # Sometimes the interpolation points sit exactly
+        # between two centroids, so in the parallel run we
+        # reset the interpolation points to the centroids
+        # found in the sequential run
+        #--------------------------------------------------------------------
+        interpolation_points = [[279000,664000], [280250,664130], 
+                                    [279280,665400], [280500,665000]]
+
+        interpolation_points=num.array(interpolation_points)
+
+        #if myid==0:
+        #    import pylab as P
+        #    boundary_polygon=num.array(boundary_polygon)
+        #    P.plot(boundary_polygon[:,0],boundary_polygon[:,1])
+        #    P.plot(interpolation_points[:,0],interpolation_points[:,1],'ko')
+        #    P.show()
+
+        fbound_gauge_values = []
+        fbound_proc_tri_ids = []
+        for i, point in enumerate(interpolation_points):
+            fbound_gauge_values.append([]) # Empty list for timeseries
+
+            try:
+                k = domain_fbound.get_triangle_containing_point(point)
+                if domain_fbound.tri_full_flag[k] == 1:
+                    fbound_proc_tri_ids.append(k)
+                else:
+                    fbound_proc_tri_ids.append(-1)            
+            except:
+                fbound_proc_tri_ids.append(-2)
+
+
+        if verbose: print 'P%d has points = %s' %(myid, fbound_proc_tri_ids)
+
+        #------------------------------------------------------------
+        # Set boundary conditions
+        #------------------------------------------------------------
+        Bf = File_boundary(sts_file+'.sts',
+                           domain_fbound,
+                           boundary_polygon=boundary_polygon)
+        Br = Reflective_boundary(domain_fbound)
+    
+        domain_fbound.set_boundary({'ocean': Bf,'otherocean': Br})
+
+        #------------------------------------------------------------
+        # Evolve the domain on each processor
+        #------------------------------------------------------------  
+        for i, t in enumerate(domain_fbound.evolve(yieldstep=yieldstep,
+                                                   finaltime=finaltime, 
+                                                   skip_initial_step = False)):
+
+            stage = domain_fbound.get_quantity('stage')
+            for i in range(4):
+                if fbound_proc_tri_ids[i] > -1:
+                    fbound_gauge_values[i].append(stage.centroid_values[fbound_proc_tri_ids[i]])
+        
+        #------------------------------------------------------------
+        # Create domain to be run sequntially on each processor
+        #------------------------------------------------------------
+        domain_drchlt = Domain(meshname)
+        domain_drchlt.set_quantities_to_be_stored(None)
+        domain_drchlt.set_starttime(time_step)
+        domain_drchlt.set_quantity('stage', tide)
+        Br = Reflective_boundary(domain_drchlt)
+        #Bd = Dirichlet_boundary([2.0+tide,220+10*tide,-220-10*tide])
+        Bd = Time_boundary(domain=domain_drchlt, function=lambda t: [2.0+t/finaltime+tide,220.+10.*tide+10.*t/finaltime,-220.-10.*tide-10.*t/finaltime])
+        #Bd = Time_boundary(domain=domain_drchlt,function=lambda t: [2.0+num.sin(t)+tide,10.*(2+20.+num.sin(t)+tide),-10.*(2+20.+num.sin(t)+tide)])
+        domain_drchlt.set_boundary({'ocean': Bd,'otherocean': Br})
+       
+        drchlt_gauge_values = []
+        drchlt_proc_tri_ids = []
+        for i, point in enumerate(interpolation_points):
+            drchlt_gauge_values.append([]) # Empty list for timeseries
+
+            try:
+                k = domain_drchlt.get_triangle_containing_point(point)
+                if domain_drchlt.tri_full_flag[k] == 1:
+                    drchlt_proc_tri_ids.append(k)
+                else:
+                    drchlt_proc_tri_ids.append(-1)            
+            except:
+                drchlt_proc_tri_ids.append(-2)
+
+
+        if verbose: print 'P%d has points = %s' %(myid, drchlt_proc_tri_ids)
+
+        #------------------------------------------------------------
+        # Evolve entire domain on each processor
+        #------------------------------------------------------------
+        for i, t in enumerate(domain_drchlt.evolve(yieldstep=yieldstep,
+                                                   finaltime=finaltime, 
+                                                   skip_initial_step = False)):
+
+            stage = domain_drchlt.get_quantity('stage')
+            for i in range(4):
+                drchlt_gauge_values[i].append(stage.centroid_values[drchlt_proc_tri_ids[i]])
+
+        #------------------------------------------------------------
+        # Compare sequential values with parallel values
+        #------------------------------------------------------------
+        barrier()
+        success = True
+        for i in range(4):
+            if fbound_proc_tri_ids[i] > -1:
+                fbound_gauge_values[i]=num.array(fbound_gauge_values[i])
+                drchlt_gauge_values[i]=num.array(drchlt_gauge_values[i])
+                #print i,fbound_gauge_values[i][4]
+                #print i,drchlt_gauge_values[i][4]
+                success = success and num.allclose(fbound_gauge_values[i], drchlt_gauge_values[i])
+                assert success#, (fbound_gauge_values[i]-drchlt_gauge_values[i])
+
+        #assert_(success)       
+
+        if not sys.platform == 'win32':
+            if myid==0: os.remove(sts_file+'.sts')
+        
+        if myid==0: os.remove(meshname)
+    
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #pypar.finalize()
+        raise AssertionError, msg
+
+
+# Test an nprocs-way run of the shallow water equations
+# against the sequential code.
+
+if __name__=="__main__":
+    #verbose=False
+    if myid ==0 and verbose: 
+        print 'PARALLEL START'
+    suite = unittest.makeSuite(Test_urs2sts_parallel,'parallel_test')
+    #suite = unittest.makeSuite(Test_urs2sts_parallel,'sequential_test')
+    runner = unittest.TextTestRunner()
+    runner.run(suite)
+
+    #------------------------------------------
+    # Run the code code and compare sequential
+    # results at 4 gauge stations
+    #------------------------------------------
+
+    
+    finalize()
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_frac_op.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_frac_op.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_frac_op.py	(revision 9424)
@@ -0,0 +1,338 @@
+import os.path
+import sys
+
+from anuga.utilities.system_tools import get_pathname_from_package
+from anuga.geometry.polygon_function import Polygon_function
+        
+from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
+from anuga.abstract_2d_finite_volumes.quantity import Quantity
+from anuga.abstract_2d_finite_volumes.util import file_function
+
+import anuga
+
+import warnings
+warnings.simplefilter("ignore")
+
+
+#------------------------------------------
+# Import pypar without the initial output
+#------------------------------------------
+class NullStream:
+    def write(self,text):
+        pass
+sys.stdout = NullStream()
+import pypar
+sys.stdout = sys.__stdout__
+
+
+from math import pi, pow, sqrt
+
+import numpy as num
+from parallel_inlet_operator import Parallel_Inlet_operator
+from anuga_parallel import distribute, myid, numprocs, finalize
+from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
+
+from anuga import Inlet_operator, Boyd_box_operator
+
+import random
+import unittest
+
+
+"""
+
+This test exercises the parallel culvert and checks values
+"""
+
+verbose = False
+nprocs = 3
+    
+
+length = 40.
+width = 16.
+
+dx = dy = 2           # Resolution: Length of subdivisions on both axes
+
+#----------------------------------------------------------------------
+# Setup initial conditions
+#----------------------------------------------------------------------
+
+def topography(x, y):
+    """Set up a weir
+    
+    A culvert will connect either side
+    """
+    # General Slope of Topography
+    z=-x/1000
+    
+    N = len(x)
+    for i in range(N):
+
+        # Sloping Embankment Across Channel
+        if 5.0 < x[i] < 10.1:
+            # Cut Out Segment for Culvert face                
+            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
+                z[i]=z[i]
+            else:
+                z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
+        if 10.0 < x[i] < 12.1:
+            z[i] +=  2.5                    # Flat Crest of Embankment
+        if 12.0 < x[i] < 14.5:
+            # Cut Out Segment for Culvert face                
+            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
+                z[i]=z[i]
+            else:
+                z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
+                   
+    return z
+
+#filename=os.path.join(path, 'example_rating_curve.csv')
+
+line0 = [[10.0, 10.0], [30.0, 10.0]]
+#line0 = [[29.0, 10.0], [30.0, 10.0]]
+line1 = [[0.0, 10.0], [0.0, 15.0]]
+Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
+Q1 = 5.0
+
+samples = 50
+
+def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
+    success = True
+
+##-----------------------------------------------------------------------
+## Setup domain
+##-----------------------------------------------------------------------
+
+    points, vertices, boundary = rectangular_cross(int(length/dx),
+                                                   int(width/dy),
+                                                   len1=length, 
+                                                   len2=width)
+
+    domain = anuga.Domain(points, vertices, boundary)   
+    domain.set_name('Test_Parallel_Frac_Op')                 # Output name
+    domain.set_default_order(2)
+
+##-----------------------------------------------------------------------
+## Distribute domain
+##-----------------------------------------------------------------------
+
+    if parallel:
+        domain = distribute(domain)
+        domain.dump_triangulation("frac_op_domain.png")
+    
+
+##-----------------------------------------------------------------------
+## Setup boundary conditions
+##-----------------------------------------------------------------------
+    
+    domain.set_quantity('elevation', topography) 
+    domain.set_quantity('friction', 0.01)         # Constant friction 
+    domain.set_quantity('stage',
+                        expression='elevation')   # Dry initial condition
+
+    
+    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
+    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
+    domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
+
+
+##-----------------------------------------------------------------------
+## Determine triangle index coinciding with test points
+##-----------------------------------------------------------------------
+
+    assert(test_points is not None)
+    assert(len(test_points) == samples)
+
+    tri_ids = []
+
+    for point in test_points:
+        try:
+            k = domain.get_triangle_containing_point(point)
+            if domain.tri_full_flag[k] == 1:
+                tri_ids.append(k)
+            else:
+                tri_ids.append(-1)            
+        except:
+            tri_ids.append(-2)
+
+    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
+
+    if not parallel: control_data = []
+
+    ################ Define Fractional Operators ##########################
+
+    inlet0 = None
+    inlet1 = None
+    boyd_box0 = None
+    
+    inlet0 = Inlet_operator(domain, line0, Q0, logging=True, description='inlet0', verbose = False)
+    inlet1 = Inlet_operator(domain, line1, Q1, logging=True, description='inlet1', verbose = False)
+    
+    # Enquiry point [ 19.    2.5] is contained in two domains in 4 proc case
+    
+    boyd_box0 = Boyd_box_operator(domain,
+                                  end_points=[[9.0, 2.5],[19.0, 2.5]],
+                                  losses=1.5,
+                                  width=5.0,
+                                  #apron=5.0,
+                                  use_momentum_jet=True,
+                                  use_velocity_head=False,
+                                  manning=0.013,
+                                  logging=True,
+                                  description='boyd_box_0',
+                                  verbose=False)
+        
+    if inlet0 is not None and verbose: inlet0.print_statistics()
+    if inlet1 is not None and verbose: inlet1.print_statistics()
+    if boyd_box0 is not None and verbose: boyd_box0.print_statistics()
+
+
+    ##-----------------------------------------------------------------------
+    ## Evolve system through time
+    ##-----------------------------------------------------------------------
+
+    for t in domain.evolve(yieldstep = 2.0, finaltime = 20.0):
+        if myid == 0 and verbose:
+            domain.write_time()
+
+        #print domain.volumetric_balance_statistics()
+    
+        stage = domain.get_quantity('stage')
+
+
+        if boyd_box0 is not None and verbose : boyd_box0.print_timestepping_statistics()
+ 
+        #for i in range(samples):
+        #    if tri_ids[i] >= 0:                
+        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
+                    
+        sys.stdout.flush()
+ 
+        pass
+
+    success = True
+
+##-----------------------------------------------------------------------
+## Assign/Test Control data
+##-----------------------------------------------------------------------
+
+    if not parallel:
+        stage = domain.get_quantity('stage')
+
+        for i in range(samples):
+            assert(tri_ids[i] >= 0)
+            control_data.append(stage.centroid_values[tri_ids[i]])
+        
+        if inlet0 is not None:
+            control_data.append(inlet0.inlet.get_average_stage())
+            control_data.append(inlet0.inlet.get_average_xmom())
+            control_data.append(inlet0.inlet.get_average_ymom())
+            control_data.append(inlet0.inlet.get_total_water_volume())
+            control_data.append(inlet0.inlet.get_average_depth())
+
+        if verbose: print 'P%d control_data = %s' %(myid, control_data)
+    else:
+        stage = domain.get_quantity('stage')
+        
+        for i in range(samples):
+            if tri_ids[i] >= 0:
+                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
+                success = success and local_success
+                if verbose: 
+                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
+                if not local_success:
+                    print 'Ouput P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
+
+
+        #assert success        
+                
+        if inlet0 is not None:
+            inlet_master_proc = inlet0.inlet.get_master_proc()
+            average_stage = inlet0.inlet.get_global_average_stage()
+            average_xmom = inlet0.inlet.get_global_average_xmom()
+            average_ymom = inlet0.inlet.get_global_average_ymom()
+            average_volume = inlet0.inlet.get_global_total_water_volume()
+            average_depth = inlet0.inlet.get_global_average_depth()
+
+            if myid == inlet_master_proc:
+                if verbose: 
+                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
+
+                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
+
+                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
+
+                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
+
+                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
+
+
+        #assert(success)
+
+    return control_data, success
+
+
+# Test an nprocs-way run of the shallow water equations
+# against the sequential code.
+
+class Test_parallel_frac_op(unittest.TestCase):
+    def test_parallel_frac_op(self):
+        #print "Expect this test to fail if not run from the parallel directory."
+        result = os.system("mpirun -np %d python test_parallel_frac_op.py" % nprocs)
+        assert_(result == 0)
+
+
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #pypar.finalize()
+        raise AssertionError, msg
+
+if __name__=="__main__":
+
+    if numprocs == 1:
+        runner = unittest.TextTestRunner()
+        suite = unittest.makeSuite(Test_parallel_frac_op, 'test')
+        #print "Running for numproc = 1"
+        runner.run(suite)
+    else:
+        #print "Running for numproc > 1"
+        pypar.barrier()
+        test_points = []
+
+        if myid == 0:
+            if verbose: print 'PARALLEL START'
+            random.seed(2)
+            for i in range(samples):
+                x = random.randrange(0,1000)/1000.0 * length
+                y = random.randrange(0,1000)/1000.0 * width
+                point = [x, y]
+                test_points.append(point)
+
+            for i in range(1,numprocs):
+                pypar.send(test_points, i)
+        else:
+            test_points = pypar.receive(0)
+
+        if myid == 0:
+            control_data, success  = run_test(parallel=False, test_points = test_points, verbose = verbose)
+
+            for proc in range(1,numprocs):
+                pypar.send(control_data, proc)
+        else:
+            control_data = pypar.receive(0)
+
+
+        pypar.barrier()
+        _, success = run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = verbose)
+
+        sys.stdout.flush()
+
+        pypar.barrier()
+
+        msg = 'Discrepency between sequential and parallel runs on P%g' % myid
+        assert success, msg
+
+    finalize()
+    
+
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_inlet_operator.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_inlet_operator.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_inlet_operator.py	(revision 9424)
@@ -0,0 +1,318 @@
+import os.path
+import sys
+
+from anuga.utilities.system_tools import get_pathname_from_package
+from anuga.geometry.polygon_function import Polygon_function
+        
+from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
+from anuga.abstract_2d_finite_volumes.quantity import Quantity
+from anuga.abstract_2d_finite_volumes.util import file_function
+
+import anuga
+
+#from anuga.structures.boyd_box_operator import Boyd_box_operator
+#from anuga.structures.inlet_operator import Inlet_operator
+                            
+#from anuga.culvert_flows.culvert_routines import boyd_generalised_culvert_model
+
+
+import warnings
+warnings.simplefilter('ignore')
+
+#------------------------------------------
+# Import pypar without the initial output
+#------------------------------------------
+class NullStream:
+    def write(self,text):
+        pass
+sys.stdout = NullStream()
+import pypar
+sys.stdout = sys.__stdout__
+
+
+from math import pi, pow, sqrt
+
+import numpy as num
+from parallel_inlet_operator import Parallel_Inlet_operator
+from anuga_parallel import distribute, myid, numprocs, finalize
+from anuga.geometry.polygon import inside_polygon, is_inside_polygon, line_intersect
+
+from parallel_operator_factory import Inlet_operator
+
+import random
+import unittest
+
+
+"""
+
+This test exercises the parallel culvert and checks values
+"""
+verbose = False
+nprocs = 3
+    
+
+length = 40.
+width = 16.
+
+dx = dy = 2.0          # Resolution: Length of subdivisions on both axes
+
+#----------------------------------------------------------------------
+# Setup initial conditions
+#----------------------------------------------------------------------
+
+def topography(x, y):
+    """Set up a weir
+    
+    A culvert will connect either side
+    """
+    # General Slope of Topography
+    z=-x/1000
+    
+    N = len(x)
+    for i in range(N):
+
+        # Sloping Embankment Across Channel
+        if 5.0 < x[i] < 10.1:
+            # Cut Out Segment for Culvert face                
+            if  1.0+(x[i]-5.0)/5.0 <  y[i]  < 4.0 - (x[i]-5.0)/5.0: 
+                z[i]=z[i]
+            else:
+                z[i] +=  0.5*(x[i] -5.0)    # Sloping Segment  U/S Face
+        if 10.0 < x[i] < 12.1:
+            z[i] +=  2.5                    # Flat Crest of Embankment
+        if 12.0 < x[i] < 14.5:
+            # Cut Out Segment for Culvert face                
+            if  2.0-(x[i]-12.0)/2.5 <  y[i]  < 3.0 + (x[i]-12.0)/2.5:
+                z[i]=z[i]
+            else:
+                z[i] +=  2.5-1.0*(x[i] -12.0) # Sloping D/S Face
+                   
+        
+    return z
+
+#filename=os.path.join(path, 'example_rating_curve.csv')
+
+line0 = [[10.0, 10.0], [30.0, 10.0]]
+#line0 = [[29.0, 10.0], [30.0, 10.0]]
+line1 = [[0.0, 10.0], [0.0, 15.0]]
+Q0 = file_function('test_hydrograph.tms', quantities=['hydrograph'])
+Q1 = 5.0
+
+samples = 50
+
+def run_test(parallel = False, control_data = None, test_points = None, verbose = False):
+    success = True
+
+    ##-----------------------------------------------------------------------
+    ## Setup domain
+    ##-----------------------------------------------------------------------
+
+    points, vertices, boundary = rectangular_cross(int(length/dx),
+                                                   int(width/dy),
+                                                   len1=length, 
+                                                   len2=width)
+
+    domain = anuga.Domain(points, vertices, boundary)   
+    domain.set_name('Test_Parallel_Frac_Op')                 # Output name
+    domain.set_default_order(2)
+
+    ##-----------------------------------------------------------------------
+    ## Distribute domain
+    ##-----------------------------------------------------------------------
+
+    if parallel:
+        domain = distribute(domain)
+        domain.dump_triangulation("frac_op_domain.png")
+    
+
+    ##-----------------------------------------------------------------------
+    ## Setup boundary conditions
+    ##-----------------------------------------------------------------------
+    
+    domain.set_quantity('elevation', topography) 
+    domain.set_quantity('friction', 0.01)         # Constant friction 
+    domain.set_quantity('stage',
+                        expression='elevation')   # Dry initial condition
+
+    
+    Bi = anuga.Dirichlet_boundary([5.0, 0.0, 0.0])
+    Br = anuga.Reflective_boundary(domain)              # Solid reflective wall
+    domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br})
+
+
+    ##-----------------------------------------------------------------------
+    ## Determine triangle index coinciding with test points
+    ##-----------------------------------------------------------------------
+
+    assert(test_points is not None)
+    assert(len(test_points) == samples)
+
+    tri_ids = []
+
+    for point in test_points:
+        try:
+            k = domain.get_triangle_containing_point(point)
+            if domain.tri_full_flag[k] == 1:
+                tri_ids.append(k)
+            else:
+                tri_ids.append(-1)            
+        except:
+            tri_ids.append(-2)
+
+    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
+
+    if not parallel: control_data = []
+
+    ################ Define Fractional Operators ##########################
+
+    inlet0 = None
+    inlet1 = None
+    boyd_box0 = None
+    
+    inlet0 = Inlet_operator(domain, line0, Q0, verbose = False, default=0.0)
+    inlet1 = Inlet_operator(domain, line1, Q1, verbose = False)
+    
+        
+    if inlet0 is not None and verbose: inlet0.print_statistics()
+    if inlet1 is not None and verbose: inlet1.print_statistics()
+    #if boyd_box0 is not None and verbose: boyd_box0.print_statistics()
+
+
+    ##-----------------------------------------------------------------------
+    ## Evolve system through time
+    ##-----------------------------------------------------------------------
+
+    for t in domain.evolve(yieldstep = 2.0, finaltime = 40.0):
+        if myid == 0 and verbose:
+            domain.write_time()
+
+        #print domain.volumetric_balance_statistics()
+    
+        stage = domain.get_quantity('stage')
+
+
+        #if boyd_box0 is not None and verbose : boyd_box0.print_timestepping_statistics()
+ 
+        #for i in range(samples):
+        #    if tri_ids[i] >= 0:                
+        #        if verbose: print 'P%d tri %d, value = %s' %(myid, i, stage.centroid_values[tri_ids[i]])
+                    
+        sys.stdout.flush()
+ 
+        pass
+
+    success = True
+
+    ##-----------------------------------------------------------------------
+    ## Assign/Test Control data
+    ##-----------------------------------------------------------------------
+
+    if not parallel:
+        stage = domain.get_quantity('stage')
+
+        for i in range(samples):
+            assert(tri_ids[i] >= 0)
+            control_data.append(stage.centroid_values[tri_ids[i]])
+        
+        if inlet0 is not None:
+            control_data.append(inlet0.inlet.get_average_stage())
+            control_data.append(inlet0.inlet.get_average_xmom())
+            control_data.append(inlet0.inlet.get_average_ymom())
+            control_data.append(inlet0.inlet.get_total_water_volume())
+            control_data.append(inlet0.inlet.get_average_depth())
+
+        if verbose: print 'P%d control_data = %s' %(myid, control_data)
+    else:
+        stage = domain.get_quantity('stage')
+        
+        for i in range(samples):
+            if tri_ids[i] >= 0:
+                local_success = num.allclose(control_data[i], stage.centroid_values[tri_ids[i]])
+                success = success and local_success
+                if verbose: 
+                    print 'P%d tri %d, control = %s, actual = %s, Success = %s' %(myid, i, control_data[i], stage.centroid_values[tri_ids[i]], local_success) 
+                
+                
+        if inlet0 is not None:
+            inlet_master_proc = inlet0.inlet.get_master_proc()
+            average_stage = inlet0.inlet.get_global_average_stage()
+            average_xmom = inlet0.inlet.get_global_average_xmom()
+            average_ymom = inlet0.inlet.get_global_average_ymom()
+            average_volume = inlet0.inlet.get_global_total_water_volume()
+            average_depth = inlet0.inlet.get_global_average_depth()
+
+            if myid == inlet_master_proc:
+                if verbose: 
+                    print 'P%d average stage, control = %s, actual = %s' %(myid, control_data[samples], average_stage)
+
+                    print 'P%d average xmom, control = %s, actual = %s' %(myid, control_data[samples+1], average_xmom)
+
+                    print 'P%d average ymom, control = %s, actual = %s' %(myid, control_data[samples+2], average_ymom)
+
+                    print 'P%d average volume, control = %s, actual = %s' %(myid, control_data[samples+3], average_volume)
+
+                    print 'P%d average depth, control = %s, actual = %s' %(myid, control_data[samples+4], average_depth)
+
+
+        assert(success)
+
+    return control_data
+
+
+# Test an nprocs-way run of the shallow water equations
+# against the sequential code.
+
+class Test_parallel_frac_op(unittest.TestCase):
+    def test_parallel_frac_op(self):
+        #print "Expect this test to fail if not run from the parallel directory."
+        result = os.system("mpirun -np %d python test_parallel_inlet_operator.py" % nprocs)
+        assert_(result == 0)
+
+
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #pypar.finalize()
+        raise AssertionError, msg
+
+if __name__=="__main__":
+    if numprocs == 1:
+        runner = unittest.TextTestRunner()
+        suite = unittest.makeSuite(Test_parallel_frac_op, 'test')
+        #print "Running for numproc = 1"
+        runner.run(suite)
+    else:
+        #print "Running for numproc > 1"
+        pypar.barrier()
+        test_points = []
+
+        if myid == 0:
+            if verbose: print 'PARALLEL START'
+            for i in range(samples):
+                x = random.randrange(0,1000)/1000.0 * length
+                y = random.randrange(0,1000)/1000.0 * width
+                point = [x, y]
+                test_points.append(point)
+
+            for i in range(1,numprocs):
+                pypar.send(test_points, i)
+        else:
+            test_points = pypar.receive(0)
+
+        if myid == 0:
+            control_data = run_test(parallel=False, test_points = test_points, verbose = verbose)
+
+            for proc in range(1,numprocs):
+                pypar.send(control_data, proc)
+        else:
+            control_data = pypar.receive(0)
+
+
+        pypar.barrier()
+        run_test(parallel=True, control_data = control_data, test_points = test_points, verbose = verbose)
+
+
+    finalize()
+    
+
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_riverwall.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_riverwall.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_riverwall.py	(revision 9424)
@@ -0,0 +1,248 @@
+"""
+Test parallel and sequential results of riverwall procedure
+"""
+
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+import unittest
+import os
+import sys
+import anuga
+import numpy as num
+
+from anuga import myid, finalize, distribute, barrier, numprocs
+
+import anuga.utilities.plot_utils as util
+
+from math import exp
+
+
+alg = 'DE0'
+verbose = False
+
+scale_me=1.0
+nprocs = 3
+
+## Set up mesh
+boundaryPolygon=[ [0., 0.], [0., 100.], [100.0, 100.0], [100.0, 0.0]]
+midResPolygon=[ [30., 30.], [30., 70.], [70., 70.], [70., 30.]]
+higherResPolygon=[ [40., 40.], [40., 60.], [60., 60.], [60., 40.]]
+# Riverwall = list of lists, each with a set of x,y,z (and optional QFactor) values
+riverWall={ 'centralWall':
+                [ [50., 0.0, -0.0],
+                  [50., 45., -0.0],
+                  [50., 46., -0.2],
+                  [50., 54., -0.2],
+                  [50., 55., -0.0], 
+                  [50., 100.0, -0.0]] 
+          }
+
+riverWall_Par={'centralWall':{'Qfactor':1.0}}
+# Try to avoid any shallow-water type solution -- becomes unstable
+#riverWall_Par={'centralWall':{'Qfactor':1.0, 's1': 0.999, 's2':0.9999, 'h1':100, 'h2':150}}
+
+# The boundary polygon + riverwall breaks the mesh into multiple regions
+# Must define the resolution in these areas with an xy point + maximum area
+# Otherwise triangle.c gets confused
+regionPtAreas=[ [99., 99., 10.0*10.0*0.5],
+                [1., 1., 10.0*10.0*0.5],
+                [45., 45., 1.0*1.0*0.5],
+                [55., 55., 1.0*1.0*0.5],
+                [65., 65., 3.0*3.0*0.5],
+                [35., 35., 3.0*3.0*0.5] ]
+
+
+
+
+  
+
+###########################################################################
+# Setup Test
+##########################################################################
+def evolution_test(parallel=False, verbose=False):
+
+    #--------------------------------------------------------------------------
+    # Setup computational domain and quantities
+    #--------------------------------------------------------------------------
+    if myid == 0:
+        anuga.create_mesh_from_regions(boundaryPolygon, 
+                                 boundary_tags={'left': [0],
+                                                'top': [1],
+                                                'right': [2],
+                                                'bottom': [3]},
+                                   maximum_triangle_area = 1.0e+20,
+                                   minimum_triangle_angle = 28.0,
+                                   filename = 'runup.msh',
+                                   interior_regions = [ [higherResPolygon, 1.*1.*0.5],
+                                                        [midResPolygon, 3.0*3.0*0.5]],
+                                   breaklines=riverWall.values(),
+                                   use_cache=False,
+                                   verbose=verbose,
+                                   regionPtArea=regionPtAreas)
+        
+        sdomain=anuga.create_domain_from_file('runup.msh')
+        
+        
+        sdomain.set_flow_algorithm(alg)
+        
+        
+        sdomain.set_name('s_riverwall')                         
+        sdomain.set_datadir('.')                         
+        sdomain.set_store_vertices_uniquely()
+        
+        #------------------
+        # Define topography
+        #------------------
+        
+        def topography(x,y):
+            return -x/150.*scale_me 
+        
+        def stagefun(x,y):
+            stg=-0.5*scale_me
+            return stg 
+        
+        sdomain.set_quantity('elevation',topography)     # Use function for elevation
+        sdomain.set_quantity('friction',0.03)             # Constant friction
+        sdomain.set_quantity('stage', stagefun)              # Constant negative initial stage
+    else:
+        sdomain = None
+        
+    #--------------------------------------------------------------------------
+    # Create the parallel domains
+    #--------------------------------------------------------------------------
+    if parallel:
+        
+        if myid == 0 and verbose : print 'DISTRIBUTING TO PARALLEL DOMAIN'
+        pdomain = distribute(sdomain, verbose=verbose)
+        pdomain.set_name('p_riverwall')
+        pdomain.set_store_vertices_uniquely()
+        
+        
+    if myid == 0 and verbose: 
+        print 60*'='
+        print 'EVOLVING pdomain'
+        print 60*'='
+            
+    setup_and_evolve(pdomain, verbose=verbose)
+ 
+    barrier()
+   
+    if myid == 0:
+        if verbose: 
+            print 60*'='
+            print 'EVOLVING sdomain'
+            print 60*'='  
+        setup_and_evolve(sdomain, verbose=verbose)
+      
+    barrier()
+    
+    #---------------------------------
+    # Now compare the merged sww files
+    #---------------------------------
+    if myid == 0:
+        if verbose: print 'COMPARING SWW FILES'
+        
+        sdomain_v = util.get_output('s_riverwall.sww')
+        sdomain_c = util.get_centroids(sdomain_v)
+
+        pdomain_v = util.get_output('p_riverwall.sww')
+        pdomain_c = util.get_centroids(pdomain_v)
+        
+
+        # Test some values against the original ordering
+        
+        if verbose:
+            
+            order = 0
+            print 'PDOMAIN CENTROID VALUES'
+            print num.linalg.norm(sdomain_c.x-pdomain_c.x,ord=order)
+            print num.linalg.norm(sdomain_c.y-pdomain_c.y,ord=order)
+            print num.linalg.norm(sdomain_c.stage[-1]-pdomain_c.stage[-1],ord=order)
+            print num.linalg.norm(sdomain_c.xmom[-1]-pdomain_c.xmom[-1],ord=order)
+            print num.linalg.norm(sdomain_c.ymom[-1]-pdomain_c.ymom[-1],ord=order)
+            print num.linalg.norm(sdomain_c.xvel[-1]-pdomain_c.xvel[-1],ord=order)
+            print num.linalg.norm(sdomain_c.yvel[-1]-pdomain_c.yvel[-1],ord=order)        
+            
+        assert num.allclose(sdomain_c.stage,pdomain_c.stage)
+        assert num.allclose(sdomain_c.xmom,pdomain_c.xmom)
+        assert num.allclose(sdomain_c.ymom,pdomain_c.ymom)
+        assert num.allclose(sdomain_c.xvel,pdomain_c.xvel)
+        assert num.allclose(sdomain_c.yvel,pdomain_c.yvel)
+        
+        assert num.allclose(sdomain_v.x,pdomain_v.x)
+        assert num.allclose(sdomain_v.y,pdomain_v.y)
+                  
+ 
+        
+        
+def setup_and_evolve(domain, verbose=False):
+    
+
+    domain.riverwallData.create_riverwalls(riverWall, riverWall_Par, verbose=verbose)
+    
+    #--------------------------
+    # Setup boundary conditions
+    #--------------------------
+    Br=anuga.Reflective_boundary(domain)            # Solid reflective wall
+    
+    def boundaryFun(t):
+        output=-0.4*exp(-t/100.)-0.1
+        output=min(output,-0.11)
+        #output=min(output,-0.3)
+        return output  
+    
+
+    Bin_tmss = anuga.Transmissive_momentum_set_stage_boundary(domain=domain, function = boundaryFun) 
+    
+    #----------------------------------------------
+    # Associate boundary tags with boundary objects
+    #----------------------------------------------
+    domain.set_boundary({'left': Br, 'right': Bin_tmss, 'top': Br, 'bottom':Br})
+    
+
+    
+    #------------------------------
+    #Evolve the system through time
+    #------------------------------
+    if verbose: print 'Evolve'
+    for t in domain.evolve(yieldstep=10.0,finaltime=150.0):
+        if myid == 0 and verbose: print domain.timestepping_statistics()
+    
+    domain.sww_merge(delete_old=True)
+
+    
+
+# Test an nprocs-way run of the shallow water equations
+# against the sequential code.
+
+class Test_parallel_riverwall(unittest.TestCase):
+    def test_parallel_riverwall(self):
+        if verbose : print "Expect this test to fail if not run from the parallel directory."
+        result = os.system("mpirun -np %d python test_parallel_riverwall.py" % nprocs)
+        assert_(result == 0)
+
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #pypar.finalize()
+        raise AssertionError, msg
+
+if __name__=="__main__":
+    if numprocs == 1: 
+        runner = unittest.TextTestRunner()
+        suite = unittest.makeSuite(Test_parallel_riverwall, 'test')
+        runner.run(suite)
+    else:
+
+        #------------------------------------------
+        # Run the code code and compare sequential
+        # and parallel values
+        #------------------------------------------
+        if myid ==0 and verbose: print 'PARALLEL START'
+        evolution_test(parallel=True, verbose=verbose)
+        finalize()
+
+
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_shallow_domain.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_shallow_domain.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_shallow_domain.py	(revision 9424)
@@ -0,0 +1,156 @@
+
+"""Test a run of the sequential shallow water domain against
+a run of the parallel shallow water domain.
+
+WARNING: This assumes that the command to run jobs is mpirun.
+Tested with MPICH and LAM (Ole)
+"""
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+
+import unittest
+import os
+import sys
+
+
+
+import numpy as num
+
+
+
+#------------------------------------------
+# Import pypar without the initial output
+#------------------------------------------
+class NullStream:
+    def write(self,text):
+        pass
+sys.stdout = NullStream()
+import pypar
+sys.stdout = sys.__stdout__
+
+
+#------------------------------------------
+# anuga imports
+#------------------------------------------
+
+from anuga.utilities.numerical_tools import ensure_numeric
+from anuga.utilities.util_ext        import double_precision
+from anuga.utilities.norms           import l1_norm, l2_norm, linf_norm
+
+from anuga import Domain
+from anuga import Reflective_boundary
+from anuga import Dirichlet_boundary
+from anuga import Time_boundary
+from anuga import Transmissive_boundary
+
+from anuga import rectangular_cross
+from anuga import create_domain_from_file
+
+
+from anuga_parallel import distribute, myid, numprocs, finalize
+
+
+#--------------------------------------------------------------------------
+# Setup parameters
+#--------------------------------------------------------------------------
+
+mesh_filename = "merimbula_10785_1.tsh"
+#mesh_filename = "test-100.tsh"
+yieldstep = 1
+finaltime = 1
+quantity = 'stage'
+nprocs = 2
+verbose = False
+
+#--------------------------------------------------------------------------
+# Setup procedures
+#--------------------------------------------------------------------------
+class Set_Stage:
+    """Set an initial condition with constant water height, for x<x0
+    """
+
+    def __init__(self, x0=0.25, x1=0.5, h=1.0):
+        self.x0 = x0
+        self.x1 = x1
+        self.h  = h
+
+    def __call__(self, x, y):
+        return self.h*((x>self.x0)&(x<self.x1))
+
+#--------------------------------------------------------------------------
+# Setup test
+#--------------------------------------------------------------------------
+def evolution_test(parallel=False):
+
+
+    domain = create_domain_from_file(mesh_filename)
+    domain.set_quantity('stage', Set_Stage(756000.0, 756500.0, 2.0))
+
+    #--------------------------------------------------------------------------
+    # Create parallel domain if requested
+    #--------------------------------------------------------------------------
+
+    if parallel:
+        if myid == 0 and verbose: print 'DISTRIBUTING PARALLEL DOMAIN'
+        domain = distribute(domain)
+
+    #------------------------------------------------------------------------------
+    # Setup boundary conditions
+    # This must currently happen *after* domain has been distributed
+    #------------------------------------------------------------------------------
+    domain.store = False
+    Br = Reflective_boundary(domain)      # Solid reflective wall
+
+    domain.set_boundary({'outflow' :Br, 'inflow' :Br, 'inner' :Br, 'exterior' :Br, 'open' :Br})
+
+    #------------------------------------------------------------------------------
+    # Evolution
+    #------------------------------------------------------------------------------
+    if parallel:
+        if myid == 0 and verbose: print 'PARALLEL EVOLVE'
+    else:
+        if verbose: print 'SEQUENTIAL EVOLVE'
+
+    for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+        pass
+
+    domain.dump_triangulation()
+
+# Test an nprocs-way run of the shallow water equations
+# against the sequential code.
+
+class Test_parallel_shallow_domain(unittest.TestCase):
+    def test_parallel_shallow_domain(self):
+        #print "Expect this test to fail if not run from the parallel directory."
+        result = os.system("mpirun -np %d python test_parallel_shallow_domain.py" % nprocs)
+        assert_(result == 0)
+
+
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #pypar.finalize()
+        raise AssertionError, msg
+
+if __name__=="__main__":
+    if numprocs == 1: 
+        runner = unittest.TextTestRunner()
+        suite = unittest.makeSuite(Test_parallel_shallow_domain, 'test')
+        runner.run(suite)
+    else:
+
+        pypar.barrier()
+        if myid ==0:
+            if verbose: print 'PARALLEL START'
+
+        evolution_test(parallel=True)
+        
+        if myid == 0:     
+            if verbose: print 'Parallel test OK'
+
+
+
+    finalize()
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_sw_flow.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_sw_flow.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_parallel_sw_flow.py	(revision 9424)
@@ -0,0 +1,218 @@
+"""
+Simple water flow example using ANUGA
+
+Water driven up a linear slope and time varying boundary,
+similar to a beach environment
+
+This is a very simple test of the parallel algorithm using the simplified parallel API
+"""
+
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+import unittest
+import os
+import sys
+#import pypar
+import numpy as num
+
+
+
+from anuga import Domain
+from anuga import Reflective_boundary
+from anuga import Dirichlet_boundary
+from anuga import Time_boundary
+from anuga import Transmissive_boundary
+
+from anuga import rectangular_cross_domain
+
+
+from anuga_parallel import distribute, myid, numprocs, send, receive, barrier, finalize
+
+#--------------------------------------------------------------------------
+# Setup parameters
+#--------------------------------------------------------------------------
+yieldstep = 0.25
+finaltime = 6.0
+nprocs = 4
+N = 29
+M = 29 
+verbose = False
+
+#---------------------------------
+# Setup Functions
+#---------------------------------
+def topography(x,y): 
+    return -x/2    
+
+###########################################################################
+# Setup Test
+##########################################################################
+def evolution_test(parallel=False, G = None, seq_interpolation_points=None, verbose=False):
+
+    #--------------------------------------------------------------------------
+    # Setup computational domain and quantities
+    #--------------------------------------------------------------------------
+    domain = rectangular_cross_domain(M, N)
+    domain.set_quantity('elevation', topography) # Use function for elevation
+    domain.set_quantity('friction', 0.0)         # Constant friction 
+    domain.set_quantity('stage', expression='elevation') # Dry initial stage
+
+    #--------------------------------------------------------------------------
+    # Create the parallel domain
+    #--------------------------------------------------------------------------
+    if parallel:
+        if myid == 0 and verbose : print 'DISTRIBUTING PARALLEL DOMAIN'
+        domain = distribute(domain, verbose=False)
+
+    #--------------------------------------------------------------------------
+    # Setup domain parameters
+    #--------------------------------------------------------------------------
+    domain.set_name('runup')                    # Set sww filename
+    domain.set_datadir('.')                     # Set output dir
+
+    domain.set_flow_algorithm('2_0')
+    domain.set_quantities_to_be_stored(None)
+
+
+    #------------------------------------------------------------------------------
+    # Setup boundary conditions
+    # This must currently happen *AFTER* domain has been distributed
+    #------------------------------------------------------------------------------
+
+    Br = Reflective_boundary(domain)      # Solid reflective wall
+    Bd = Dirichlet_boundary([-0.2,0.,0.]) # Constant boundary values
+
+    # Associate boundary tags with boundary objects
+    domain.set_boundary({'left': Br, 'right': Bd, 'top': Br, 'bottom': Br})
+
+    #------------------------------------------------------------------------------
+    # Find which sub_domain in which the interpolation points are located 
+    #
+    # Sometimes the interpolation points sit exactly
+    # between two centroids, so in the parallel run we
+    # reset the interpolation points to the centroids
+    # found in the sequential run
+    #------------------------------------------------------------------------------
+    interpolation_points = [[0.4,0.5], [0.6,0.5], [0.8,0.5], [0.9,0.5]]
+
+
+    gauge_values = []
+    tri_ids = []
+    for i, point in enumerate(interpolation_points):
+        gauge_values.append([]) # Empty list for timeseries
+
+        #if is_inside_polygon(point, domain.get_boundary_polygon()):
+        #print "Point ", myid, i, point
+        try:
+            k = domain.get_triangle_containing_point(point)
+            if domain.tri_full_flag[k] == 1:
+                tri_ids.append(k)
+            else:
+                tri_ids.append(-1)            
+        except:
+            tri_ids.append(-2)
+
+        #print "  tri_ids ",myid, i, tri_ids[-1]
+		
+    if verbose: print 'P%d has points = %s' %(myid, tri_ids)
+
+
+    c_coord = domain.get_centroid_coordinates()
+    interpolation_points = []
+    for id in tri_ids:
+        if id<1:
+            if verbose: print 'WARNING: Interpolation point not within the domain!'
+        interpolation_points.append(c_coord[id,:])
+            
+    #------------------------------------------------------------------------------
+    # Evolve system through time
+    #------------------------------------------------------------------------------
+    time = []
+
+    if parallel:
+        if myid == 0 and verbose: print 'PARALLEL EVOLVE'
+    else:
+        if myid == 0 and verbose: print 'SEQUENTIAL EVOLVE'
+    
+
+    for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+        if myid == 0 and verbose : domain.write_time()
+
+        # Record time series at known points
+        time.append(domain.get_time())
+
+        stage = domain.get_quantity('stage')
+
+        for i in range(4):
+            if tri_ids[i] > -1:
+                gauge_values[i].append(stage.centroid_values[tri_ids[i]])
+
+
+    #----------------------------------------
+    # Setup test arrays during sequential run
+    #----------------------------------------
+    if not parallel:
+        G = []
+        for i in range(4):
+            G.append(gauge_values[i])
+
+    success = True
+
+    for i in range(4):
+        if tri_ids[i] > -1:
+            #print num.max(num.array(gauge_values[i])- num.array(G[i]))
+            success = success and num.allclose(gauge_values[i], G[i])
+
+    assert_(success)
+    
+    return G, interpolation_points
+
+# Test an nprocs-way run of the shallow water equations
+# against the sequential code.
+
+class Test_parallel_sw_flow(unittest.TestCase):
+    def test_parallel_sw_flow(self):
+        if verbose : print "Expect this test to fail if not run from the parallel directory."
+        result = os.system("mpirun -np %d python test_parallel_sw_flow.py" % nprocs)
+        assert_(result == 0)
+
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #pypar.finalize()
+        raise AssertionError, msg
+
+if __name__=="__main__":
+    if numprocs == 1: 
+        runner = unittest.TextTestRunner()
+        suite = unittest.makeSuite(Test_parallel_sw_flow, 'test')
+        runner.run(suite)
+    else:
+
+        #------------------------------------------
+        # Run the sequential code on each processor
+        # and save results at 4 gauge stations to
+        # array G
+        #------------------------------------------
+        barrier()
+        if myid == 0 and verbose: print 'SEQUENTIAL START'
+
+        G , interpolation_points = evolution_test(parallel=False,verbose=verbose)
+        G = num.array(G,num.float)
+
+        barrier()
+        
+        #------------------------------------------
+        # Run the code code and compare sequential
+        # results at 4 gauge stations
+        #------------------------------------------
+        if myid ==0 and verbose: print 'PARALLEL START'
+
+        evolution_test(parallel=True, G=G, seq_interpolation_points = interpolation_points, verbose= verbose)
+        
+        finalize()
+
+
Index: /trunk/anuga_core/source/anuga_parallel/tests/test_sequential_dist_sw_flow.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/tests/test_sequential_dist_sw_flow.py	(revision 9424)
+++ /trunk/anuga_core/source/anuga_parallel/tests/test_sequential_dist_sw_flow.py	(revision 9424)
@@ -0,0 +1,312 @@
+"""
+Simple water flow example using ANUGA
+
+Water driven up a linear slope and time varying boundary,
+similar to a beach environment
+
+This is a very simple test of the parallel algorithm using the simplified parallel API
+"""
+
+
+#------------------------------------------------------------------------------
+# Import necessary modules
+#------------------------------------------------------------------------------
+import unittest
+import os
+import sys
+#import pypar
+import numpy as num
+
+
+
+from anuga import Domain
+from anuga import Reflective_boundary
+from anuga import Dirichlet_boundary
+from anuga import Time_boundary
+from anuga import Transmissive_boundary
+
+from anuga import rectangular_cross_domain
+
+
+from anuga_parallel import distribute, myid, numprocs, send, receive, barrier, finalize
+
+from anuga_parallel.sequential_distribute import sequential_distribute_dump
+from anuga_parallel.sequential_distribute import sequential_distribute_load
+
+import anuga.utilities.plot_utils as util
+
+
+#--------------------------------------------------------------------------
+# Setup parameters
+#--------------------------------------------------------------------------
+yieldstep = 0.25
+finaltime = 3.0
+nprocs = 4
+N = 29
+M = 29 
+verbose = False
+
+
+new_parameters = {}
+new_parameters['ghost_layer_width'] = 2
+
+#---------------------------------
+# Setup Functions
+#---------------------------------
+def topography(x,y): 
+    return -x/2    
+
+###########################################################################
+# Setup Test
+##########################################################################
+def evolution_test(parallel=False, verbose=False):
+
+    #--------------------------------------------------------------------------
+    # Setup computational domain and quantities
+    #--------------------------------------------------------------------------
+    if myid == 0:
+        domain = rectangular_cross_domain(M, N)
+        domain.set_name('odomain')                    # Set sww filename
+        domain.set_datadir('.')   
+        domain.set_quantity('elevation', topography) # Use function for elevation
+        domain.set_quantity('friction', 0.0)         # Constant friction 
+        domain.set_quantity('stage', expression='elevation') # Dry initial stage
+    else:
+        domain = None
+        
+    #--------------------------------------------------------------------------
+    # Create pickled partition
+    #--------------------------------------------------------------------------
+    if myid == 0:
+        if verbose: print 'DUMPING PARTITION DATA'
+        sequential_distribute_dump(domain, numprocs, verbose=verbose, parameters=new_parameters)    
+
+    #--------------------------------------------------------------------------
+    # Create the parallel domains
+    #--------------------------------------------------------------------------
+    if parallel:
+        
+        if myid == 0 and verbose : print 'DISTRIBUTING TO PARALLEL DOMAIN'
+        pdomain = distribute(domain, verbose=verbose, parameters=new_parameters)
+        pdomain.set_name('pdomain')
+        
+        if myid == 0 and verbose : print 'LOADING IN PARALLEL DOMAIN'
+        sdomain = sequential_distribute_load(filename='odomain', verbose = verbose)
+        sdomain.set_name('sdomain')
+        
+    if myid == 0 and verbose: print 'EVOLVING pdomain'    
+    setup_and_evolve(pdomain, verbose=verbose)
+ 
+    if myid == 0 and verbose: print 'EVOLVING sdomain'   
+    setup_and_evolve(sdomain, verbose=verbose)
+    
+    if myid == 0:
+        if verbose: print 'EVOLVING odomain'   
+        setup_and_evolve(domain, verbose=verbose)
+    
+
+    if myid == 0:
+        parameter_file=open('odomain.txt', 'w')
+        from pprint import pprint
+        pprint(domain.get_algorithm_parameters(),parameter_file,indent=4)
+        parameter_file.close()
+
+        parameter_file=open('sdomain.txt', 'w')
+        from pprint import pprint
+        pprint(sdomain.get_algorithm_parameters(),parameter_file,indent=4)
+        parameter_file.close()
+
+        parameter_file=open('pdomain.txt', 'w')
+        from pprint import pprint
+        pprint(pdomain.get_algorithm_parameters(),parameter_file,indent=4)
+        parameter_file.close()        
+    
+    assert num.allclose(pdomain.quantities['stage'].centroid_values, sdomain.quantities['stage'].centroid_values)
+    assert num.allclose(pdomain.quantities['stage'].vertex_values, sdomain.quantities['stage'].vertex_values)
+    
+    assert num.allclose(pdomain.vertex_coordinates, sdomain.vertex_coordinates)
+    assert num.allclose(pdomain.centroid_coordinates, sdomain.centroid_coordinates)
+    
+    
+
+    #---------------------------------
+    # Now compare the merged sww files
+    #---------------------------------
+    if myid == 0:
+        if verbose: print 'COMPARING SWW FILES'
+        
+        odomain_v = util.get_output('odomain.sww')
+        odomain_c = util.get_centroids(odomain_v)
+
+        pdomain_v = util.get_output('pdomain.sww')
+        pdomain_c = util.get_centroids(pdomain_v)
+        
+        sdomain_v = util.get_output('sdomain.sww')
+        sdomain_c = util.get_centroids(sdomain_v)
+
+        # Test some values against the original ordering
+        
+        if verbose:
+            
+            order = 0
+            print 'PDOMAIN CENTROID VALUES'
+            print num.linalg.norm(odomain_c.x-pdomain_c.x,ord=order)
+            print num.linalg.norm(odomain_c.y-pdomain_c.y,ord=order)
+            print num.linalg.norm(odomain_c.stage[-1]-pdomain_c.stage[-1],ord=order)
+            print num.linalg.norm(odomain_c.xmom[-1]-pdomain_c.xmom[-1],ord=order)
+            print num.linalg.norm(odomain_c.ymom[-1]-pdomain_c.ymom[-1],ord=order)
+            print num.linalg.norm(odomain_c.xvel[-1]-pdomain_c.xvel[-1],ord=order)
+            print num.linalg.norm(odomain_c.yvel[-1]-pdomain_c.yvel[-1],ord=order)        
+            
+             
+            print 'SDOMAIN CENTROID VALUES'        
+            print num.linalg.norm(odomain_c.x-sdomain_c.x,ord=order)
+            print num.linalg.norm(odomain_c.y-sdomain_c.y,ord=order)
+            print num.linalg.norm(odomain_c.stage[-1]-sdomain_c.stage[-1],ord=order)
+            print num.linalg.norm(odomain_c.xmom[-1]-sdomain_c.xmom[-1],ord=order)
+            print num.linalg.norm(odomain_c.ymom[-1]-sdomain_c.ymom[-1],ord=order)
+            print num.linalg.norm(odomain_c.xvel[-1]-sdomain_c.xvel[-1],ord=order)
+            print num.linalg.norm(odomain_c.yvel[-1]-sdomain_c.yvel[-1],ord=order)
+            
+            print 'PDOMAIN VERTEX VALUES'        
+            print num.linalg.norm(odomain_v.stage[-1]-pdomain_v.stage[-1],ord=order)
+            print num.linalg.norm(odomain_v.xmom[-1]-pdomain_v.xmom[-1],ord=order)
+            print num.linalg.norm(odomain_v.ymom[-1]-pdomain_v.ymom[-1],ord=order)
+            print num.linalg.norm(odomain_v.xvel[-1]-pdomain_v.xvel[-1],ord=order)
+            print num.linalg.norm(odomain_v.yvel[-1]-pdomain_v.yvel[-1],ord=order)
+            
+            print 'SDOMAIN VERTEX VALUES'     
+            print num.linalg.norm(odomain_v.stage[-1]-sdomain_v.stage[-1],ord=order)
+            print num.linalg.norm(odomain_v.xmom[-1]-sdomain_v.xmom[-1],ord=order)
+            print num.linalg.norm(odomain_v.ymom[-1]-sdomain_v.ymom[-1],ord=order)
+            print num.linalg.norm(odomain_v.xvel[-1]-sdomain_v.xvel[-1],ord=order)
+            print num.linalg.norm(odomain_v.yvel[-1]-sdomain_v.yvel[-1],ord=order)
+            
+            
+            
+
+        assert num.allclose(odomain_c.stage,pdomain_c.stage)
+        assert num.allclose(odomain_c.xmom,pdomain_c.xmom)
+        assert num.allclose(odomain_c.ymom,pdomain_c.ymom)
+        assert num.allclose(odomain_c.xvel,pdomain_c.xvel)
+        assert num.allclose(odomain_c.yvel,pdomain_c.yvel)
+        
+        assert num.allclose(odomain_v.x,pdomain_v.x)
+        assert num.allclose(odomain_v.y,pdomain_v.y)
+                
+        assert num.linalg.norm(odomain_v.x-pdomain_v.x,ord=0) == 0
+        assert num.linalg.norm(odomain_v.y-pdomain_v.y,ord=0) == 0
+        assert num.linalg.norm(odomain_v.stage[-1]-pdomain_v.stage[-1],ord=0) < 100
+        assert num.linalg.norm(odomain_v.xmom[-1]-pdomain_v.xmom[-1],ord=0) < 100 
+        assert num.linalg.norm(odomain_v.ymom[-1]-pdomain_v.ymom[-1],ord=0) < 100
+        assert num.linalg.norm(odomain_v.xvel[-1]-pdomain_v.xvel[-1],ord=0) < 100
+        assert num.linalg.norm(odomain_v.yvel[-1]-pdomain_v.yvel[-1],ord=0) < 100     
+        
+        assert num.allclose(odomain_c.x,sdomain_c.x)
+        assert num.allclose(odomain_c.y,sdomain_c.y)
+        assert num.allclose(odomain_c.stage,sdomain_c.stage)
+        assert num.allclose(odomain_c.xmom,sdomain_c.xmom)
+        assert num.allclose(odomain_c.ymom,sdomain_c.ymom)
+        assert num.allclose(odomain_c.xvel,sdomain_c.xvel)
+        assert num.allclose(odomain_c.yvel,sdomain_c.yvel)
+        
+        assert num.allclose(odomain_v.x,sdomain_v.x)
+        assert num.allclose(odomain_v.y,sdomain_v.y)
+        
+        order = 0
+        assert num.linalg.norm(odomain_v.x-sdomain_v.x,ord=order) == 0
+        assert num.linalg.norm(odomain_v.y-sdomain_v.y,ord=order) == 0
+        assert num.linalg.norm(odomain_v.stage[-1]-sdomain_v.stage[-1],ord=order) < 100
+        assert num.linalg.norm(odomain_v.xmom[-1]-sdomain_v.xmom[-1],ord=order) < 100
+        assert num.linalg.norm(odomain_v.ymom[-1]-sdomain_v.ymom[-1],ord=order) < 100
+        assert num.linalg.norm(odomain_v.xvel[-1]-sdomain_v.xvel[-1],ord=order) < 100
+        assert num.linalg.norm(odomain_v.yvel[-1]-sdomain_v.yvel[-1],ord=order) < 100        
+                
+        # COMPARE CENTROID PDOMAIN SDOMAIN  
+        assert num.allclose(pdomain_c.x,sdomain_c.x)
+        assert num.allclose(pdomain_c.y,sdomain_c.y)
+        assert num.allclose(pdomain_c.stage[-1],sdomain_c.stage[-1])
+        assert num.allclose(pdomain_c.xmom[-1],sdomain_c.xmom[-1])
+        assert num.allclose(pdomain_c.ymom[-1],sdomain_c.ymom[-1])
+        assert num.allclose(pdomain_c.xvel[-1],sdomain_c.xvel[-1])
+        assert num.allclose(pdomain_c.yvel[-1],sdomain_c.yvel[-1])
+            
+            
+        # COMPARE VERTEX PDOMAIN SDOMAIN
+        assert num.allclose(pdomain_v.x,sdomain_v.x)
+        assert num.allclose(pdomain_v.y,sdomain_v.y)
+        assert num.allclose(pdomain_v.stage[-1],sdomain_v.stage[-1])
+        assert num.allclose(pdomain_v.xmom[-1],sdomain_v.xmom[-1])
+        assert num.allclose(pdomain_v.ymom[-1],sdomain_v.ymom[-1])
+        assert num.allclose(pdomain_v.xvel[-1],sdomain_v.xvel[-1])
+        assert num.allclose(pdomain_v.yvel[-1],sdomain_v.yvel[-1])   
+            
+
+        
+        
+def setup_and_evolve(domain, verbose=False):
+
+    #--------------------------------------------------------------------------
+    # Setup domain parameters
+    #--------------------------------------------------------------------------
+    domain.set_flow_algorithm('DE0')
+    #domain.set_store_vertices_uniquely()
+
+    #------------------------------------------------------------------------------
+    # Setup boundary conditions
+    # This must currently happen *AFTER* domain has been distributed
+    #------------------------------------------------------------------------------
+
+    Br = Reflective_boundary(domain)      # Solid reflective wall
+    Bd = Dirichlet_boundary([-0.2,0.,0.]) # Constant boundary values
+
+    # Associate boundary tags with boundary objects
+    domain.set_boundary({'left': Br, 'right': Bd, 'top': Br, 'bottom': Br})
+
+    #------------------------------------------------------------------------------
+    # Evolve
+    #------------------------------------------------------------------------------
+    for t in domain.evolve(yieldstep = yieldstep, finaltime = finaltime):
+        if myid == 0 and verbose : domain.write_time()
+        #if myid == 0 and verbose : print domain.quantities['stage'].get_maximum_value()
+
+
+    domain.sww_merge()
+    
+
+# Test an nprocs-way run of the shallow water equations
+# against the sequential code.
+
+class Test_parallel_sw_flow(unittest.TestCase):
+    def test_parallel_sw_flow(self):
+        if verbose : print "Expect this test to fail if not run from the parallel directory."
+        result = os.system("mpirun -np %d python test_sequential_dist_sw_flow.py" % nprocs)
+        assert_(result == 0)
+
+# Because we are doing assertions outside of the TestCase class
+# the PyUnit defined assert_ function can't be used.
+def assert_(condition, msg="Assertion Failed"):
+    if condition == False:
+        #pypar.finalize()
+        raise AssertionError, msg
+
+if __name__=="__main__":
+    if numprocs == 1: 
+        runner = unittest.TextTestRunner()
+        suite = unittest.makeSuite(Test_parallel_sw_flow, 'test')
+        runner.run(suite)
+    else:
+
+
+        
+        #------------------------------------------
+        # Run the code code and compare sequential
+        # results at 4 gauge stations
+        #------------------------------------------
+        if myid ==0 and verbose: print 'PARALLEL START'
+
+        evolution_test(parallel=True, verbose=verbose)
+        
+        finalize()
+
+
