Index: /trunk/anuga_core/source/anuga_parallel/distribute_mesh.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/distribute_mesh.py	(revision 8604)
+++ /trunk/anuga_core/source/anuga_parallel/distribute_mesh.py	(revision 8605)
@@ -1529,5 +1529,6 @@
         x = num.array(submesh["ghost_quan"][k][p], num.float)
         pypar.send(x,p, bypass=True)
-        
+
+
 
 #########################################################
@@ -1721,5 +1722,15 @@
            number_of_full_nodes, number_of_full_triangles, tri_map, node_map,\
            ghost_layer_width
-          
+
+
+
+
+
+
+
+
+
+
+
 
 
@@ -1727,5 +1738,5 @@
 #
 # Extract the submesh that will belong to the
-# "host processor" (i.e. processor zero)
+# processor 0 (i.e. processor zero)
 #
 #  *) See the documentation for build_submesh
@@ -1739,27 +1750,34 @@
 #
 #########################################################
-def extract_hostmesh(submesh, triangles_per_proc):
+def extract_submesh(submesh, triangles_per_proc, p=0):
 
     
     submesh_cell = {}
-    submesh_cell["ghost_layer_width"] = submesh["ghost_layer_width"][0]
-    submesh_cell["full_nodes"] = submesh["full_nodes"][0]
-    submesh_cell["ghost_nodes"] = submesh["ghost_nodes"][0]
-    submesh_cell["full_triangles"] = submesh["full_triangles"][0]
-    submesh_cell["ghost_triangles"] = submesh["ghost_triangles"][0]
-    submesh_cell["full_boundary"] = submesh["full_boundary"][0]
-    submesh_cell["ghost_boundary"] = submesh["ghost_boundary"][0]
-    submesh_cell["ghost_commun"] = submesh["ghost_commun"][0]
-    submesh_cell["full_commun"] = submesh["full_commun"][0]
+    submesh_cell["ghost_layer_width"] = submesh["ghost_layer_width"][p]
+    submesh_cell["full_nodes"] = submesh["full_nodes"][p]
+    submesh_cell["ghost_nodes"] = submesh["ghost_nodes"][p]
+    submesh_cell["full_triangles"] = submesh["full_triangles"][p]
+    submesh_cell["ghost_triangles"] = submesh["ghost_triangles"][p]
+    submesh_cell["full_boundary"] = submesh["full_boundary"][p]
+    submesh_cell["ghost_boundary"] = submesh["ghost_boundary"][p]
+    submesh_cell["ghost_commun"] = submesh["ghost_commun"][p]
+    submesh_cell["full_commun"] = submesh["full_commun"][p]
     submesh_cell["full_quan"] ={}
     submesh_cell["ghost_quan"]={}
     for k in submesh["full_quan"]:
-        submesh_cell["full_quan"][k] = submesh["full_quan"][k][0]
-        submesh_cell["ghost_quan"][k] = submesh["ghost_quan"][k][0]
+        submesh_cell["full_quan"][k] = submesh["full_quan"][k][p]
+        submesh_cell["ghost_quan"][k] = submesh["ghost_quan"][k][p]
+
+
+    # FIXME SR: I think there is already a structure with this info in the mesh
+    lower_t = 0
+    for i in range(p):
+        lower_t = lower_t+triangles_per_proc[i]
+    upper_t = lower_t+triangles_per_proc[p]
 
     numprocs = len(triangles_per_proc)
     points, vertices, boundary, quantities, ghost_recv_dict, \
             full_send_dict, tri_map, node_map, ghost_layer_width = \
-            build_local_mesh(submesh_cell, 0, triangles_per_proc[0], numprocs)
+            build_local_mesh(submesh_cell, lower_t, upper_t, numprocs)
 
 
Index: /trunk/anuga_core/source/anuga_parallel/parallel_api.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/parallel_api.py	(revision 8604)
+++ /trunk/anuga_core/source/anuga_parallel/parallel_api.py	(revision 8605)
@@ -16,5 +16,5 @@
     from anuga_parallel.distribute_mesh  import send_submesh
     from anuga_parallel.distribute_mesh  import rec_submesh
-    from anuga_parallel.distribute_mesh  import extract_hostmesh
+    from anuga_parallel.distribute_mesh  import extract_submesh
 
     # Mesh partitioning using Metis
@@ -298,5 +298,5 @@
     points, vertices, boundary, quantities, \
             ghost_recv_dict, full_send_dict, tri_map, node_map, ghost_layer_width =\
-              extract_hostmesh(submesh, triangles_per_proc)
+              extract_submesh(submesh, triangles_per_proc,0)
 
     # Keep track of the number full nodes and triangles.
Index: /trunk/anuga_core/source/anuga_parallel/parallel_test.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/parallel_test.py	(revision 8604)
+++ /trunk/anuga_core/source/anuga_parallel/parallel_test.py	(revision 8605)
@@ -11,5 +11,5 @@
 from anuga_parallel.distribute_mesh import build_submesh
 from anuga_parallel.distribute_mesh import submesh_full, submesh_ghost, submesh_quantities
-from anuga_parallel.distribute_mesh import extract_hostmesh, rec_submesh, send_submesh
+from anuga_parallel.distribute_mesh import extract_submesh, rec_submesh, send_submesh
 
 from anuga_parallel.interface import myid, numprocs, barrier
@@ -215,8 +215,8 @@
 
             #----------------------------------------------------------------------------------
-            # Test extract_hostmesh
+            # Test extract_submesh
             #----------------------------------------------------------------------------------
             points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict  =\
-            extract_hostmesh(submesh, triangles_per_proc)
+            extract_submesh(submesh, triangles_per_proc)
 
 
Index: /trunk/anuga_core/source/anuga_parallel/run_parallel_merimbula_test.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_merimbula_test.py	(revision 8604)
+++ /trunk/anuga_core/source/anuga_parallel/run_parallel_merimbula_test.py	(revision 8605)
@@ -51,5 +51,5 @@
 from build_submesh import build_submesh
 from build_local   import build_local_mesh
-from build_commun  import send_submesh, rec_submesh, extract_hostmesh
+from build_commun  import send_submesh, rec_submesh, extract_submesh
 
 
@@ -111,5 +111,5 @@
     
     points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict = \
-              extract_hostmesh(submesh, triangles_per_proc)
+              extract_submesh(submesh, triangles_per_proc)
 
 # read in the mesh partition that belongs to this
Index: /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_merimbula_test.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_merimbula_test.py	(revision 8604)
+++ /trunk/anuga_core/source/anuga_parallel/run_parallel_sw_merimbula_test.py	(revision 8605)
@@ -58,5 +58,5 @@
 from distribute_mesh  import build_submesh
 from distribute_mesh  import build_local_mesh
-from distribute_mesh  import send_submesh, rec_submesh, extract_hostmesh
+from distribute_mesh  import send_submesh, rec_submesh, extract_submesh
 
 
@@ -139,5 +139,5 @@
 
     points, vertices, boundary, quantities, ghost_recv_dict, full_send_dict = \
-             extract_hostmesh(submesh, triangles_per_proc)
+             extract_submesh(submesh, triangles_per_proc)
 
 # Read in the mesh partition that belongs to this
Index: /trunk/anuga_core/source/anuga_parallel/sequential_distribute.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/sequential_distribute.py	(revision 8605)
+++ /trunk/anuga_core/source/anuga_parallel/sequential_distribute.py	(revision 8605)
@@ -0,0 +1,206 @@
+"""Trying to lump parallel stuff into simpler interface
+
+
+"""
+
+import numpy as num
+
+from anuga_parallel.distribute_mesh  import send_submesh
+from anuga_parallel.distribute_mesh  import rec_submesh
+from anuga_parallel.distribute_mesh  import extract_submesh
+
+# Mesh partitioning using Metis
+from anuga_parallel.distribute_mesh import build_submesh
+from anuga_parallel.distribute_mesh import pmesh_divide_metis_with_map
+
+from anuga_parallel.parallel_shallow_water import Parallel_domain
+
+
+
+def sequential_distribute(domain, numprocs=1, verbose=False, debug=False, parameters = None):
+    """ Distribute the domain, create parallel domain and pickle result
+    """
+
+
+    if debug:
+        verbose = True
+
+
+
+    # FIXME: Dummy assignment (until boundaries are refactored to
+    # be independent of domains until they are applied)
+    bdmap = {}
+    for tag in domain.get_boundary_tags():
+        bdmap[tag] = None
+
+    domain.set_boundary(bdmap)
+
+
+    if numprocs == 1 : return # Bypass
+
+
+    domain_name = domain.get_name()
+    domain_dir = domain.get_datadir()
+    domain_store = domain.get_store()
+    domain_minimum_storable_height = domain.minimum_storable_height
+    domain_flow_algorithm = domain.get_flow_algorithm()
+    domain_minimum_allowed_height = domain.get_minimum_allowed_height()
+    georef = domain.geo_reference
+    number_of_global_triangles = domain.number_of_triangles
+    number_of_global_nodes = domain.number_of_nodes
+    boundary_map = domain.boundary_map
+
+
+    #sequential_distribute_mesh(domain, numprocs, verbose=verbose, debug=debug, parameters=parameters)
+
+
+    # Subdivide the mesh
+    if verbose: print 'sequential_distribute: Subdivide mesh'
+    new_nodes, new_triangles, new_boundary, triangles_per_proc, quantities, \
+           s2p_map, p2s_map = \
+           pmesh_divide_metis_with_map(domain, numprocs)
+
+    #PETE: s2p_map (maps serial domain triangles to parallel domain triangles)
+    #      sp2_map (maps parallel domain triangles to domain triangles)
+
+
+
+    # Build the mesh that should be assigned to each processor,
+    # this includes ghost nodes and the communication pattern
+    if verbose: print 'sequential_distribute: Build submeshes'
+    submesh = build_submesh(new_nodes, new_triangles, new_boundary, quantities, triangles_per_proc, parameters)
+
+    if debug:
+        for p in range(numprocs):
+            N = len(submesh['ghost_nodes'][p])
+            M = len(submesh['ghost_triangles'][p])
+            print 'There are %d ghost nodes and %d ghost triangles on proc %d'\
+                  %(N, M, p)
+
+    #if debug:
+    #    from pprint import pprint
+    #    pprint(submesh)
+
+
+    # extract data to create parallel domain
+    if verbose: print 'sequential_distribute: Distribute submeshes'
+    for p in range(0, numprocs):
+
+        # Build the local mesh for processor 0
+        points, vertices, boundary, quantities, \
+            ghost_recv_dict, full_send_dict, tri_map, node_map, ghost_layer_width =\
+              extract_submesh(submesh, triangles_per_proc, p)
+
+
+#        from pprint import pprint
+#        print '='*80
+#        print p
+#        print '='*80
+#        pprint(tri_map)
+#        print len(tri_map)
+
+        # Keep track of the number full nodes and triangles.
+        # This is useful later if one needs access to a ghost-free domain
+        # Here, we do it for process 0. The others are done in rec_submesh.
+        number_of_full_nodes = len(submesh['full_nodes'][p])
+        number_of_full_triangles = len(submesh['full_triangles'][p])
+
+        # Extract l2g maps
+        tri_l2g  = extract_l2g_map(tri_map)
+        node_l2g = extract_l2g_map(node_map)
+
+
+        s2p_map = None
+        p2s_map = None
+
+        #------------------------------------------------------------------------
+        # Build the parallel domain for this processor using partion structures
+        #------------------------------------------------------------------------
+
+        if verbose:
+            print 'sequential_distribute: P%g, no_full_nodes = %g, no_full_triangles = %g' % (p, number_of_full_nodes, number_of_full_triangles)
+
+
+        parallel_domain = Parallel_domain(points, vertices, boundary,
+                                 full_send_dict=full_send_dict,
+                                 ghost_recv_dict=ghost_recv_dict,
+                                 number_of_full_nodes=number_of_full_nodes,
+                                 number_of_full_triangles=number_of_full_triangles,
+                                 geo_reference=georef,
+                                 number_of_global_triangles = number_of_global_triangles,
+                                 number_of_global_nodes = number_of_global_nodes,
+                                 processor = p,
+                                 numproc = numprocs,
+                                 s2p_map = s2p_map,
+                                 p2s_map = p2s_map, ## jj added this
+                                 tri_l2g = tri_l2g, ## SR added this
+                                 node_l2g = node_l2g,
+                                 ghost_layer_width = ghost_layer_width)
+
+        #------------------------------------------------------------------------
+        # Transfer initial conditions to each subdomain
+        #------------------------------------------------------------------------
+        for q in quantities:
+            parallel_domain.set_quantity(q, quantities[q])
+
+
+        #------------------------------------------------------------------------
+        # Transfer boundary conditions to each subdomain
+        #------------------------------------------------------------------------
+        boundary_map['ghost'] = None  # Add binding to ghost boundary
+        parallel_domain.set_boundary(boundary_map)
+
+
+        #------------------------------------------------------------------------
+        # Transfer other attributes to each subdomain
+        #------------------------------------------------------------------------
+        parallel_domain.set_name(domain_name)
+        parallel_domain.set_datadir(domain_dir)
+        parallel_domain.set_store(domain_store)
+        parallel_domain.set_minimum_storable_height(domain_minimum_storable_height)
+        parallel_domain.set_minimum_allowed_height(domain_minimum_allowed_height)
+        parallel_domain.set_flow_algorithm(domain_flow_algorithm)
+        parallel_domain.geo_reference = georef
+
+
+
+        #-----------------------------------------------------------------------
+        # Now let's store the parallel_domain via cPickle
+        #-----------------------------------------------------------------------
+        import cPickle
+        pickle_name = domain_name + '_P%g_%g.pickle'% (numprocs,p)
+        f = file(pickle_name, 'wb')
+        cPickle.dump(parallel_domain, f, protocol=cPickle.HIGHEST_PROTOCOL)
+        f.close()
+
+
+    return
+
+
+
+def extract_l2g_map(map):
+    # Extract l2g data  from corresponding map
+    # Maps
+
+    import numpy as num
+
+    b = num.arange(len(map))
+
+    l_ids = num.extract(map>-1,map)
+    g_ids = num.extract(map>-1,b)
+
+
+#    print len(g_ids)
+#    print len(l_ids)
+#    print l_ids
+#    print g_ids
+
+    l2g = num.zeros_like(g_ids)
+    l2g[l_ids] = g_ids
+
+    return l2g
+
+
+
+
+
Index: /trunk/anuga_core/source/anuga_parallel/test_distribute_mesh.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_distribute_mesh.py	(revision 8604)
+++ /trunk/anuga_core/source/anuga_parallel/test_distribute_mesh.py	(revision 8605)
@@ -12,5 +12,5 @@
 from anuga_parallel.distribute_mesh import build_submesh
 from anuga_parallel.distribute_mesh import submesh_full, submesh_ghost, submesh_quantities
-from anuga_parallel.distribute_mesh import extract_hostmesh, rec_submesh, send_submesh
+from anuga_parallel.distribute_mesh import extract_submesh, rec_submesh, send_submesh
 
 import numpy as num
@@ -1132,6 +1132,382 @@
 
 
-
-
+    def test_build_extract_submesh_3(self):
+        """
+        Test 3 way extract_submesh
+        """
+
+        nodes = [[0.0, 0.0], [0.0, 0.5], [0.0, 1.0], [0.5, 0.0], [0.5, 0.5], [0.5, 1.0], [1.0, 0.0], \
+        [1.0, 0.5], [1.0, 1.0], [0.25, 0.25], [0.25, 0.75], [0.75, 0.25], [0.75, 0.75]]
+
+
+        triangles = [[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8], [0, 9, 1], \
+        [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5], [3, 9, 0], [3, 11, 4], \
+        [6, 11, 3], [7, 11, 6], [4, 11, 7]]
+
+
+        boundary = {(13, 1): 'bottom', (7, 1): 'left', (3, 1): 'right', (14, 1): 'right', \
+        (11, 1): 'bottom', (10, 1): 'top', (5, 1): 'left', (4, 1): 'top'}
+
+        triangles_per_proc = [5, 6, 5]
+
+
+        quantities = {'stage': num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]),  'elevation': num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]),  'ymomentum': num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]]),  'friction': num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]),  'xmomentum': num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])}
+
+
+
+        true_submesh = {'full_boundary': [{(3, 1): 'right', (4, 1): 'top'},\
+        {(5, 1): 'left', (10, 1): 'top', (7, 1): 'left'}, \
+        {(13, 1): 'bottom', (14, 1): 'right', (11, 1): 'bottom'}],
+                        'ghost_nodes': [num.array([[  0.  ,   0.  ,   0.  ],
+       [  1.  ,   0.  ,   0.5 ],
+       [  2.  ,   0.  ,   1.  ],
+       [  6.  ,   1.  ,   0.  ],
+       [ 10.  ,   0.25,   0.75],
+       [ 11.  ,   0.75,   0.25]]), num.array([[  3.  ,   0.5 ,   0.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  8.  ,   1.  ,   1.  ],
+       [ 11.  ,   0.75,   0.25],
+       [ 12.  ,   0.75,   0.75]]), num.array([[  1.  ,   0.  ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  8.  ,   1.  ,   1.  ],
+       [ 12.  ,   0.75,   0.75]])],
+                        'full_nodes': [num.array([[  3.  ,   0.5 ,   0.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  8.  ,   1.  ,   1.  ],
+       [  9.  ,   0.25,   0.25],
+       [ 12.  ,   0.75,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
+       [  1.  ,   0.  ,   0.5 ],
+       [  2.  ,   0.  ,   1.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  5.  ,   0.5 ,   1.  ],
+       [  9.  ,   0.25,   0.25],
+       [ 10.  ,   0.25,   0.75]]), num.array([[  0.  ,   0.  ,   0.  ],
+       [  3.  ,   0.5 ,   0.  ],
+       [  4.  ,   0.5 ,   0.5 ],
+       [  6.  ,   1.  ,   0.  ],
+       [  7.  ,   1.  ,   0.5 ],
+       [  9.  ,   0.25,   0.25],
+       [ 11.  ,   0.75,   0.25]])],
+                        'ghost_triangles': [num.array([[ 5,  0,  9,  1],
+       [ 6,  1,  9,  4],
+       [ 8,  4, 10,  1],
+       [ 9,  5, 10,  4],
+       [10,  2, 10,  5],
+       [11,  3,  9,  0],
+       [12,  3, 11,  4],
+       [13,  6, 11,  3],
+       [14,  7, 11,  6],
+       [15,  4, 11,  7]]), num.array([[ 0,  4,  9,  3],
+       [ 1,  4, 12,  5],
+       [ 2,  7, 12,  4],
+       [ 4,  5, 12,  8],
+       [11,  3,  9,  0],
+       [12,  3, 11,  4]]), num.array([[ 0,  4,  9,  3],
+       [ 1,  4, 12,  5],
+       [ 2,  7, 12,  4],
+       [ 3,  8, 12,  7],
+       [ 5,  0,  9,  1],
+       [ 6,  1,  9,  4]])],
+                        'ghost_boundary': [{(13, 1): 'ghost', (8, 0): 'ghost', (14, 1): 'ghost', \
+                        (11, 1): 'ghost', (10, 1): 'ghost', (5, 1): 'ghost', (10, 2): 'ghost'}, \
+                        {(12, 2): 'ghost', (12, 0): 'ghost', (2, 1): 'ghost', (11, 1): 'ghost',\
+                        (2, 2): 'ghost', (4, 1): 'ghost', (4, 0): 'ghost'}, {(3, 2): 'ghost', \
+                        (6, 1): 'ghost', (3, 1): 'ghost', (5, 1): 'ghost', (1, 0): 'ghost', (1, 1): 'ghost'}],
+                        'full_triangles': [[[4, 9, 3], [4, 12, 5], [7, 12, 4], [8, 12, 7], [5, 12, 8]], \
+                        [[0, 9, 1], [1, 9, 4], [1, 10, 2], [4, 10, 1], [5, 10, 4], [2, 10, 5]], \
+                        [[3, 9, 0], [3, 11, 4], [6, 11, 3], [7, 11, 6], [4, 11, 7]]],
+                        'full_commun': [{0: [1, 2], 1: [1, 2], 2: [1, 2], 3: [2], 4: [1]}, \
+                        {5: [0, 2], 6: [0, 2], 7: [], 8: [0], 9: [0], 10: [0]}, \
+                        {11: [0, 1], 12: [0, 1], 13: [0], 14: [0], 15: [0]}],
+                        'ghost_commun': [num.array([[ 5,  1],
+       [ 6,  1],
+       [ 8,  1],
+       [ 9,  1],
+       [10,  1],
+       [11,  2],
+       [12,  2],
+       [13,  2],
+       [14,  2],
+       [15,  2]]), num.array([[ 0,  0],
+       [ 1,  0],
+       [ 2,  0],
+       [ 4,  0],
+       [11,  2],
+       [12,  2]]), num.array([[0, 0],
+       [1, 0],
+       [2, 0],
+       [3, 0],
+       [5, 1],
+       [6, 1]])],  'ghost_quan': {'stage': [num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ]])],  'elevation': [num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.25 , -0.375, -0.5  ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ]]), num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ]])],  'ymomentum': [num.array([[ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ]]), num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]])], 'xmomentum': [num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])]},  'full_quan': {'stage': [num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]])],  'elevation': [num.array([[-0.25 , -0.125, -0.25 ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]]), num.array([[-0.   , -0.125, -0.   ],
+       [-0.   , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.125, -0.25 ],
+       [-0.   , -0.125, -0.25 ]]), num.array([[-0.25 , -0.125, -0.   ],
+       [-0.25 , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.25 ],
+       [-0.5  , -0.375, -0.5  ],
+       [-0.25 , -0.375, -0.5  ]])],  'ymomentum': [num.array([[ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.5 ],
+       [ 0.5 ,  0.25,  0.5 ],
+       [ 0.5 ,  0.75,  1.  ],
+       [ 0.5 ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  0.5 ],
+       [ 1.  ,  0.75,  1.  ]]), num.array([[ 0.  ,  0.25,  0.  ],
+       [ 0.  ,  0.25,  0.5 ],
+       [ 0.  ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.  ],
+       [ 0.5 ,  0.25,  0.5 ]])],  'friction': [num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]]), num.array([[ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.],
+       [ 0.,  0.,  0.]])],  'xmomentum': [num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]]), num.array([[ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.],
+       [ 2.,  2.,  2.]])]}}
+
+
+        # Subdivide into non-overlapping partitions
+        submesh = build_submesh(nodes, triangles, boundary, quantities,
+                  triangles_per_proc, parameters = None)
+
+
+        for i in range(3):
+            assert num.allclose(true_submesh['full_triangles'][i],submesh['full_triangles'][i])
+            assert num.allclose(true_submesh['full_nodes'][i],submesh['full_nodes'][i])
+            assert num.allclose(true_submesh['ghost_triangles'][i],submesh['ghost_triangles'][i])
+            assert num.allclose(true_submesh['ghost_nodes'][i],submesh['ghost_nodes'][i])
+            assert num.allclose(true_submesh['ghost_commun'][i],submesh['ghost_commun'][i])
+
+        assert true_submesh['full_boundary'] == submesh['full_boundary']
+        assert true_submesh['full_commun'] == submesh['full_commun']
+
+        for key, value in true_submesh['ghost_quan'].iteritems():
+            for i in range(3):
+                assert num.allclose(true_submesh['ghost_quan'][key][i],submesh['ghost_quan'][key][i])
+                assert num.allclose(true_submesh['full_quan'][key][i],submesh['full_quan'][key][i])
+
+
+        # Now test the extract_submesh for the 3 processors
+
+        submesh_cell_0 = extract_submesh(submesh,triangles_per_proc,0)
+        submesh_cell_1 = extract_submesh(submesh,triangles_per_proc,1)
+        submesh_cell_2 = extract_submesh(submesh,triangles_per_proc,2)
+
+
+        from pprint import pprint
+
+        pprint(submesh_cell_1)
 
 #-------------------------------------------------------------
Index: /trunk/anuga_core/source/anuga_parallel/test_parallel_distribute_mesh.py
===================================================================
--- /trunk/anuga_core/source/anuga_parallel/test_parallel_distribute_mesh.py	(revision 8604)
+++ /trunk/anuga_core/source/anuga_parallel/test_parallel_distribute_mesh.py	(revision 8605)
@@ -13,5 +13,5 @@
 from anuga_parallel.distribute_mesh import build_submesh
 from anuga_parallel.distribute_mesh import submesh_full, submesh_ghost, submesh_quantities
-from anuga_parallel.distribute_mesh import extract_hostmesh, rec_submesh, send_submesh
+from anuga_parallel.distribute_mesh import extract_submesh, rec_submesh, send_submesh
 
 from anuga_parallel import myid, numprocs, barrier, finalize
@@ -193,9 +193,9 @@
 
         #----------------------------------------------------------------------------------
-        # Test extract_hostmesh
+        # Test extract_submesh
         #----------------------------------------------------------------------------------
         points, vertices, boundary, quantities, \
                 ghost_recv_dict, full_send_dict, tri_map, node_map, ghost_layer_width =\
-        extract_hostmesh(submesh, triangles_per_proc)
+        extract_submesh(submesh, triangles_per_proc)
 
 
