Changeset 6086

Timestamp:

Dec 19, 2008, 10:38:08 AM (15 years ago)

Author:

rwilson

Message:

Changes to handle large files when Scientific.IO.NetCDF provides the feature.

Location:

anuga_core/source/anuga

Files:

• abstract_2d_finite_volumes/test_util.py (modified) (1 diff)
• abstract_2d_finite_volumes/util.py (modified) (2 diffs)
• config.py (modified) (9 diffs)
• coordinate_transforms/test_geo_reference.py (modified) (4 diffs)
• fit_interpolate/interpolate.py (modified) (2 diffs)
• geospatial_data/geospatial_data.py (modified) (4 diffs)
• geospatial_data/test_geospatial_data.py (modified) (4 diffs)
• load_mesh/loadASCII.py (modified) (3 diffs)
• shallow_water/benchmark_sww2dem.py (modified) (2 diffs)
• shallow_water/data_manager.py (modified) (31 diffs)
• shallow_water/most2nc.py (modified) (2 diffs)
• shallow_water/shallow_water_domain.py (modified) (2 diffs)
• shallow_water/test_data_manager.py (modified) (52 diffs)
• shallow_water/test_most2nc.py (modified) (1 diff)
• shallow_water/test_shallow_water_domain.py (modified) (2 diffs)
• shallow_water/test_system.py (modified) (3 diffs)
• test_all.py (modified) (10 diffs)
• utilities/test_system_tools.py (modified) (3 diffs)

anuga_core/source/anuga/abstract_2d_finite_volumes/test_util.py

@@ -1834 +1834 @@
 if __name__ == "__main__":
     suite = unittest.makeSuite(Test_Util,'test')
-#    suite = unittest.makeSuite(Test_Util,'test_remove_lone_verts')
+#    suite = unittest.makeSuite(Test_Util,'test_sww2csv_gauges')
 #    runner = unittest.TextTestRunner(verbosity=2)
     runner = unittest.TextTestRunner(verbosity=1)

anuga_core/source/anuga/abstract_2d_finite_volumes/util.py

@@ -25 +25 @@
 from anuga.utilities.system_tools import get_revision_number
 from anuga.utilities.system_tools import store_version_info
+
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 
 
@@ -272 +274 @@
     if verbose: print 'Reading', filename
 
-    fid = NetCDFFile(filename, 'r')
+    fid = NetCDFFile(filename, netcdf_mode_r)
 
     if type(quantity_names) == types.StringType:

anuga_core/source/anuga/config.py

@@ -2 +2 @@
 """
 
-#--------------------
+################################################################################
 # Numerical constants
-#--------------------
-epsilon = 1.0e-12 # Smallest number - used for safe division
-max_float = 1.0e36 # Largest number - used to initialise (max, min) ranges
-default_smoothing_parameter = 0.001 # Default alpha for penalised
-                                    # least squares fitting
-                                    
-single_precision = 1.0e-6           # Smallest single precision number
+################################################################################
+
+epsilon = 1.0e-12                    # Smallest number - used for safe division
+max_float = 1.0e36                   # Largest number - used to initialise 
+                                     # (max, min) ranges
+default_smoothing_parameter = 0.001  # Default alpha for penalised
+                                     # least squares fitting
+single_precision = 1.0e-6            # Smallest single precision number
 velocity_protection = 1.0e-6                                     
 
-#-------------------------------------------
-# Standard filenames, directories and system 
-# parameters used by ANUGA
-#-------------------------------------------
+################################################################################
+# Standard filenames, directories and system parameters used by ANUGA
+################################################################################
+
 pmesh_filename = '.\\pmesh'
 version_filename = 'stored_version_info.py'
@@ -28 +29 @@
 major_revision = '1.0beta'
 
-
-#-------------------
+################################################################################
 # Physical constants
-#-------------------
+################################################################################
+
 manning = 0.03  # Manning's friction coefficient
 #g = 9.80665    # Gravity - FIXME reinstate this and fix unit tests.
@@ -38 +39 @@
 #The 'official' average is 9.80665
 
-eta_w = 3.0e-3 #Wind stress coefficient
-rho_a = 1.2e-3 #Atmospheric density
-rho_w = 1023   #Fluid density [kg/m^3] (rho_w = 1023 for salt water)
-
-
-#-----------------------------------------------------
+eta_w = 3.0e-3 # Wind stress coefficient
+rho_a = 1.2e-3 # Atmospheric density
+rho_w = 1023   # Fluid density [kg/m^3] (rho_w = 1023 for salt water)
+
+################################################################################
 # Limiters - used with linear reconstruction of vertex 
 # values from centroid values
-#-----------------------------------------------------
+################################################################################
 
 # Betas [0;1] control the allowed steepness of gradient for second order
@@ -63 +63 @@
 beta_vh_dry = 0.2
 
-
 # Alpha_balance controls how limiters are balanced between deep and shallow.
-# A large value will favour the deep water limiters, allowing the a closer hug to the coastline.
-# This will minimise 'creep' but at the same time cause smaller time steps
+# A large value will favour the deep water limiters, allowing the a closer hug
+# to the coastline.  This will minimise 'creep' but at the same time cause
+# smaller time steps
 # Range:
-
 alpha_balance = 2.0 
 
@@ -85 +84 @@
 # This option is good with tight_slope_limiters, especially for large domains.
 use_centroid_velocities = True
-
         
 # FIXME (Ole) Maybe get rid of order altogether and use beta_w
@@ -91 +89 @@
 default_order = 1
 
-
-#-------------
+################################################################################
 # Timestepping
-#-------------
+################################################################################
 
 CFL = 1.0  # CFL condition assigned to domain.CFL - controls timestep size
       
 # Choose type of timestepping,
+#timestepping_method = 'rk2'   # 2nd Order TVD scheme
 timestepping_method = 'euler' # 1st order euler
-#timestepping_method = 'rk2'   # 2nd Order TVD scheme
 
 # rk2 is a little more stable than euler, so rk2 timestepping
@@ -109 +106 @@
 beta_rk2   = 1.6
 
-
-
 # Option to search for signatures where isolated triangles are
 # responsible for a small global timestep.
@@ -116 +111 @@
 # overall computation.
 # This facility is experimental.
-
 # protect_against_isolated_degenerate_timesteps = False
 protect_against_isolated_degenerate_timesteps = False
 
-
 min_timestep = 1.0e-6 # Minimal timestep accepted in ANUGA
 max_timestep = 1.0e+3
-max_smallsteps = 50  # Max number of degenerate steps allowed b4 trying first order
-
-#Perhaps minimal timestep could be based on the geometry as follows:
-#Define maximal possible speed in open water v_max, e.g. 500m/s (soundspeed?)
-#Then work out minimal internal distance in mesh r_min and set
-#min_timestep = r_min/v_max
-#
-#Max speeds are calculated in the flux function as
-#
-#lambda = v +/- sqrt(gh)
+max_smallsteps = 50   # Max number of degenerate steps allowed b4
+                      # trying first order
+
+# Perhaps minimal timestep could be based on the geometry as follows:
+# Define maximal possible speed in open water v_max, e.g. 500m/s (soundspeed?)
+# Then work out minimal internal distance in mesh r_min and set
+# min_timestep = r_min/v_max
+#
+# Max speeds are calculated in the flux function as
+#
+# lambda = v +/- sqrt(gh)
 #
 # so with 500 m/s, h ~ 500^2/g = 2500 m well out of the domain of the
 # shallow water wave equation
 #
-#The actual soundspeed can be as high as 1530m/s
-#(see http://staff.washington.edu/aganse/public.projects/clustering/clustering.html),
-#but that would only happen with h>225000m in this equation. Why ?
-#The maximal speed we specify is really related to the max speed
-#of surface pertubation
-#
-#v_max = 100 #For use in domain_ext.c
-#sound_speed = 500
-
-
-#---------------------------------------------------
+# The actual soundspeed can be as high as 1530m/s
+# (see http://staff.washington.edu/aganse/public.projects/clustering/clustering.html),
+# but that would only happen with h>225000m in this equation. Why ?
+# The maximal speed we specify is really related to the max speed
+# of surface pertubation
+#
+# v_max = 100 #For use in domain_ext.c
+# sound_speed = 500
+
+################################################################################
 # Ranges specific to the shallow water wave equation
-# These control maximal and minimal values of 
-# quantities
-#---------------------------------------------------
+# These control maximal and minimal values of quantities
+################################################################################
 
 # Water depth below which it is considered to be 0 in the model
@@ -163 +155 @@
 maximum_allowed_speed = 0.0 # Maximal particle speed of water
 #maximum_allowed_speed = 1.0 # Maximal particle speed of water
-                            # Too large (100) creates 'flopping' water
-                            # Too small (0) creates 'creep'
+                             # Too large (100) creates 'flopping' water
+                             # Too small (0) creates 'creep'
                             
 maximum_froude_number = 100.0 # To be used in limiters.
 
-
-#------------------------------------------------------------
+################################################################################
 # Performance parameters used to invoke various optimisations
-#------------------------------------------------------------
+################################################################################
 
 use_extensions = True # Use C-extensions
-use_psyco = True # Use psyco optimisations
+use_psyco = True      # Use psyco optimisations
 
 optimise_dry_cells = True # Exclude dry and still cells from flux computation
 optimised_gradient_limiter = True # Use hardwired gradient limiter
-use_edge_limiter = False # The edge limiter is better, but most runs have been using vertex limiting. Validations passed with this one True 9th May 2008, but many unit tests need backward compatibility flag set FIXME(Ole).
+use_edge_limiter = False  # The edge limiter is better, but most runs have been
+                          # using vertex limiting. Validations passed with this
+                          # one True 9th May 2008, but many unit tests need
+                          # backward compatibility flag set FIXME(Ole).
 
 points_file_block_line_size = 500 # Number of lines read in from a points file
                                   # when blocking
 
-
-
-
-
-
-    
+################################################################################
+# Dynamically-defined constants.
+################################################################################
+
+# Determine if we can read/write large NetCDF files
+
+netcdf_mode_w = 'w'
+netcdf_mode_a = 'a'
+netcdf_mode_r = 'r'
+
+try:
+    import tempfile
+    from Scientific.IO.NetCDF import NetCDFFile
+
+    fname = tempfile.mktemp()
+    fid = NetCDFFile(fname, netcdf_mode_w)
+    fid.close()
+    netcdf_mode_w = 'wl'
+#    log('Using NetCDF large file mode')
+except IOError:
+    pass
+

anuga_core/source/anuga/coordinate_transforms/test_geo_reference.py

@@ -8 +8 @@
 from geo_reference import *
 from Numeric import allclose,array
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 
 
@@ -32 +33 @@
         file_name = tempfile.mktemp(".geo_referenceTest")
         
-        out_file = NetCDFFile(file_name, 'w')
+        out_file = NetCDFFile(file_name, netcdf_mode_w)
         g.write_NetCDF(out_file)
         out_file.close()
         
-        in_file = NetCDFFile(file_name, 'r')
+        in_file = NetCDFFile(file_name, netcdf_mode_r)
         new_g = Geo_reference(NetCDFObject=in_file)
         in_file.close()
@@ -49 +50 @@
         file_name = tempfile.mktemp(".geo_referenceTest")
         
-        outfile = NetCDFFile(file_name, 'w')
+        outfile = NetCDFFile(file_name, netcdf_mode_w)
         outfile.xllcorner = g.get_xllcorner() 
         outfile.yllcorner =  g.get_yllcorner() 
@@ -55 +56 @@
         outfile.close()
         
-        in_file = NetCDFFile(file_name, 'r')
+        in_file = NetCDFFile(file_name, netcdf_mode_r)
         new_g = Geo_reference(NetCDFObject=in_file)
         in_file.close()

anuga_core/source/anuga/fit_interpolate/interpolate.py

@@ -40 +40 @@
 from anuga.fit_interpolate.general_fit_interpolate import FitInterpolate
 from anuga.abstract_2d_finite_volumes.util import file_function
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 
 # Interpolation specific exceptions
@@ -1311 +1312 @@
     stdout = sys.stdout
     sys.stdout = fd
-    fid = NetCDFFile(file_name, 'r') 
+    fid = NetCDFFile(file_name, netcdf_mode_r) 
     sys.stdout = stdout
     fd.close()

anuga_core/source/anuga/geospatial_data/geospatial_data.py

@@ -21 +21 @@
 from anuga.utilities.anuga_exceptions import ANUGAError
 from anuga.config import points_file_block_line_size as MAX_READ_LINES
-
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 
 DEFAULT_ATTRIBUTE = 'elevation'
@@ -793 +793 @@
 
             # Throws prints to screen if file not present
-            self.fid = NetCDFFile(self.file_name, 'r')
+            self.fid = NetCDFFile(self.file_name, netcdf_mode_r)
 
             (self.blocking_georef,
@@ -1002 +1002 @@
 
     # Throws prints to screen if file not present
-    fid = NetCDFFile(file_name, 'r')
+    fid = NetCDFFile(file_name, netcdf_mode_r)
 
     pointlist = array(fid.variables['points'])
@@ -1253 +1253 @@
 
     # NetCDF file definition
-    outfile = NetCDFFile(file_name, 'w')
+    outfile = NetCDFFile(file_name, netcdf_mode_w)
 
     # Create new file

anuga_core/source/anuga/geospatial_data/test_geospatial_data.py

@@ -14 +14 @@
 from anuga.utilities.anuga_exceptions import ANUGAError
 from anuga.utilities.system_tools import get_host_name
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 
 class Test_Geospatial_data(unittest.TestCase):
@@ -1131 +1132 @@
         fileName = tempfile.mktemp(".pts")
         # NetCDF file definition
-        outfile = NetCDFFile(fileName, 'w')
+        outfile = NetCDFFile(fileName, netcdf_mode_w)
         
         # dimension definitions
@@ -1288 +1289 @@
         FN = 'test_points.pts'
         # NetCDF file definition
-        outfile = NetCDFFile(FN, 'w')
+        outfile = NetCDFFile(FN, netcdf_mode_w)
         
         # dimension definitions
@@ -1329 +1330 @@
         FN = 'test_points.pts'
         # NetCDF file definition
-        outfile = NetCDFFile(FN, 'w')
+        outfile = NetCDFFile(FN, netcdf_mode_w)
 
         # Make up an arbitrary georef

anuga_core/source/anuga/load_mesh/loadASCII.py

@@ -66 +66 @@
 
 from Scientific.IO.NetCDF import NetCDFFile
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
+
 
 import exceptions
@@ -682 +684 @@
     # NetCDF file definition
     try:
-        outfile = NetCDFFile(file_name, 'w')
+        outfile = NetCDFFile(file_name, netcdf_mode_w)
     except IOError:
         msg = 'File %s could not be created' % file_name
@@ -844 +846 @@
 
     #throws prints to screen if file not present
-    fid = NetCDFFile(file_name, 'r')
+    fid = NetCDFFile(file_name, netcdf_mode_r)
     mesh = {}
 

anuga_core/source/anuga/shallow_water/benchmark_sww2dem.py

@@ -34 +34 @@
 from anuga.fit_interpolate.interpolate import benchmark_interpolate
 from anuga.shallow_water.data_manager import Write_sww, export_grid
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
+
 
 def mem_usage():
@@ -105 +107 @@
         
     # NetCDF file definition
-    fid = NetCDFFile(sww_fileName, 'w')
+    fid = NetCDFFile(sww_fileName, netcdf_mode_w)
     sww = Write_sww()
     sww.store_header(fid, 0,

anuga_core/source/anuga/shallow_water/data_manager.py

@@ -78 +78 @@
 from anuga.config import minimum_storable_height as \
      default_minimum_storable_height
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 from anuga.config import max_float
 from anuga.utilities.numerical_tools import ensure_numeric,  mean
@@ -300 +301 @@
     # @param extension 
     # @param mode The mode of the underlying file.
-    def __init__(self, domain, extension, mode='w'):
-        assert mode in ['r', 'w', 'a'], \
+    def __init__(self, domain, extension, mode=netcdf_mode_w):
+        assert mode[0] in ['r', 'w', 'a'], \
                "Mode %s must be either:\n" % mode + \
                "   'w' (write)\n" + \
@@ -341 +342 @@
     # @param max_size ??
     # @param recursion ??
-    # @note Prepare the undelying data file if mode is 'w'.
-    def __init__(self, domain, mode='w', max_size=2000000000, recursion=False):
+    # @note Prepare the underlying data file if mode starts with 'w'.
+    def __init__(self, domain, mode=netcdf_mode_w, max_size=2000000000, recursion=False):
         from Scientific.IO.NetCDF import NetCDFFile
         from Numeric import Int, Float, Float32
@@ -363 +364 @@
         # NetCDF file definition
         fid = NetCDFFile(self.filename, mode)
-        if mode == 'w':
+        if mode[0] == 'w':
             description = 'Output from anuga.abstract_2d_finite_volumes ' \
                           'suitable for plotting'
@@ -429 +430 @@
 
         # append to the NetCDF file
-        fid = NetCDFFile(self.filename, 'a')
-
-        # Get the variables
-        x = fid.variables['x']
-        y = fid.variables['y']
-        z = fid.variables['elevation']
-        volumes = fid.variables['volumes']
+        fid = NetCDFFile(self.filename, netcdf_mode_a)
+
+#        # Get the variables
+#        x = fid.variables['x']
+#        y = fid.variables['y']
+#        z = fid.variables['elevation']
+#        volumes = fid.variables['volumes']
 
         # Get X, Y and bed elevation Z
@@ -445 +446 @@
         self.writer.store_triangulation(fid,
                                         points,
-                                        V.astype(volumes.typecode()),
+#                                        V.astype(volumes.typecode()),
+                                        V.astype(Float32),
                                         Z,
                                         points_georeference=\
@@ -475 +477 @@
         while not file_open and retries < 10:
             try:
-                fid = NetCDFFile(self.filename, 'a') # Open existing file
+                fid = NetCDFFile(self.filename, netcdf_mode_a) # Open existing file
             except IOError:
                 # This could happen if someone was reading the file.
@@ -623 +625 @@
     # @param domain ??
     # @param mode Mode of underlying data file (default WRITE).
-    def __init__(self, domain, mode='w'):
+    def __init__(self, domain, mode=netcdf_mode_w):
         from Scientific.IO.NetCDF import NetCDFFile
         from Numeric import Int, Float, Float
@@ -633 +635 @@
         # NetCDF file definition
         fid = NetCDFFile(self.filename, mode)
-        if mode == 'w':
+        if mode[0] == 'w':
             #Create new file
             fid.institution = 'Geoscience Australia'
@@ -683 +685 @@
 
         #Get NetCDF
-        fid = NetCDFFile(self.filename, 'a')
+        fid = NetCDFFile(self.filename, netcdf_mode_a)
 
         # Get the variables
@@ -718 +720 @@
         while not file_open and retries < 10:
             try:
-                fid = NetCDFFile(self.filename, 'a')
+                fid = NetCDFFile(self.filename, netcdf_mode_a)
             except IOError:
                 #This could happen if someone was reading the file.
@@ -1134 +1136 @@
     FN = create_filename('.', basefilename, 'sww', size)
     print 'Reading from ', FN
-    fid = NetCDFFile(FN, 'r')  #Open existing file for read
+    fid = NetCDFFile(FN, netcdf_mode_r)  #Open existing file for read
 
     # Get the variables
@@ -1257 +1259 @@
 
     # Get NetCDF
-    infile = NetCDFFile(filename1, 'r')  #Open existing file for read
-    outfile = NetCDFFile(filename2, 'w')  #Open new file
+    infile = NetCDFFile(filename1, netcdf_mode_r)  #Open existing file for read
+    outfile = NetCDFFile(filename2, netcdf_mode_w)  #Open new file
 
     # Copy dimensions
@@ -1303 +1305 @@
 # @return A class instance of required domain and mode.
 #Get data objects
-def get_dataobject(domain, mode='w'):
+def get_dataobject(domain, mode=netcdf_mode_w):
     """Return instance of class of given format using filename
     """
@@ -1389 +1391 @@
 
     # Get NetCDF
-    infile = NetCDFFile(root + '.dem', 'r') 
+    infile = NetCDFFile(root + '.dem', netcdf_mode_r) 
 
     if verbose: print 'Reading DEM from %s' %(root + '.dem')
@@ -1419 +1421 @@
 
     # NetCDF file definition
-    outfile = NetCDFFile(ptsname, 'w')
+    outfile = NetCDFFile(ptsname, netcdf_mode_w)
 
     # Create new file
@@ -2700 +2702 @@
 
     # NetCDF file definition
-    fid = NetCDFFile(netcdfname, 'w')
+    fid = NetCDFFile(netcdfname, netcdf_mode_w)
 
     #Create new file
@@ -2829 +2831 @@
     if verbose: print 'Reading files %s_*.nc' % basename_in
 
-    file_h = NetCDFFile(basename_in + '_ha.nc', 'r') # Wave amplitude (cm)
-    file_u = NetCDFFile(basename_in + '_ua.nc', 'r') # Velocity (x) (cm/s)
-    file_v = NetCDFFile(basename_in + '_va.nc', 'r') # Velocity (y) (cm/s)
-    file_e = NetCDFFile(basename_in + '_e.nc', 'r')  # Elevation (z) (m)
+    file_h = NetCDFFile(basename_in + '_ha.nc', netcdf_mode_r) # Wave amplitude (cm)
+    file_u = NetCDFFile(basename_in + '_ua.nc', netcdf_mode_r) # Velocity (x) (cm/s)
+    file_v = NetCDFFile(basename_in + '_va.nc', netcdf_mode_r) # Velocity (y) (cm/s)
+    file_e = NetCDFFile(basename_in + '_e.nc', netcdf_mode_r)  # Elevation (z) (m)
 
     if basename_out is None:
@@ -3033 +3035 @@
 
     # NetCDF file definition
-    outfile = NetCDFFile(swwname, 'w')
+    outfile = NetCDFFile(swwname, netcdf_mode_w)
 
     description = 'Converted from Ferret files: %s, %s, %s, %s' \
@@ -3263 +3265 @@
     from Scientific.IO.NetCDF import NetCDFFile
 
-    fid = NetCDFFile(filename + '.tms', 'w')
+    fid = NetCDFFile(filename + '.tms', netcdf_mode_w)
 
     fid.institution = 'Geoscience Australia'
@@ -3312 +3314 @@
 
     #Get NetCDF
-    fid = NetCDFFile(file_name, 'r')
+    fid = NetCDFFile(file_name, netcdf_mode_r)
 
     # Get the variables
@@ -3355 +3357 @@
     if verbose: print 'Reading from ', filename
 
-    fid = NetCDFFile(filename, 'r')    # Open existing file for read
+    fid = NetCDFFile(filename, netcdf_mode_r)    # Open existing file for read
     time = fid.variables['time']       # Timesteps
     if t is None:
@@ -3647 +3649 @@
 
     #Open existing netcdf file to read
-    infile = NetCDFFile(inname, 'r')
+    infile = NetCDFFile(inname, netcdf_mode_r)
 
     if verbose: print 'Reading DEM from %s' % inname
@@ -3684 +3686 @@
 
     #Open netcdf file for output
-    outfile = NetCDFFile(outname, 'w')
+    outfile = NetCDFFile(outname, netcdf_mode_w)
 
     #Create new file
@@ -3925 +3927 @@
 
     # NetCDF file definition
-    outfile = NetCDFFile(sww_file, 'w')
+    outfile = NetCDFFile(sww_file, netcdf_mode_w)
 
     #Create new file
@@ -4276 +4278 @@
 
         # NetCDF file definition
-        self.outfile = NetCDFFile(file_name, 'w')
+        self.outfile = NetCDFFile(file_name, netcdf_mode_w)
         outfile = self.outfile
 
@@ -4557 +4559 @@
 
     # NetCDF file definition
-    outfile = NetCDFFile(file_out, 'w')
+    outfile = NetCDFFile(file_out, netcdf_mode_w)
 
     #Create new file
@@ -5136 +5138 @@
     if verbose: print 'Output to ', swwname
 
-    outfile = NetCDFFile(swwname, 'w')
+    outfile = NetCDFFile(swwname, netcdf_mode_w)
 
     # For a different way of doing this, check out tsh2sww
@@ -5528 +5530 @@
 
     # NetCDF file definition
-    outfile = NetCDFFile(stsname, 'w')
+    outfile = NetCDFFile(stsname, netcdf_mode_w)
 
     description = 'Converted from URS mux2 files: %s' % basename_in
@@ -5628 +5630 @@
 
     try:
-        fid = NetCDFFile(stsname + '.sts', 'r')
+        fid = NetCDFFile(stsname + '.sts', netcdf_mode_r)
     except:
         msg = 'Cannot open %s' % stsname + '.sts'
@@ -6718 +6720 @@
     if verbose: print 'Reading from ', filename
 
-    fid = NetCDFFile(filename, 'r')    # Open existing file for read
+    fid = NetCDFFile(filename, netcdf_mode_r)    # Open existing file for read
     time = fid.variables['time'][:]    # Time vector
     time += fid.starttime[0]

anuga_core/source/anuga/shallow_water/most2nc.py

@@ -9 +9 @@
 import sys
 from Scientific.IO.NetCDF import NetCDFFile
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 
 
@@ -75 +76 @@
     if verbose: print 'writing results'
 
-    out_file = NetCDFFile(output_file,'w')
+    out_file = NetCDFFile(output_file, netcdf_mode_w)
 
     out_file.createDimension(long_name,nx)

anuga_core/source/anuga/shallow_water/shallow_water_domain.py

@@ -108 +108 @@
 from anuga.config import use_edge_limiter
 from anuga.config import use_centroid_velocities
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 
 from anuga.fit_interpolate.interpolate import Modeltime_too_late, Modeltime_too_early
@@ -625 +626 @@
 
         # Initialise writer
-        self.writer = get_dataobject(self, mode = 'w')
+        self.writer = get_dataobject(self, mode=netcdf_mode_w)
 
         # Store vertices and connectivity

anuga_core/source/anuga/shallow_water/test_data_manager.py

@@ -22 +22 @@
 from anuga.coordinate_transforms.redfearn import degminsec2decimal_degrees
 from anuga.abstract_2d_finite_volumes.util import file_function
-
 from anuga.utilities.system_tools import get_pathname_from_package
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 
 # This is needed to run the tests of local functions
@@ -32 +32 @@
 from anuga.geospatial_data.geospatial_data import Geospatial_data
 
+
 class Test_Data_Manager(unittest.TestCase):
     # Class variable
@@ -42 +43 @@
         import time
         from mesh_factory import rectangular
-
         
         self.verbose = Test_Data_Manager.verbose
@@ -105 +105 @@
 
         for ext in ['_ha.nc', '_ua.nc', '_va.nc', '_e.nc']:
-            fid = NetCDFFile(self.test_MOST_file + ext, 'w')
+            fid = NetCDFFile(self.test_MOST_file + ext, netcdf_mode_w)
 
             fid.createDimension(long_name,nx)
@@ -180 +180 @@
         sww.store_connectivity()
 
-        fid = NetCDFFile(sww.filename, 'r')  #Open existing file for append
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)  #Open existing file for append
 
         # Get the variables
@@ -214 +214 @@
         #Check contents
         #Get NetCDF
-        fid = NetCDFFile(sww.filename, 'r')  #Open existing file for append
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)  #Open existing file for append
 
         # Get the variables
@@ -247 +247 @@
             
         # Get NetCDF
-        fid = NetCDFFile(sww.filename, 'r') # Open existing file for append
+        fid = NetCDFFile(sww.filename, netcdf_mode_r) # Open existing file for append
 
         # Get the variables
@@ -326 +326 @@
             
         # Get NetCDF
-        fid = NetCDFFile(sww.filename, 'r') # Open existing file for append
+        fid = NetCDFFile(sww.filename, netcdf_mode_r) # Open existing file for append
 
         # Get the variables
@@ -372 +372 @@
         #Check contents
         #Get NetCDF
-        fid = NetCDFFile(sww.filename, 'r')  #Open existing file for append
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)  #Open existing file for append
 
         # Get the variables
@@ -413 +413 @@
         #Check contents
         #Get NetCDF
-        fid = NetCDFFile(sww.filename, 'r')  #Open existing file for append
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)  #Open existing file for append
 
 
@@ -465 +465 @@
         #Check contents
         #Get NetCDF
-        fid = NetCDFFile(sww.filename, 'r')  #Open existing file for append
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)  #Open existing file for append
 
         # Get the variables
@@ -522 +522 @@
         #Check contents
         #Get NetCDF
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, inetcdf_mode_r)
 
         # Get the variables
@@ -586 +586 @@
 
             #Get NetCDF
-            fid = NetCDFFile(self.domain.writer.filename, 'r')
+            fid = NetCDFFile(self.domain.writer.filename, netcdf_mode_r)
             stage_file = fid.variables['stage']
             
@@ -627 +627 @@
         #Check contents
         #Get NetCDF
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
 
@@ -677 +677 @@
         #Check contents
         #Get NetCDF
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -787 +787 @@
         #Check contents
         #Get NetCDF
-        fid = NetCDFFile(root+'.pts', 'r')
+        fid = NetCDFFile(root+'.pts', netcdf_mode_r)
 
         # Get the variables
@@ -908 +908 @@
         #Check contents
         #Get NetCDF
-        fid = NetCDFFile(root+'.pts', 'r')
+        fid = NetCDFFile(root+'.pts', netcdf_mode_r)
 
         # Get the variables
@@ -1039 +1039 @@
         #Check contents
         #Get NetCDF
-        fid = NetCDFFile(root+'.pts', 'r')
+        fid = NetCDFFile(root+'.pts', netcdf_mode_r)
 
         # Get the variables
@@ -1180 +1180 @@
         #Check contents
         #Get NetCDF
-        fid = NetCDFFile(root+'.pts', 'r')
+        fid = NetCDFFile(root+'.pts', netcdf_mode_r)
 
         # Get the variables
@@ -1266 +1266 @@
         #Get NetCDF
 
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -1460 +1460 @@
         #Get NetCDF
 
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -1542 +1542 @@
         #Get NetCDF
 
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -1681 +1681 @@
         #Get NetCDF
 
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -1828 +1828 @@
         #Get NetCDF
 
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -1991 +1991 @@
         #Get NetCDF
 
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -2182 +2182 @@
         #Get NetCDF
 
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -2330 +2330 @@
         #Get NetCDF
 
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -2443 +2443 @@
         #Get NetCDF
 
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -2593 +2593 @@
         #Get NetCDF
 
-        fid = NetCDFFile(swwfile, 'r')
+        fid = NetCDFFile(swwfile, netcdf_mode_r)
 
         # Get the variables
@@ -2704 +2704 @@
         #Get NetCDF
 
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -2800 +2800 @@
 
         # Check contents in NetCDF
-        fid = NetCDFFile(sww.filename, 'r')
+        fid = NetCDFFile(sww.filename, netcdf_mode_r)
 
         # Get the variables
@@ -3174 +3174 @@
         from anuga.coordinate_transforms.redfearn import redfearn
         import os
-        fid1 = NetCDFFile('test_ha.nc','w')
-        fid2 = NetCDFFile('test_ua.nc','w')
-        fid3 = NetCDFFile('test_va.nc','w')
-        fid4 = NetCDFFile('test_e.nc','w')
+        fid1 = NetCDFFile('test_ha.nc',netcdf_mode_w)
+        fid2 = NetCDFFile('test_ua.nc',netcdf_mode_w)
+        fid3 = NetCDFFile('test_va.nc',netcdf_mode_w)
+        fid4 = NetCDFFile('test_e.nc',netcdf_mode_w)
 
         h1_list = [150.66667,150.83334,151.]
@@ -3262 +3262 @@
         fid4.sync(); fid4.close()
 
-        fid1 = NetCDFFile('test_ha.nc','r')
-        fid2 = NetCDFFile('test_e.nc','r')
-        fid3 = NetCDFFile('test_va.nc','r')
+        fid1 = NetCDFFile('test_ha.nc',netcdf_mode_r)
+        fid2 = NetCDFFile('test_e.nc',netcdf_mode_r)
+        fid3 = NetCDFFile('test_va.nc',netcdf_mode_r)
 
 
@@ -3334 +3334 @@
         from anuga.coordinate_transforms.redfearn import redfearn
         import os
-        fid1 = NetCDFFile('test_ha.nc','w')
-        fid2 = NetCDFFile('test_ua.nc','w')
-        fid3 = NetCDFFile('test_va.nc','w')
-        fid4 = NetCDFFile('test_e.nc','w')
+        fid1 = NetCDFFile('test_ha.nc',netcdf_mode_w)
+        fid2 = NetCDFFile('test_ua.nc',netcdf_mode_w)
+        fid3 = NetCDFFile('test_va.nc',netcdf_mode_w)
+        fid4 = NetCDFFile('test_e.nc',netcdf_mode_w)
 
         h1_list = [150.66667,150.83334,151.]
@@ -3426 +3426 @@
         fid4.sync(); fid4.close()
 
-        fid1 = NetCDFFile('test_ha.nc','r')
-        fid2 = NetCDFFile('test_e.nc','r')
-        fid3 = NetCDFFile('test_va.nc','r')
+        fid1 = NetCDFFile('test_ha.nc',netcdf_mode_r)
+        fid2 = NetCDFFile('test_e.nc',netcdf_mode_r)
+        fid3 = NetCDFFile('test_va.nc',netcdf_mode_r)
 
 
@@ -3491 +3491 @@
 
         #Read output file 'small.sww'
-        #fid = NetCDFFile('small.sww', 'r')
+        #fid = NetCDFFile('small.sww', netcdf_mode_r)
         fid = NetCDFFile(self.test_MOST_file + '.sww')
 
@@ -3969 +3969 @@
 
         filename = root + '.dem'
-        fid = NetCDFFile(filename, 'w')
+        fid = NetCDFFile(filename, netcdf_mode_w)
 
         fid.institution = 'Geoscience Australia'
@@ -4026 +4026 @@
 
         #Open decimated NetCDF file
-        fid = NetCDFFile(root + '_100.dem', 'r')
+        fid = NetCDFFile(root + '_100.dem', netcdf_mode_r)
 
         # Get decimated elevation
@@ -4052 +4052 @@
 
         filename = root + '.dem'
-        fid = NetCDFFile(filename, 'w')
+        fid = NetCDFFile(filename, netcdf_mode_w)
 
         fid.institution = 'Geoscience Australia'
@@ -4121 +4121 @@
 
         #Open decimated NetCDF file
-        fid = NetCDFFile(root + '_100.dem', 'r')
+        fid = NetCDFFile(root + '_100.dem', netcdf_mode_r)
 
         # Get decimated elevation
@@ -4304 +4304 @@
         # check the sww file
 
-        fid = NetCDFFile(sww_file, 'r')    #Open existing file for read
+        fid = NetCDFFile(sww_file, netcdf_mode_r)    #Open existing file for read
         x = fid.variables['x'][:]
         y = fid.variables['y'][:]
@@ -4614 +4614 @@
         # check the sww file
 
-        fid = NetCDFFile(sww_file, 'r')    #Open existing file for read
+        fid = NetCDFFile(sww_file, netcdf_mode_r)    #Open existing file for read
         x = fid.variables['x'][:]
         y = fid.variables['y'][:]
@@ -4808 +4808 @@
         # check the sww file
 
-        fid = NetCDFFile(sww_file, 'r')    #Open existing file for read
+        fid = NetCDFFile(sww_file, netcdf_mode_r)    #Open existing file for read
         x = fid.variables['x'][:]
         y = fid.variables['y'][:]
@@ -7118 +7118 @@
 
             # Read in sts file for this source file
-            fid = NetCDFFile(sts_name_out+'.sts', 'r') # Open existing file for read
+            fid = NetCDFFile(sts_name_out+'.sts', netcdf_mode_r) # Open existing file for read
             x = fid.variables['x'][:]+fid.xllcorner    # x-coordinates of vertices
             y = fid.variables['y'][:]+fid.yllcorner    # y-coordinates of vertices
@@ -7274 +7274 @@
         
         # read in sts file for combined source
-        fid = NetCDFFile(sts_name_out+'.sts', 'r')    #Open existing file for read
+        fid = NetCDFFile(sts_name_out+'.sts', netcdf_mode_r)    #Open existing file for read
         x = fid.variables['x'][:]+fid.xllcorner   #x-coordinates of vertices
         y = fid.variables['y'][:]+fid.yllcorner   #y-coordinates of vertices
@@ -10007 +10007 @@
         
         filename = tempfile.mktemp("_data_manager.sww")
-        outfile = NetCDFFile(filename, "w")
+        outfile = NetCDFFile(filename, netcdf_mode_w)
         points_utm = array([[0.,0.],[1.,1.], [0.,1.]])
         volumes = (0,1,2)
@@ -10039 +10039 @@
         
         filename = tempfile.mktemp("_data_manager.sww")
-        outfile = NetCDFFile(filename, "w")
+        outfile = NetCDFFile(filename, netcdf_mode_w)
         points_utm = array([[0.,0.],[1.,1.], [0.,1.]])
         volumes = (0,1,2)
@@ -10074 +10074 @@
         
         filename = tempfile.mktemp("_data_manager.sww")
-        outfile = NetCDFFile(filename, "w")
+        outfile = NetCDFFile(filename, netcdf_mode_w)
         points_utm = array([[0.,0.],[1.,1.], [0.,1.]])
         volumes = (0,1,2)
@@ -10113 +10113 @@
         
         filename = tempfile.mktemp("_data_manager.sww")
-        outfile = NetCDFFile(filename, "w")
+        outfile = NetCDFFile(filename, netcdf_mode_w)
         points_utm = array([[0.,0.],[1.,1.], [0.,1.]])
         volumes = (0,1,2)
@@ -10149 +10149 @@
         
         filename = tempfile.mktemp("_data_manager.sww")
-        outfile = NetCDFFile(filename, "w")
+        outfile = NetCDFFile(filename, netcdf_mode_w)
         points_utm = array([[0.,0.],[1.,1.], [0.,1.]])
         volumes = (0,1,2)
@@ -10472 +10472 @@
 
         # Check that mimimum_storable_height works
-        fid = NetCDFFile(swwfile, 'r') # Open existing file
+        fid = NetCDFFile(swwfile, netcdf_mode_r) # Open existing file
         
         stage = fid.variables['stage'][:]
@@ -11049 +11049 @@
 
     suite = unittest.makeSuite(Test_Data_Manager,'test')
-    #suite = unittest.makeSuite(Test_Data_Manager,'test_file_boundary_stsI_beyond_model_time')
+#    suite = unittest.makeSuite(Test_Data_Manager,'test_export_gridIII')
     #suite = unittest.makeSuite(Test_Data_Manager,'test_file_boundary_stsIV_sinewave_ordering')
     #suite = unittest.makeSuite(Test_Data_Manager,'test_get_flow_through_cross_section_with_geo')

anuga_core/source/anuga/shallow_water/test_most2nc.py

@@ -1 +1 @@
 import unittest
 from Numeric import allclose, asarray
+from Scientific.IO.NetCDF import NetCDFFile
 import most2nc
-from Scientific.IO.NetCDF import NetCDFFile
 import os
 

anuga_core/source/anuga/shallow_water/test_shallow_water_domain.py

@@ -6 +6 @@
 
 from anuga.config import g, epsilon
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 from Numeric import allclose, alltrue, array, zeros, ones, Float, take
 from anuga.utilities.numerical_tools import mean
@@ -6188 +6189 @@
 
             #Get NetCDF
-            fid = NetCDFFile(domain.writer.filename, 'r')
+            fid = NetCDFFile(domain.writer.filename, netcdf_mode_r)
             stage_file = fid.variables['stage']
             

anuga_core/source/anuga/shallow_water/test_system.py

@@ -16 +16 @@
 from anuga.pmesh.mesh import Mesh
 from anuga.abstract_2d_finite_volumes.pmesh2domain import pmesh_instance_to_domain_instance
-
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 
 
@@ -114 +114 @@
 
         # do an assertion on the time of the produced sww file
-        fid = NetCDFFile(filename, 'r')    #Open existing file for read
+        fid = NetCDFFile(filename, netcdf_mode_r)    #Open existing file for read
         times = fid.variables['time'][:]
         #print "times", times
@@ -165 +165 @@
 
         # do an assertion on the time of the produced sww file
-        fid = NetCDFFile(filename, 'r')    #Open existing file for read
+        fid = NetCDFFile(filename, netcdf_mode_r)    #Open existing file for read
         times = fid.variables['time'][:]
         stage = fid.variables['stage'][:]

anuga_core/source/anuga/test_all.py

@@ -20 +20 @@
 exclude_files = []
 
-#if sys.platform != 'win32':  
+#if sys.platform != 'win32':
 #    exclude_files.append('test_advection.py') #Weave doesn't work on Linux
-
 # Exclude test_advection on all platforms for the time being. See ticket:205
 #exclude_files.append('test_advection.py') #Weave doesn't work on Linux
 
+# Directories that should not be searched for test files.
 
-# Directories that should not be searched for test files.    
-exclude_dirs = ['pypar_dist', #Special requirements
+exclude_dirs = ['pypar_dist',                        #Special requirements
                 'props', 'wcprops', 'prop-base', 'text-base', '.svn', #Svn
                 'tmp']
-
 
 print "The following directories will be skipped over;"
@@ -38 +36 @@
 print ""
 
+
 def get_test_files(path):
-
-
     try:
         files = os.listdir(path)
@@ -53 +50 @@
     path_files = []
     for file in files:
-
         absolute_filename = path + os.sep + file
 
         #sys.path.append('pmesh')
         if os.path.isdir(absolute_filename):
-            sys.path.append(file) #FIXME: May cause name conflicts between pyvolution\mesh.py and pmesh\mesh.py on some systems
+            # FIXME: May cause name conflicts between pyvolution\mesh.py and
+            #        pmesh\mesh.py on some systems
+            sys.path.append(file)
             path_files.append(file)
-            print  file + ',', 
-            more_test_files, more_path_files =\
-                             get_test_files(absolute_filename)
-            
+            print file + ',',
+            more_test_files, more_path_files = \
+                    get_test_files(absolute_filename)
             test_files += more_test_files
             path_files += more_path_files
@@ -70 +67 @@
         else:
             pass
-        
+
     return test_files, path_files
-
 
 
@@ -83 +79 @@
 
     files.sort() # Ensure same order on all platforms
-    
+
     print
     print
@@ -89 +85 @@
     print
     print 'Files tested;'
-    #print_files = []
     for file in files:
-        #print_files += file + ' '
         print file + ',',
     print
@@ -98 +92 @@
         for file in exclude_files:
             print 'WARNING: File '+ file + ' to be excluded from testing'
-            try:    
+            try:
                 files.remove(file)
             except ValueError, e:
-                msg = 'File "%s" was not found in test suite.\n' %file
-                msg += 'Original error is "%s"\n' %e
-                msg += 'Perhaps it should be removed from exclude list?' 
+                msg = 'File "%s" was not found in test suite.\n' % file
+                msg += 'Original error is "%s"\n' % e
+                msg += 'Perhaps it should be removed from exclude list?'
                 raise Exception, msg
 
@@ -109 +103 @@
     moduleNames = map(filenameToModuleName, files)
     modules = map(__import__, moduleNames)
-    
+
     # Fix up the system path
     for file in path_files:
         sys.path.remove(file)
-        
+
     load = unittest.defaultTestLoader.loadTestsFromModule
     testCaseClasses = map(load, modules)
 
-    
     if test_verbose is True:
         # Test the code by setting verbose to True.
@@ -128 +121 @@
                     # these are the test functions
                     try:
-                        # Calls class method set_verbose in the test case classes
-                        # print 'Tests', tests._tests[0]
-                        # print 'Type', type(tests._tests[0])                        
+                        # Calls class method set_verbose in test case classes
                         tests._tests[0].set_verbose()
                     except:
-                        pass # No all classes have set_verbose
+                        pass                # No all classes have set_verbose
     return unittest.TestSuite(testCaseClasses)
+
 
 def check_anuga_import():
@@ -143 +135 @@
         print "Python cannot import ANUGA module."
         print "Check you have followed all steps of its installation."
-        import sys; sys.exit() 
+        import sys
+        sys.exit()
 
-    
+
 if __name__ == '__main__':
     check_anuga_import()
+
     if len(sys.argv) > 1 and sys.argv[1][0].upper() == 'V':
         test_verbose = True
-        saveout = sys.stdout   
+        saveout = sys.stdout
         filename = ".temp"
         fid = open(filename, 'w')
         sys.stdout = fid
     else:
-        test_verbose = False        
+        test_verbose = False
     suite = regressionTest(test_verbose)
     runner = unittest.TextTestRunner() #verbosity=2
     runner.run(suite)
-    
+
     # Cleaning up
     if len(sys.argv) > 1 and sys.argv[1][0].upper() == 'V':
-        sys.stdout = saveout 
+        sys.stdout = saveout
         #fid.close() # This was causing an error in windows
         #os.remove(filename)

anuga_core/source/anuga/utilities/test_system_tools.py

@@ -6 +6 @@
 import zlib
 from os.path import join, split, sep
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 
 
@@ -85 +86 @@
             # First file
             filename1 = mktemp(suffix='.nc', dir='.')
-            fid = NetCDFFile(filename1, 'w')
+            fid = NetCDFFile(filename1, netcdf_mode_w)
             fid.createDimension('two', 2)
             fid.createVariable('test_array', Float,
@@ -94 +95 @@
             # Second file
             filename2 = mktemp(suffix='.nc', dir='.')
-            fid = NetCDFFile(filename2, 'w')
+            fid = NetCDFFile(filename2, netcdf_mode_w)
             fid.createDimension('two', 2)
             fid.createVariable('test_array', Float,