Changeset 7772

Timestamp:

Jun 3, 2010, 12:45:54 PM (14 years ago)

Author:

hudson

Message:

Fixed a bunch of failing tests - only 3 still failing.

Location:

trunk/anuga_core/source/anuga

Files:

• abstract_2d_finite_volumes/util.py (modified) (1 diff)
• file/csv_file.py (modified) (2 diffs)
• file/test_csv.py (modified) (2 diffs)
• interface.py (modified) (1 diff)
• shallow_water/data_manager.py (modified) (1 diff)
• shallow_water/test_data_manager.py (modified) (3 diffs)

trunk/anuga_core/source/anuga/abstract_2d_finite_volumes/util.py

@@ -330 +330 @@
             mkdir(dir)
     return dir
-
-
-##
-# @brief 
-# @param filename 
-# @param separator_value 
-# @return 
-# @note TEMP
-def get_data_from_file(filename, separator_value=','):
-    """Temporary Interface to new location"""
-    from anuga.shallow_water.data_manager import \
-                        get_data_from_file as dm_get_data_from_file
-    log.critical('get_data_from_file has moved from util.py')
-    log.critical('Please use "from anuga.shallow_water.data_manager import '
-                 'get_data_from_file"')
-    
-    return dm_get_data_from_file(filename,separator_value = ',')
 
 

trunk/anuga_core/source/anuga/file/csv_file.py

@@ -70 +70 @@
 
           
-def load_csv_as_array(file_name, delimiter=','):
+def load_csv_as_array(file_name, delimiter = ','):
     """
     Convert CSV files of the form:
@@ -89 +89 @@
     X, _ = load_csv_as_dict(file_name, delimiter=delimiter)
 
-    Y = {}
+
+    # Return result as a dict of arrays
+    ret = {}
     for key in X.keys():
-        Y[key] = num.array([float(x) for x in X[key]])
-
-    return Y
-
+        ret[key] = num.array([float(x) for x in X[key]])
+            
+    return ret
+
+
+def load_csv_as_matrix(file_name, delimiter = ','):
+    """
+    Convert CSV files of the form:
+
+    time, discharge, velocity
+    0.0,  1.2,       0.0
+    0.1,  3.2,       1.1
+    ...
+
+    to a numeric matrix.
+
+    file_name The path to the file to read.
+    delimiter is the delimiter used to separate the fields    
+
+    See underlying function load_csv_as_dict for more details.
+    """
+
+    X, title_indices = load_csv_as_dict(file_name, delimiter=delimiter)
+
+    col_titles = title_indices.keys()
+
+    # Return result as a 2D array
+    ret = num.zeros((len(X[col_titles[0]]), len(title_indices)), float)
+
+    header = []
+    for col_title in col_titles:
+        index = title_indices[col_title]
+        header.append(col_title)
+        for i, x in enumerate(X[col_title]):
+            ret[i, index] = float(x)
+
+    return header, ret
+
+
+
+##
+# @brief Store keyword params into a CSV file.
+# @param verbose True if this function is to be verbose.
+# @param kwargs Dictionary of keyword args to store.
+# @note If kwargs dict contains 'file_name' key, that has the output filename.
+#       If not, make up a filename in the output directory.
+def store_parameters(verbose=False, **kwargs):
+    """
+    Store "kwargs" into a temp csv file, if "completed" is in kwargs,
+    csv file is kwargs[file_name] else it is kwargs[output_dir]+details_temp.csv
+
+    Must have a file_name keyword arg, this is what is writing to.
+    might be a better way to do this using CSV module Writer and writeDict.
+
+    writes file to "output_dir" unless "completed" is in kwargs, then
+    it writes to "file_name" kwargs
+    """
+
+    import types
+
+    # Check that kwargs is a dictionary
+    if type(kwargs) != types.DictType:
+        raise TypeError
+
+    # is 'completed' in kwargs?
+    completed = kwargs.has_key('completed')
+
+    # get file name and removes from dict and assert that a file_name exists
+    if completed:
+        try:
+            file = str(kwargs['file_name'])
+        except:
+            raise 'kwargs must have file_name'
+    else:
+        # write temp file in output directory
+        try:
+            file = str(kwargs['output_dir']) + 'detail_temp.csv'
+        except:
+            raise 'kwargs must have output_dir'
+
+    # extracts the header info and the new line info
+    line = ''
+    header = ''
+    count = 0
+    keys = kwargs.keys()
+    keys.sort()
+
+    # used the sorted keys to create the header and line data
+    for k in keys:
+        header += str(k)
+        line += str(kwargs[k])
+        count += 1
+        if count < len(kwargs):
+            header += ','
+            line += ','
+    header += '\n'
+    line += '\n'
+
+    # checks the header info, if the same, then write, if not create a new file
+    # try to open!
+    try:
+        fid = open(file, 'r')
+        file_header = fid.readline()
+        fid.close()
+        if verbose: log.critical('read file header %s' % file_header)
+    except:
+        msg = 'try to create new file: %s' % file
+        if verbose: log.critical(msg)
+        #tries to open file, maybe directory is bad
+        try:
+            fid = open(file, 'w')
+            fid.write(header)
+            fid.close()
+            file_header=header
+        except:
+            msg = 'cannot create new file: %s' % file
+            raise Exception, msg
+
+    # if header is same or this is a new file
+    if file_header == str(header):
+        fid = open(file, 'a')
+        fid.write(line)
+        fid.close()
+    else:
+        # backup plan,
+        # if header is different and has completed will append info to
+        # end of details_temp.cvs file in output directory
+        file = str(kwargs['output_dir']) + 'detail_temp.csv'
+        fid = open(file, 'a')
+        fid.write(header)
+        fid.write(line)
+        fid.close()
+
+        if verbose:
+            log.critical('file %s', file_header.strip('\n'))
+            log.critical('head %s', header.strip('\n'))
+        if file_header.strip('\n') == str(header):
+            log.critical('they equal')
+
+        msg = 'WARNING: File header does not match input info, ' \
+              'the input variables have changed, suggest you change file name'
+        log.critical(msg)
+

trunk/anuga_core/source/anuga/file/test_csv.py

@@ -4 +4 @@
 import numpy as num
 
-from csv_file import load_csv_as_array, load_csv_as_dict
-
+from csv_file import load_csv_as_array, load_csv_as_dict, store_parameters, \
+                        load_csv_as_matrix
 
 class Test_csv(unittest.TestCase):
@@ -13 +13 @@
     def tearDown(self):
         pass
+
+    def _create_csv_file(self):
+        """
+            Create a dummy csv file.
+            Return its filename.
+        """      
+        filename = tempfile.mktemp(".txt")
+        file = open(filename,"w")
+        file.write("elevation, stage\n\
+1.0, 3  \n\
+0.0, 4 \n\
+4.0, 3 \n\
+1.0, 6 \n")
+        file.close()
+        return filename
         
     def test_get_data_from_file1(self):
-        fileName = tempfile.mktemp(".txt")
-#        print"filename",fileName
-        file = open(fileName,"w")
-        file.write("elevation stage\n\
-1.3 3  \n\
-0.0 4 \n\
-4.5 3.5 \n\
-1.0 6 \n")
-        file.close()
-
-        x = load_csv_as_array(fileName, delimiter=' ')  
-        
-       # header, x = load_csv_as_array(fileName, delimiter=' ')
-        os.remove(fileName)
-
-        assert num.allclose(x['elevation'], [1.3, 0.0,4.5, 1.0])
-        assert num.allclose(x['stage'], [3.0, 4.0,3.5, 6.0])        
-        
+        filename = self._create_csv_file()
+
+        x = load_csv_as_array(filename)  
+        
+        os.remove(filename)
+
+        assert num.allclose(x['elevation'], [1.0, 0.0, 4.0, 1.0])
+        assert num.allclose(x['stage'], [3.0, 4.0, 3.0, 6.0])        
+
+
+    def test_get_data_from_file(self):
+        filename = self._create_csv_file()
+        
+        header,x = load_csv_as_matrix(filename)
+        os.remove(filename)
+        
+        assert num.allclose(x[:,0], [1.0, 0.0,4.0, 1.0])
+
+        
+    def test_store_parameters(self):
+        """tests store temporary file
+        """
+        
+        from os import sep, getenv
+        
+        output_dir=''
+        file_name='details.csv'
+        
+        kwargs = {'file_name':'new2.txt',
+                  'output_dir':output_dir,
+                  'file_name':file_name,
+                  'who':'me',
+                  'what':'detail',
+                  'how':2,
+                  'why':241,
+#                  'completed':345
+                  }
+        store_parameters(verbose=False,**kwargs)
+
+        temp='detail_temp.csv'
+        fid = open(temp)
+        file_header = fid.readline()
+        file_line = fid.readline()
+        fid.close()
+        
+        
+        keys = kwargs.keys()
+        keys.sort()
+        line=''
+        header=''
+        count=0
+        #used the sorted keys to create the header and line data
+        for k in keys:
+#            print "%s = %s" %(k, kwargs[k]) 
+            header = header+str(k)
+            line = line+str(kwargs[k])
+            count+=1
+            if count <len(kwargs):
+                header = header+','
+                line = line+','
+        header+='\n'
+        line+='\n'
+        
+        
+        #file exists
+        assert os.access(temp, os.F_OK)
+        assert header == file_header
+        assert line == file_line
+        
+        os.remove(temp)
+        
+    def test_store_parameters1(self):
+        """tests store in temporary file and other file 
+        """
+        
+        from os import sep, getenv
+        
+        output_dir=''
+        file_name='details.csv'
+        
+        kwargs = {'file_name':'new2.txt',
+                  'output_dir':output_dir,
+                  'file_name':file_name,
+                  'who':'me',
+                  'what':'detail',
+                  'how':2,
+                  'why':241,
+#                  'completed':345
+                  }
+        store_parameters(verbose=False,**kwargs)
+        
+        kwargs['how']=55
+        kwargs['completed']=345
+
+        keys = kwargs.keys()
+        keys.sort()
+        line=''
+        header=''
+        count=0
+        #used the sorted keys to create the header and line data
+        for k in keys:
+#            print "%s = %s" %(k, kwargs[k]) 
+            header = header+str(k)
+            line = line+str(kwargs[k])
+            count+=1
+            if count <len(kwargs):
+                header = header+','
+                line = line+','
+        header+='\n'
+        line+='\n'
+        
+        kwargs['how']=55
+        kwargs['completed']=345
+        
+        store_parameters(verbose=False,**kwargs)
+        
+#        temp='detail_temp.csv'
+        fid = open(file_name)
+        file_header = fid.readline()
+        file_line1 = fid.readline()
+        file_line2 = fid.readline()
+        fid.close()
+        
+        
+        #file exists
+#        print 'header',header,'line',line
+#        print 'file_header',file_header,'file_line1',file_line1,'file_line2',file_line2
+        assert os.access(file_name, os.F_OK)
+        assert header == file_header
+        assert line == file_line1
+        
+        temp='detail_temp.csv'
+        os.remove(temp)
+        os.remove(file_name)        
+        
+    def test_store_parameters2(self):
+        """tests appending the data to the end of an existing file
+        """
+        
+        from os import sep, getenv
+        
+        output_dir=''
+        file_name='details.csv'
+        
+        kwargs = {'file_name':'new2.txt',
+                  'output_dir':output_dir,
+                  'file_name':file_name,
+                  'who':'me',
+                  'what':'detail',
+                  'how':2,
+                  'why':241,
+                  'completed':345
+                  }
+        store_parameters(verbose=False,**kwargs)
+        
+        kwargs['how']=55
+        kwargs['completed']=23.54532
+        
+        store_parameters(verbose=False,**kwargs)
+        
+        keys = kwargs.keys()
+        keys.sort()
+        line=''
+        header=''
+        count=0
+        #used the sorted keys to create the header and line data
+        for k in keys:
+#            print "%s = %s" %(k, kwargs[k]) 
+            header = header+str(k)
+            line = line+str(kwargs[k])
+            count+=1
+            if count <len(kwargs):
+                header = header+','
+                line = line+','
+        header+='\n'
+        line+='\n'
+        
+        fid = open(file_name)
+        file_header = fid.readline()
+        file_line1 = fid.readline()
+        file_line2 = fid.readline()
+        fid.close()
+        
+        assert os.access(file_name, os.F_OK)
+        assert header == file_header
+        assert line == file_line2
+        
+        os.remove(file_name)        
+        
+
+
 
 #################################################################################

trunk/anuga_core/source/anuga/interface.py

@@ -25 +25 @@
 from anuga.abstract_2d_finite_volumes.mesh_factory import rectangular_cross
 
-from anuga.shallow_water.data_manager import export_grid, create_sts_boundary
+from anuga.shallow_water.data_manager import export_grid
 from anuga.shallow_water.data_manager import csv2building_polygons
+
+from anuga.file.sts import create_sts_boundary
 
 from anuga.geometry.polygon import read_polygon, plot_polygons, polygon_area

trunk/anuga_core/source/anuga/shallow_water/data_manager.py

@@ -805 +805 @@
 
 
-##
-# @brief Store keyword params into a CSV file.
-# @param verbose True if this function is to be verbose.
-# @param kwargs Dictionary of keyword args to store.
-# @note If kwargs dict contains 'file_name' key, that has the output filename.
-#       If not, make up a filename in the output directory.
-def store_parameters(verbose=False, **kwargs):
-    """
-    Store "kwargs" into a temp csv file, if "completed" is in kwargs,
-    csv file is kwargs[file_name] else it is kwargs[output_dir]+details_temp.csv
-
-    Must have a file_name keyword arg, this is what is writing to.
-    might be a better way to do this using CSV module Writer and writeDict.
-
-    writes file to "output_dir" unless "completed" is in kwargs, then
-    it writes to "file_name" kwargs
-    """
-
-    import types
-
-    # Check that kwargs is a dictionary
-    if type(kwargs) != types.DictType:
-        raise TypeError
-
-    # is 'completed' in kwargs?
-    completed = kwargs.has_key('completed')
-
-    # get file name and removes from dict and assert that a file_name exists
-    if completed:
-        try:
-            file = str(kwargs['file_name'])
-        except:
-            raise 'kwargs must have file_name'
-    else:
-        # write temp file in output directory
-        try:
-            file = str(kwargs['output_dir']) + 'detail_temp.csv'
-        except:
-            raise 'kwargs must have output_dir'
-
-    # extracts the header info and the new line info
-    line = ''
-    header = ''
-    count = 0
-    keys = kwargs.keys()
-    keys.sort()
-
-    # used the sorted keys to create the header and line data
-    for k in keys:
-        header += str(k)
-        line += str(kwargs[k])
-        count += 1
-        if count < len(kwargs):
-            header += ','
-            line += ','
-    header += '\n'
-    line += '\n'
-
-    # checks the header info, if the same, then write, if not create a new file
-    # try to open!
-    try:
-        fid = open(file, 'r')
-        file_header = fid.readline()
-        fid.close()
-        if verbose: log.critical('read file header %s' % file_header)
-    except:
-        msg = 'try to create new file: %s' % file
-        if verbose: log.critical(msg)
-        #tries to open file, maybe directory is bad
-        try:
-            fid = open(file, 'w')
-            fid.write(header)
-            fid.close()
-            file_header=header
-        except:
-            msg = 'cannot create new file: %s' % file
-            raise Exception, msg
-
-    # if header is same or this is a new file
-    if file_header == str(header):
-        fid = open(file, 'a')
-        fid.write(line)
-        fid.close()
-    else:
-        # backup plan,
-        # if header is different and has completed will append info to
-        # end of details_temp.cvs file in output directory
-        file = str(kwargs['output_dir']) + 'detail_temp.csv'
-        fid = open(file, 'a')
-        fid.write(header)
-        fid.write(line)
-        fid.close()
-
-        if verbose:
-            log.critical('file %s', file_header.strip('\n'))
-            log.critical('head %s', header.strip('\n'))
-        if file_header.strip('\n') == str(header):
-            log.critical('they equal')
-
-        msg = 'WARNING: File header does not match input info, ' \
-              'the input variables have changed, suggest you change file name'
-        log.critical(msg)
-
-
 
 ##

trunk/anuga_core/source/anuga/shallow_water/test_data_manager.py

@@ -16 +16 @@
 from Scientific.IO.NetCDF import NetCDFFile
 
+from anuga.anuga_exceptions import ANUGAError
 from anuga.shallow_water.data_manager import *
 from anuga.shallow_water.sww_file import SWW_file
@@ -23 +24 @@
 from anuga.utilities.system_tools import get_pathname_from_package
 from anuga.utilities.file_utils import del_dir
-from anuga.file.csv_file import load_csv_as_dict, load_csv_as_array
-from anuga.anuga_exceptions import ANUGAError
 from anuga.utilities.numerical_tools import ensure_numeric, mean
 from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a
 from anuga.config import netcdf_float, epsilon, g
+
+from anuga.file.csv_file import load_csv_as_dict, load_csv_as_array
+from anuga.file.sts import create_sts_boundary
 
 
@@ -2252 +2254 @@
                                            new_origin)),points_utm)
         os.remove(filename)
-        
-    def test_get_data_from_file(self):
-#    from anuga.abstract_2d_finite_volumes.util import get_data_from_file
-        
-        import os
-       
-        fileName = tempfile.mktemp(".txt")
-#        print"filename",fileName
-        file = open(fileName,"w")
-        file.write("elevation, stage\n\
-1.0, 3  \n\
-0.0, 4 \n\
-4.0, 3 \n\
-1.0, 6 \n")
-        file.close()
-        
-        header,x = get_data_from_file(fileName)
-#        print 'x',x
-        os.remove(fileName)
-        
-        assert num.allclose(x[:,0], [1.0, 0.0,4.0, 1.0])
-        
-    def test_get_data_from_file1(self):
-#    from anuga.abstract_2d_finite_volumes.util import get_data_from_file
-        
-        import os
-       
-        fileName = tempfile.mktemp(".txt")
-#        print"filename",fileName
-        file = open(fileName,"w")
-        file.write("elevation stage\n\
-1.3 3  \n\
-0.0 4 \n\
-4.5 3.5 \n\
-1.0 6 \n")
-        file.close()
-        
-        header, x = get_data_from_file(fileName,separator_value=' ')
-        os.remove(fileName)
-#        x = get_data_from_file(fileName)
-#        print '1x',x[:,0]
-        
-        assert num.allclose(x[:,0], [1.3, 0.0,4.5, 1.0])
-        
-    def test_store_parameters(self):
-        """tests store temporary file
-        """
-        
-        from os import sep, getenv
-        
-        output_dir=''
-        file_name='details.csv'
-        
-        kwargs = {'file_name':'new2.txt',
-                  'output_dir':output_dir,
-                  'file_name':file_name,
-                  'who':'me',
-                  'what':'detail',
-                  'how':2,
-                  'why':241,
-#                  'completed':345
-                  }
-        store_parameters(verbose=False,**kwargs)
-
-        temp='detail_temp.csv'
-        fid = open(temp)
-        file_header = fid.readline()
-        file_line = fid.readline()
-        fid.close()
-        
-        
-        keys = kwargs.keys()
-        keys.sort()
-        line=''
-        header=''
-        count=0
-        #used the sorted keys to create the header and line data
-        for k in keys:
-#            print "%s = %s" %(k, kwargs[k]) 
-            header = header+str(k)
-            line = line+str(kwargs[k])
-            count+=1
-            if count <len(kwargs):
-                header = header+','
-                line = line+','
-        header+='\n'
-        line+='\n'
-        
-        
-        #file exists
-        assert access(temp,F_OK)
-        assert header == file_header
-        assert line == file_line
-        
-        os.remove(temp)
-        
-    def test_store_parameters1(self):
-        """tests store in temporary file and other file 
-        """
-        
-        from os import sep, getenv
-        
-        output_dir=''
-        file_name='details.csv'
-        
-        kwargs = {'file_name':'new2.txt',
-                  'output_dir':output_dir,
-                  'file_name':file_name,
-                  'who':'me',
-                  'what':'detail',
-                  'how':2,
-                  'why':241,
-#                  'completed':345
-                  }
-        store_parameters(verbose=False,**kwargs)
-        
-        kwargs['how']=55
-        kwargs['completed']=345
-
-        keys = kwargs.keys()
-        keys.sort()
-        line=''
-        header=''
-        count=0
-        #used the sorted keys to create the header and line data
-        for k in keys:
-#            print "%s = %s" %(k, kwargs[k]) 
-            header = header+str(k)
-            line = line+str(kwargs[k])
-            count+=1
-            if count <len(kwargs):
-                header = header+','
-                line = line+','
-        header+='\n'
-        line+='\n'
-        
-        kwargs['how']=55
-        kwargs['completed']=345
-        
-        store_parameters(verbose=False,**kwargs)
-        
-#        temp='detail_temp.csv'
-        fid = open(file_name)
-        file_header = fid.readline()
-        file_line1 = fid.readline()
-        file_line2 = fid.readline()
-        fid.close()
-        
-        
-        #file exists
-#        print 'header',header,'line',line
-#        print 'file_header',file_header,'file_line1',file_line1,'file_line2',file_line2
-        assert access(file_name,F_OK)
-        assert header == file_header
-        assert line == file_line1
-        
-        temp='detail_temp.csv'
-        os.remove(temp)
-        os.remove(file_name)        
-        
-    def test_store_parameters2(self):
-        """tests appending the data to the end of an existing file
-        """
-        
-        from os import sep, getenv
-        
-        output_dir=''
-        file_name='details.csv'
-        
-        kwargs = {'file_name':'new2.txt',
-                  'output_dir':output_dir,
-                  'file_name':file_name,
-                  'who':'me',
-                  'what':'detail',
-                  'how':2,
-                  'why':241,
-                  'completed':345
-                  }
-        store_parameters(verbose=False,**kwargs)
-        
-        kwargs['how']=55
-        kwargs['completed']=23.54532
-        
-        store_parameters(verbose=False,**kwargs)
-        
-        keys = kwargs.keys()
-        keys.sort()
-        line=''
-        header=''
-        count=0
-        #used the sorted keys to create the header and line data
-        for k in keys:
-#            print "%s = %s" %(k, kwargs[k]) 
-            header = header+str(k)
-            line = line+str(kwargs[k])
-            count+=1
-            if count <len(kwargs):
-                header = header+','
-                line = line+','
-        header+='\n'
-        line+='\n'
-        
-        fid = open(file_name)
-        file_header = fid.readline()
-        file_line1 = fid.readline()
-        file_line2 = fid.readline()
-        fid.close()
-        
-        assert access(file_name,F_OK)
-        assert header == file_header
-        assert line == file_line2
-        
-        os.remove(file_name)