Changeset 4500


Ignore:
Timestamp:
May 28, 2007, 1:11:31 PM (17 years ago)
Author:
nick
Message:

added functions which were removed from abstract_2d_finite_volumes.util.py and move tests

Also added umask to the main call of data_manager, this will force all files created by anuga to have the umask defined in the config.py file

Location:
anuga_core/source/anuga/shallow_water
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • anuga_core/source/anuga/shallow_water/data_manager.py

    r4497 r4500  
    6161import csv
    6262import os
     63import shutil
    6364from struct import unpack
    6465import array as p_array
    6566#import time, os
    66 from os import sep, path
     67from os import sep, path, remove, mkdir, access, F_OK, W_OK
    6768
    6869from Numeric import concatenate, array, Float, Int, Int32, resize, sometrue, \
     
    7071     NewAxis, ArrayType
    7172from Scientific.IO.NetCDF import NetCDFFile
     73#from shutil import copy
     74from os.path import exists, basename
    7275
    7376
     
    50725075    def close(self):
    50735076        self.mux_file.close()
    5074 
     5077       
    50755078    #### END URS UNGRIDDED 2 SWW ###
    50765079
     5080       
     5081def start_screen_catcher(dir_name, myid='', numprocs='', extra_info='',
     5082                         print_to_screen=False, verbose=False):
     5083    """
     5084    Used to store screen output and errors to file, if run on multiple
     5085    processes eachprocessor will have its own output and error file.
     5086   
     5087    extra_info - is used as a string that can identify outputs with another
     5088    string eg. '_other'
     5089    """
     5090    import sys
     5091    dir_name = dir_name
     5092    if access(dir_name,W_OK) == 0:
     5093        if verbose: print 'Make directory %s' %dir_name
     5094        if verbose: print "myid", myid
     5095        mkdir (dir_name,0777)
     5096    if myid <>'':
     5097        myid = '_'+str(myid)
     5098    if numprocs <>'':
     5099        numprocs = '_'+str(numprocs)
     5100    if extra_info <>'':
     5101        extra_info = '_'+str(extra_info)
     5102    screen_output_name = dir_name + "screen_output%s%s%s.txt" %(myid,numprocs,extra_info)
     5103    screen_error_name = dir_name + "screen_error%s%s%s.txt" %(myid,numprocs,extra_info)
     5104    print screen_output_name
     5105    #used to catch screen output to file
     5106    sys.stdout = Screen_Catcher(screen_output_name)
     5107    sys.stderr = Screen_Catcher(screen_error_name)
     5108
     5109class Screen_Catcher:
     5110    """this simply catches the screen output and stores it to file defined by
     5111    start_screen_catcher (above)
     5112    """
     5113   
     5114    def __init__(self, filename):
     5115        self.filename = filename
     5116 
     5117        if exists(self.filename)is True:
     5118            print'Old existing file "%s" has been deleted' %(self.filename)
     5119            remove(self.filename)
     5120
     5121    def write(self, stuff):
     5122        fid = open(self.filename, 'a')
     5123        fid.write(stuff)
     5124#        if print_to_screen: print stuff
     5125
     5126def copy_code_files(dir_name, filename1, filename2):
     5127    """Copies "filename1" and "filename2" to "dir_name". Very useful for
     5128    information management
     5129    filename1 and filename2 are both absolute pathnames   
     5130    """
     5131
     5132    if access(dir_name,F_OK) == 0:
     5133        print 'Make directory %s' %dir_name
     5134        mkdir (dir_name,0777)
     5135    shutil.copy(filename1, dir_name + sep + basename(filename1))
     5136    shutil.copy(filename2, dir_name + sep + basename(filename2))
     5137#    copy (__file__, project.output_run_time_dir + basename(__file__))
     5138    print 'Files %s and %s copied' %(filename1, filename2)
     5139
     5140def get_data_from_file(filename,separator_value = ','):
     5141    """
     5142    Read in data information from file and
     5143   
     5144    Returns:
     5145        header_fields, a string? of the first line separated
     5146        by the 'separator_value'
     5147       
     5148        data, a array (N data columns X M lines) in the file
     5149        excluding the header
     5150       
     5151    NOTE: wont deal with columns with different lenghts and there must be
     5152    no blank lines at the end.
     5153    """
     5154   
     5155    from os import sep, getcwd, access, F_OK, mkdir
     5156    from Numeric import array, resize,shape,Float
     5157    import string
     5158    fid = open(filename)
     5159    lines = fid.readlines()
     5160   
     5161    fid.close()
     5162   
     5163    header_line = lines[0]
     5164    header_fields = header_line.split(separator_value)
     5165
     5166    #array to store data, number in there is to allow float...
     5167    #i'm sure there is a better way!
     5168    data=array([],typecode=Float)
     5169    data=resize(data,((len(lines)-1),len(header_fields)))
     5170#    print 'number of fields',range(len(header_fields))
     5171#    print 'number of lines',len(lines), shape(data)
     5172#    print'data',data[1,1],header_line
     5173
     5174    array_number = 0
     5175    line_number = 1
     5176    while line_number < (len(lines)):
     5177        for i in range(len(header_fields)):
     5178            #this get line below the header, explaining the +1
     5179            #and also the line_number can be used as the array index
     5180            fields = lines[line_number].split(separator_value)
     5181            #assign to array
     5182            data[array_number,i] = float(fields[i])
     5183           
     5184        line_number = line_number +1
     5185        array_number = array_number +1
     5186       
     5187    return header_fields, data
     5188
     5189def store_parameters(verbose=False,**kwargs):
     5190    """
     5191    Must have a file_name keyword arg, this is what is writing to.
     5192    might be a better way to do this using CSV module Writer and writeDict
     5193   
     5194    writes file to "output_dir" unless "completed" is in kwargs, then it writes to
     5195    "file_name" kwargs
     5196     Returns a object which is a subset of the original
     5197        and the data points and attributes in this new object refer to
     5198        the indices provided
     5199       
     5200        Input
     5201            indices- a list of integers that represent the new object
     5202        Output
     5203            New geospatial data object representing points specified by
     5204            the indices
     5205    """
     5206    import types
     5207    import os
     5208   
     5209    # Check that kwargs is a dictionary
     5210    if type(kwargs) != types.DictType:
     5211        raise TypeError
     5212   
     5213    try:
     5214        kwargs['completed']
     5215        completed=True
     5216    except:
     5217        completed=False
     5218 
     5219    #get file name and removes from dict and assert that a file_name exists
     5220    if completed:
     5221        try:
     5222            file = str(kwargs.pop('file_name'))
     5223        except:
     5224            raise 'kwargs must have file_name'
     5225    else:
     5226        try:
     5227            file = str(kwargs.pop('output_dir'))+'detail_temp.csv'
     5228        except:
     5229            raise 'kwargs must have output_dir'
     5230       
     5231    #extracts the header info and the new line info
     5232    line=''
     5233    header=''
     5234    count=0
     5235    keys = kwargs.keys()
     5236    keys.sort()
     5237   
     5238#    for k in kwargs.keys():
     5239    #used the sorted keys to create the header and line data
     5240    for k in keys:
     5241        print "%s = %s" %(k, kwargs[k])
     5242        header = header+str(k)
     5243        line = line+str(kwargs[k])
     5244        count+=1
     5245        if count <len(kwargs):
     5246            header = header+','
     5247            line = line+','
     5248
     5249
     5250    # checks the header info, if the same, then write, if not create a new file
     5251    #try to open!
     5252#    print'file name',file
     5253    try:
     5254        fid = open(file,"r")
     5255        file_header=fid.readline()
     5256        fid.close()
     5257        if verbose: print 'read file header %s' %file_header
     5258       
     5259    except:
     5260        msg = 'try to create new file',file
     5261        if verbose: print msg
     5262        #tries to open file, maybe directory is bad
     5263        try:
     5264            fid = open(file,"w")
     5265            fid.writelines(header+'\n')
     5266            fid.close()
     5267            file_header=header
     5268        except:
     5269            msg = 'cannot create new file',file
     5270            raise msg
     5271           
     5272    #if header is same or this is a new file
     5273    if file_header.strip('\n')==str(header):
     5274        fid=open(file,"a")
     5275        #write new line
     5276        fid.writelines(line+'\n')
     5277        fid.close()
     5278    else:
     5279        #backup plan, if header is different and has completed will append info to
     5280        #end of details_temp.cvs file in output directory
     5281        file = str(kwargs['output_dir'])+'detail_temp.csv'
     5282        fid=open(file,"a")
     5283        fid.writelines(header+'\n')
     5284        fid.writelines(line+'\n')
     5285        fid.close()
     5286        print 'file',file_header.strip('\n')
     5287        print 'head',header.strip('\n')
     5288        msg = 'WARNING: File header does not match input info, the input variables have changed, suggest to change file name'
     5289        print msg
     5290
     5291
    50775292#-------------------------------------------------------------
    5078 if __name__ == "__main__":   
    5079         pass
    5080 
     5293if __name__ == "__main__":
     5294    #setting umask from config to force permissions for all files and directories
     5295    # created to the same. (it was noticed the "mpirun" doesn't honour the umask
     5296    # set in your .bashrc etc file)
     5297    from config import umask
     5298    import os
     5299    os.umask(umask)
     5300
  • anuga_core/source/anuga/shallow_water/test_data_manager.py

    r4489 r4500  
    67006700                                           new_origin)),points_utm)
    67016701        os.remove(filename)
     6702       
     6703    def test_get_data_from_file(self):
     6704#    from anuga.abstract_2d_finite_volumes.util import get_data_from_file
     6705       
     6706        import os
     6707       
     6708        fileName = tempfile.mktemp(".txt")
     6709#        print"filename",fileName
     6710        file = open(fileName,"w")
     6711        file.write("elevation, stage\n\
     67121.0, 3  \n\
     67130.0, 4 \n\
     67144.0, 3 \n\
     67151.0, 6 \n")
     6716        file.close()
     6717       
     6718        header,x = get_data_from_file(fileName)
     6719#        print 'x',x
     6720        os.remove(fileName)
     6721       
     6722        assert allclose(x[:,0], [1.0, 0.0,4.0, 1.0])
     6723       
     6724    def test_get_data_from_file1(self):
     6725#    from anuga.abstract_2d_finite_volumes.util import get_data_from_file
     6726       
     6727        import os
     6728       
     6729        fileName = tempfile.mktemp(".txt")
     6730#        print"filename",fileName
     6731        file = open(fileName,"w")
     6732        file.write("elevation stage\n\
     67331.3 3  \n\
     67340.0 4 \n\
     67354.5 3.5 \n\
     67361.0 6 \n")
     6737        file.close()
     6738       
     6739        header, x = get_data_from_file(fileName,separator_value=' ')
     6740        os.remove(fileName)
     6741#        x = get_data_from_file(fileName)
     6742#        print '1x',x[:,0]
     6743       
     6744        assert allclose(x[:,0], [1.3, 0.0,4.5, 1.0])
     6745       
     6746    def xxxtest_store_parameters(self):
     6747       
     6748        from os import sep, getenv
     6749       
     6750        home = getenv('INUNDATIONHOME')
     6751        output_dir=home+sep+'data'
     6752       
     6753        kwargs = {'file_name':'new2.txt','output_dir':output_dir,'who':'me', 'what':'detail', 'how':2, 'why':241,'Completed':'yes'}
     6754#        {'data_origin': data_georef.get_origin(),
     6755#                  'mesh_origin': mesh_georef.get_origin(),
     6756#                  'alpha': alpha,
     6757#                  'verbose': verbose}
     6758       
     6759#        fileName = tempfile.mktemp(".csv")
     6760#        store_parameters(kwargs,fileName)
     6761#        file_name='temp.csv'
     6762        store_parameters(**kwargs)
     6763#        store_parameters(file_name=file_name, scenario_name='dampier',\
     6764#                         who="me", time_thining=12,tide=2.6)
     6765
     6766       
     6767        fid = open(str(kwargs.pop('file_name')))
     6768        header = fid.readline()
     6769        line = fid.readline()
     6770        fid.close()
     6771        print'header',header,'lines',line
     6772       
     6773        #file exists
     6774#        assert access(file_name,F_OK)
     6775        a=['who,time']
     6776#        assert allclose(header, a)
     6777#        assert allclose(header, [who,time])
     6778#        assert header=='who,time1
     6779#        assert allclose(line,['me',12])
     6780
     6781
     6782
    67026783#-------------------------------------------------------------
    67036784if __name__ == "__main__":
Note: See TracChangeset for help on using the changeset viewer.