Ignore:
Timestamp:
Jun 30, 2009, 2:07:41 PM (16 years ago)
Author:
ole
Message:

Merged numpy branch back into the trunk.

In ~/sandpit/anuga/anuga_core/source
svn merge -r 6246:HEAD ../../branches/numpy .

In ~/sandpit/anuga/anuga_validation
svn merge -r 6417:HEAD ../branches/numpy_anuga_validation .

In ~/sandpit/anuga/misc
svn merge -r 6809:HEAD ../branches/numpy_misc .

For all merges, I used numpy version where conflicts existed

The suites test_all.py (in source/anuga) and validate_all.py passed using Python2.5 with numpy on my Ubuntu Linux box.

Location:
anuga_validation/automated_validation_tests/patong_beach_validation
Files:
4 edited
1 copied

Legend:

Unmodified
Added
Removed
  • anuga_validation/automated_validation_tests/patong_beach_validation/build_urs_boundary.py

    r6844 r7276  
    1111
    1212from Scientific.IO.NetCDF import NetCDFFile
    13 import Numeric as num
     13import numpy as num
    1414
    1515from anuga.shallow_water.data_manager import urs2sts
    16 
    17 import Numeric as num
    1816
    1917
     
    148146
    149147        weight_factor = 1.0
    150         mux_weights = weight_factor*num.ones(len(mux_filenames), num.Float)
     148        mux_weights = weight_factor*num.ones(len(mux_filenames), num.float)
    151149           
    152150        order_filename = project.urs_order
  • anuga_validation/automated_validation_tests/patong_beach_validation/cmpsww.py

    r7040 r7276  
    1313import getopt
    1414from Scientific.IO.NetCDF import NetCDFFile
    15 import Numeric as num
     15import numpy as num
    1616from anuga.config import netcdf_mode_r
    1717
     
    2121#####
    2222
    23 # allowable 'slop' when testing two float values
    24 epsilon = 1.0e-9
     23# Global for the '-q' quiet flag
     24quiet = None
     25
     26# default tolerances - same as numpy defaults
     27default_abs_tolerance = 1.0e-08
     28default_rel_tolerance = 1.0000000000000001e-05
    2529
    2630# Global attributes that should exist and be same in both files
     
    6973# @param variables A list of variable names to compare.
    7074# @return Returns if files 'equal', else raises RuntimeError.
    71 def files_are_the_same(files, globals=None, timesteps=None, variables=None):
     75def files_are_the_same(files, globals=None, timesteps=None, variables=None,
     76                       rel_tolerance=default_rel_tolerance,
     77                       abs_tolerance=default_abs_tolerance):
    7278    # split out the filenames and check they exist
    7379    (file1, file2) = files
     
    222228    #####
    223229
     230    error_msg = ''
     231    glob_vars_bad = {}
     232    data_vars_bad = {}
     233
    224234    # check values of global attributes
    225235    for glob_name in globals:
    226236        if getattr(fid1, glob_name) != getattr(fid2, glob_name):
    227             error_msg += ("\nFiles differ in global '%s':\n"
    228                           "%s: '%s',\n"
    229                           "%s: '%s'\n"
    230                           % (glob_name, filename1, str(g1), filename2, str(g2)))
     237            print("\nFiles differ in global '%s':\n"
     238                  "%s: '%s',\n"
     239                  "%s: '%s'"
     240                  % (glob_name, filename1, str(g1), filename2, str(g2)))
     241            glob_vars_bad[glob_name] = glob_vars_bad.get(glob_name, 0) + 1
    231242            error = True
    232243
    233244    # check data variables, be clever with time series data
     245    max_rel_difference = -1
     246    diff_count = 0
    234247    for var_name in variables:
    235248        var_dims = expected_variables[var_name]
     
    239252                var1 = num.array(fid1.variables[var_name][t,:])
    240253                var2 = num.array(fid2.variables[var_name][t,:])
    241                 if var1 != var2:
     254                if not num.allclose(var1, var2,
     255                                    rtol=rel_tolerance, atol=abs_tolerance):
     256                    error = True
    242257                    for i in xrange(len(var1)):
    243                         if var1[i] != var2[i]:
    244                             error_msg += ('\nFiles differ in variable '
    245                                               '%s[%d,%d]:\n'
    246                                           '%s: %f\n'
    247                                           '%s: %f\n'
    248                                           'difference=%f\n'
    249                                           % (var_name, t, i,
    250                                              filename1, var1[i],
    251                                              filename2, var2[i],
    252                                              var1[i]-var2[i]))
    253                             break
    254                     error = True
     258                        if not num.allclose(var1[i], var2[i],
     259                                            rtol=rel_tolerance,
     260                                            atol=abs_tolerance):
     261                            abs_difference = num.abs(var1[i]-var2[i])
     262                            max_a_b = num.max(num.abs(var1[i]),
     263                                              num.abs(var2[i]))
     264                            rel_difference = num.abs(abs_difference/max_a_b)
     265
     266                            if not quiet:
     267                                print('\nFiles differ in variable '
     268                                      '%s[%d,%d]:\n'
     269                                      '%s: %f\n'
     270                                      '%s: %f\n'
     271                                      'abs. difference=%e, rel. difference=%e\n'
     272                                      % (var_name, t, i,
     273                                         filename1, var1[i],
     274                                         filename2, var2[i],
     275                                         abs_difference,
     276                                         rel_difference))
     277
     278                            if rel_difference > max_rel_difference:
     279                                max_rel_difference = rel_difference
     280                                max_rel_difference_abs = abs_difference
     281                                max_rel_difference_a = var1[i]
     282                                max_rel_difference_b = var2[i]
     283
     284                            data_vars_bad[var_name] = data_vars_bad.get(var_name, 0) + 1
     285                            diff_count += 1
    255286        else:
    256287            # simple data, check whole thing at once
    257288            var1 = num.array(fid1.variables[var_name][:])
    258289            var2 = num.array(fid2.variables[var_name][:])
    259             if not num.allclose(var1, var2):
     290            if not num.allclose(var1, var2,
     291                                rtol=rel_tolerance, atol=abs_tolerance):
    260292                for j in xrange(len(var1)):
    261                     if abs(var1[j] - var2[j]) > epsilon:
    262                         error_msg += ('\nFiles differ in variable '
    263                                       '%s[%d]:\n'
    264                                       '%s: %f\n'
    265                                       '%s: %f\n'
    266                                       'difference=%f\n'
    267                                        % (var_name, j, 
    268                                           filename1, var1[j],
    269                                           filename2, var2[j],
    270                                           var1[j]-var2[j]))
    271                         break
     293                    if not num.allclose(var1[j], var2[j],
     294                                          rtol=rel_tolerance, atol=abs_tolerance):
     295                        abs_difference = num.abs(var1[j]-var2[j])
     296                        max_a_b = num.max(num.abs(var1[j]),
     297                                          num.abs(var2[j]))
     298                        rel_difference = num.abs(abs_difference/max_a_b)
     299
     300                        if not quiet:
     301                            print('\nFiles differ in variable '
     302                                  '%s[%d]:\n'
     303                                  '%s: %f\n'
     304                                  '%s: %f\n'
     305                                  'abs. difference=%e, rel. difference=%e\n'
     306                                   % (var_name, j, 
     307                                      filename1, var1[j],
     308                                      filename2, var2[j],
     309                                      abs_difference,
     310                                      rel_difference))
     311
     312                        if rel_difference > max_rel_difference:
     313                            max_rel_difference = rel_difference
     314                            max_rel_difference_abs = abs_difference
     315                            max_rel_difference_a = var1[j]
     316                            max_rel_difference_b = var2[j]
     317
     318                        data_vars_bad[var_name] = data_vars_bad.get(var_name, 0) + 1
     319                        diff_count += 1
    272320                error = True
    273321
     
    280328
    281329    if error:
     330        error_msg += ('\nNumber of data differences=%d\n'
     331                      'Maximum relative data difference=%e\n'
     332                      'associated absolute difference=%e\n'
     333                      "associated 'a' value=%e\n"
     334                      "associated 'b' value=%e\n"
     335                      % (diff_count, max_rel_difference, max_rel_difference_abs,
     336                         max_rel_difference_a, max_rel_difference_b))
     337        error_msg += ('\nglob_vars bad=%s\n' % str(glob_vars_bad))
     338        error_msg += ('\ndata_vars bad=%s\n' % str(data_vars_bad))
    282339        raise RuntimeError, error_msg
    283340
     
    293350    a('where <options> is zero or more of:\n')
    294351    a('                   -h        print this help\n')
     352    a('                   -q        be quiet, print only summary of differences\n')
    295353    a("                   -a <val>  set absolute threshold of 'equivalent'\n")
    296354    a("                   -r <val>  set relative threshold of 'equivalent'\n")
     
    319377# @return The status code the program will exit with.
    320378def main(argv=None):
     379    global quiet
     380
    321381    if argv is None:
    322382        argv = sys.argv
     
    324384    try:
    325385        try:
    326             opts, args = getopt.getopt(argv[1:], 'hg:t:v:',
     386            opts, args = getopt.getopt(argv[1:], 'hqa:g:r:t:v:',
    327387                                       ['help', 'globals',
    328388                                        'variables', 'timesteps'])
     
    338398    timesteps = None
    339399    variables = None
     400    quiet = False
     401    rel_tolerance = default_rel_tolerance
     402    abs_tolerance = default_abs_tolerance
    340403    for opt, arg in opts:
    341404        if opt in ('-h', '--help'):
    342405            print usage()
    343406            sys.exit(0)
     407        elif opt in ('-q', '--quiet'):
     408            quiet = True
     409        elif opt in ('-a', '--absolute'):
     410            abs_tolerance = float(arg)
     411        elif opt in ('-r', '--relative'):
     412            rel_tolerance = float(arg)
    344413        elif opt in ('-g', '--globals'):
    345414            globals = arg.split(',')
     
    357426    try:
    358427        files_are_the_same(args, globals=globals,
    359                            timesteps=timesteps, variables=variables)
     428                           timesteps=timesteps, variables=variables,
     429                           rel_tolerance=rel_tolerance,
     430                           abs_tolerance=abs_tolerance)
    360431    except RuntimeError, msg:
    361432         print msg
  • anuga_validation/automated_validation_tests/patong_beach_validation/run_model.py

    r6927 r7276  
    2828# Related major packages
    2929from Scientific.IO.NetCDF import NetCDFFile
    30 import Numeric as num
     30#import numpy as num
    3131
    3232from anuga.interface import create_domain_from_regions
     
    4444from anuga.utilities.polygon import read_polygon, Polygon_function
    4545from anuga.caching import cache
     46
     47import anuga.utilities.log as log
    4648
    4749# Application specific imports
     
    7072#-------------------------------------------------------------------------------
    7173
    72 print 'Create computational domain'
     74log.critical('Create computational domain')
    7375
    7476# Create the STS file
     
    7678# We need to use caching instead!
    7779
    78 print 'project.mux_data_folder=%s' % project.mux_data_folder
     80log.critical( 'project.mux_data_folder=%s' % project.mux_data_folder)
    7981if not os.path.exists(project.event_sts + '.sts'):
    8082    bub.build_urs_boundary(project.mux_input_filename, project.event_sts)
     
    111113                                    use_cache=True,
    112114                                    verbose=True)
    113 print domain.statistics()
     115log.critical(domain.statistics())
    114116
    115117# FIXME(Ole): How can we make this more automatic?
     
    125127#-------------------------------------------------------------------------------
    126128
    127 print 'Setup initial conditions'
     129log.critical('Setup initial conditions')
    128130
    129131# Set the initial stage in the offcoast region only
     
    144146if project.use_buildings:
    145147    # Add buildings from file
    146     print 'Reading building polygons'   
     148    log.critical('Reading building polygons')
    147149    building_polygons, building_heights = csv2building_polygons(project.building_polygon)
    148150    #clipping_polygons=project.building_area_polygons)
    149151
    150     print 'Creating %d building polygons' % len(building_polygons)
     152    log.critical('Creating %d building polygons' % len(building_polygons))
    151153    def create_polygon_function(building_polygons, geo_reference=None):
    152154        L = []
    153155        for i, key in enumerate(building_polygons):
    154             if i%100==0: print i
     156            if i%100==0: log.critical(i)
    155157            poly = building_polygons[key]
    156158            elev = building_heights[key]
     
    161163        return buildings
    162164
    163     print 'Creating %d building polygons' % len(building_polygons)
     165    log.critical('Creating %d building polygons' % len(building_polygons))
    164166    buildings = cache(create_polygon_function,
    165167                      building_polygons,
     
    167169                      verbose=True)
    168170
    169     print 'Adding buildings'
     171    log.critical('Adding buildings')
    170172    domain.add_quantity('elevation',
    171173                        buildings,
     
    178180#-------------------------------------------------------------------------------
    179181
    180 print 'Set boundary - available tags:', domain.get_boundary_tags()
     182log.critical('Set boundary - available tags:' % domain.get_boundary_tags())
    181183
    182184Br = Reflective_boundary(domain)
     
    204206for t in domain.evolve(yieldstep=2000,
    205207                       finaltime=6000):
    206     print domain.timestepping_statistics()
    207     print domain.boundary_statistics(tags='ocean')
     208    log.critical(domain.timestepping_statistics())
     209    log.critical(domain.boundary_statistics(tags='ocean'))
    208210
    209211# Start detailed model
     
    211213                       finaltime=project.finaltime,
    212214                       skip_initial_step=True):
    213     print domain.timestepping_statistics()
    214     print domain.boundary_statistics(tags='ocean')
     215    log.critical(domain.timestepping_statistics())
     216    log.critical(domain.boundary_statistics(tags='ocean'))
    215217   
    216 print 'Simulation took %.2f seconds' %(time.time()-t0)
     218log.critical('Simulation took %.2f seconds' %(time.time()-t0))
    217219     
  • anuga_validation/automated_validation_tests/patong_beach_validation/validate.py

    r7267 r7276  
    1313import time
    1414import shutil
    15 import platform
    1615
    1716from anuga.utilities.system_tools import get_web_file, untar_file, file_length
     
    3332           'http://dfn.dl.sourceforge.net/sourceforge/anuga/'             # de
    3433          ]
     34
    3535### for testing
    3636##MIRRORS = ['http://10.7.64.243/patong_validation_data/']       # local linux box
     
    4646# these names must be of the form <scene>.sww.<type>.tgz
    4747# as code below depends upon it.
    48 Optional_Data_Objects = ('patong.sww.TRIAL.tgz',
     48Optional_Data_Objects = (
     49                         'patong.sww.TRIAL.tgz',
    4950                         'patong.sww.BASIC.tgz',
    5051                         'patong.sww.FINAL.tgz'
     
    6667OUTPUT_SWW = 'patong.sww'
    6768
    68 # default name of python to run
    69 PythonName = 'python'
    70 
    7169
    7270def setup():
    73     '''Prepare for the validation run.
    74 
    75     Check we have required data set in project.py.
    76     '''
     71    '''Prepare for the validation run.'''
    7772   
    7873    pass
     
    302297    # run the simulation, produce SWW file
    303298    log.info('Running the simulation ...')
    304     cmd = '%s run_model.py > %s' % (PythonName, RUNMODEL_STDOUT)
     299    cmd = 'python run_model.py > %s' % RUNMODEL_STDOUT
    305300    log.debug("run_simulation: doing '%s'" % cmd)
    306301    res = os.system(cmd)
     
    349344    # compare SWW files here and there
    350345    new_output_sww = os.path.join(output_directory, expected_sww)
    351     cmd = '%s cmpsww.py %s %s > cmpsww.stdout' % (PythonName, local_sww, new_output_sww)
     346    cmd = 'python cmpsww.py %s %s > cmpsww.stdout' % (local_sww, new_output_sww)
    352347    log.debug("check_that_output_is_as_expected: doing '%s'" % cmd)
    353348    res = os.system(cmd)
     
    381376################################################################################
    382377
    383 # determine what python we are, initialize PythonName
    384 py_tuple = platform.python_version_tuple()
    385 PythonName = 'python%s.%s' % (py_tuple[0], py_tuple[1])
    386 
    387378# set logging levels
    388379log.console_logging_level = log.INFO
    389380log.log_logging_level = log.DEBUG
    390 log_filename = log.log_filename
    391381
    392382setup()
    393383
    394384# prepare user for what is about to happen
    395 
    396 msg = '''
     385log.critical('''
    397386Please note that this validation test is accurate only on 64bit Linux or
    398387Windows.  Running the validation on a 32bit operating system will result in
     
    412401if you wish.  If not supplied in environment variables you will be prompted for
    413402the information.
    414 '''
    415 
    416 log.critical(msg)
     403''')
     404
    417405
    418406# make sure local data is up to date
     
    443431for odo in Optional_Data_Objects:
    444432    start_time = time.time()
     433
    445434    (_, vtype, _) = odo.rsplit('.', 2)
    446435    vtype = vtype.lower()
     
    452441        (expected_sww, _) = valid_sww.rsplit('.', 1)
    453442        check_that_output_is_as_expected(expected_sww, valid_sww)
    454     shutil.move(log_filename, '%s.%s' % (log_filename, vtype))
     443
    455444    stop_time = time.time()
    456     log.critical("'%s' validation took %.1fs" % (vtype, stop_time - start_time))
     445    log.critical("'%s' validation took %.1fs\n\n\n" % (vtype, stop_time - start_time))
    457446
    458447# clean up
     448log.critical('Tearing down ...')
    459449teardown()
Note: See TracChangeset for help on using the changeset viewer.