Changeset 6891


Ignore:
Timestamp:
Apr 24, 2009, 9:35:25 AM (16 years ago)
Author:
rwilson
Message:

Better handling of errors, display on screen.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • anuga_validation/automated_validation_tests/patong_beach_validation/validate.py

    r6890 r6891  
    106106        # try each mirror when getting file
    107107        for mirror in mirrors:
    108             log.info('Fetching remote file %s from mirror %s'
    109                      % (remote, mirror))
     108            log.debug('Fetching remote file %s from mirror %s'
     109                      % (remote, mirror))
    110110
    111111            remote_url = mirror + remote + cache_defeat
    112             log.debug('Trying to fetching file %s' % remote_url)
    113             new_auth = get_web_file(remote_url, local, auth=auth)
    114             if new_auth != False and not is_html(local):
     112            (result, auth) = get_web_file(remote_url, local, auth=auth)
     113            if result and is_html(local)==False:
    115114                log.debug('Success fetching file %s' % remote_url)
    116                 return new_auth
    117 
    118         log.info('Could not fetch file %s' % remote_url)
    119         return None           
     115                return (True, auth)
     116
     117        log.info('Could not fetch file %s' % local)
     118        return (False, auth)           
    120119               
    121120
    … …  
    139138        '''Update object 'obj' using authentication tuple 'auth'.
    140139       
    141         Return updated auth object (success) or None (failure).
     140        Return (True, <updated_auth>) if all went well,
     141        else (False, <updated_auth>).
    142142        '''
    143143
    … …  
    156156        if not os.path.exists(local_digest) or not os.path.exists(local_file):
    157157            # no digest or no object, download both digest and object
    158             auth = get_remote_from_mirrors(obj_digest, local_digest, auth, mirrors)
    159             if auth:
    160                 auth = get_remote_from_mirrors(obj, local_file, auth, mirrors)
     158            (res, auth) = get_remote_from_mirrors(obj_digest, local_digest, auth, mirrors)
     159            if res:
     160                (res, auth) = get_remote_from_mirrors(obj, local_file, auth, mirrors)
    161161        else:
    162162            # download object digest to remote data directory
    163             auth = get_remote_from_mirrors(obj_digest, remote_digest, auth, mirrors)
    164             if auth:
     163            (res, auth) = get_remote_from_mirrors(obj_digest, remote_digest, auth, mirrors)
     164            if res:
    165165                if not files_same(local_data_digest, remote_data_digest):
    166166                    # digests differ, refresh object
    167                     log.info('Local file %s is out of date' % obj)
     167                    log.info('Local file %s is out of date, refreshing ...' % obj)
    168168                    shutil.move(remote_digest, local_digest)
    169                     auth = get_remote_from_mirrors(obj, local_file, auth, mirrors)
     169                    (res, auth) = get_remote_from_mirrors(obj, local_file, auth, mirrors)
    170170               
    171         return auth
     171        return (res, auth)
    172172
    173173    # create local data directory if required
    … …  
    183183    # success, refresh local files
    184184    auth = None
    185     for data_object in Mandatory_Data_Objects:
    186         auth = refresh_object(data_object, auth, mirrors)
    187         if auth is None:
    188             log.debug('Failed fetching %s, returning None' % data_object)
    189             return None
    190     for data_object in Optional_Data_Objects:
    191         auth = refresh_object(data_object, auth, mirrors)
    192         if auth is None:
    193             log.debug('Failed fetching %s, returning None' % data_object)
    194             return None
    195 
    196     log.info('Local data has been refreshed.')
    197     return True
     185    result = True
     186    for data_object in data_objects:
     187        log.debug('refresh_local_data: getting %s from mirrors, auth=%s'
     188                  % (data_object, str(auth)))
     189        (res, auth) = refresh_object(data_object, auth, mirrors)
     190        log.debug('refresh_local_data: returned (res,auth)=%s,%s'
     191                  % (str(res), str(auth)))
     192        if res == False:
     193            log.debug('Failed fetching %s, returning False' % data_object)
     194            result = False
     195
     196    log.info('Local data has been refreshed, result=%s.' % str(result))
     197    return result
    198198
    199199
    … …  
    208208    log.critical('Checking if you have the required files to run:')
    209209
    210     have_mandatory_files = True
     210    # get max width of object name string
    211211    max_width = 0
    212212    for obj in Mandatory_Data_Objects:
    … …  
    215215        max_width = max(len(obj), max_width)
    216216
    217     # if we don't have all mandatory object, can't run
     217    # if we don't have *all* mandatory object, can't run
     218    have_mandatory_files = True
    218219    for obj in Mandatory_Data_Objects:
    219220        obj_path = os.path.join(Local_Data_Directory, obj)
    … …  
    224225            have_mandatory_files = False
    225226
    226     # one or more of these must exist
    227     have_optional_files = True
     227    # at least *one* of these must exist
     228    have_optional_files = False
    228229    for obj in Optional_Data_Objects:
    229230        obj_path = os.path.join(Local_Data_Directory, obj)
    230231        if os.path.exists(obj_path):
     232            have_optional_files = True
    231233            log.info('\t%s  found' % obj.ljust(max_width))
    232234        else:
    233235            log.info('\t%s  MISSING!' % obj.ljust(max_width))
    234             have_optional_files = False
    235236
    236237    if not have_mandatory_files or not have_optional_files:
    … …  
    244245
    245246
    246 def run_simulation(vtype, sim_obj):
    247     '''Run the Patong simulation.'''
    248    
    249     # now untar file/directory objects
    250     for obj in Mandatory_Data_Objects:
    251         tar_path = os.path.join(Local_Data_Directory, obj)
    252         log.info('Untarring %s in directory %s ...'
    253                  % (tar_path, Local_Data_Directory))
    254         untar_file(tar_path, target_dir=Local_Data_Directory)
    255 
    256     # untar the vtype object
    257     tar_path = os.path.join(Local_Data_Directory, sim_obj)
    258     log.info('Untarring %s in directory %s ...'
    259              % (tar_path, Local_Data_Directory))
    260     untar_file(tar_path, target_dir=Local_Data_Directory)
    261 
     247def set_environment():
    262248    # modify environment so we use the local data
    263249    new_inundationhome = os.path.join(Local_Data_Directory, '')
    … …  
    265251    new_muxhome = os.path.join(Local_Data_Directory, 'data')
    266252    os.environ['MUXHOME'] = new_muxhome
     253
     254
     255def run_simulation(vtype, sim_obj):
     256    '''Run a simulation.
     257
     258    Returns True if all went well, else False.
     259    '''
     260   
     261    # untar the object
     262    tar_path = os.path.join(Local_Data_Directory, sim_obj)
     263    log.info('Untarring %s in directory %s ...'
     264             % (tar_path, Local_Data_Directory))
     265    untar_file(tar_path, target_dir=Local_Data_Directory)
    267266
    268267    # modify project.py template
    … …  
    279278    fd.close()
    280279   
    281     # We import here, _after_ environment variables are set
     280    # import new project.py
    282281    import project
    283282
    … …  
    285284    log.critical('Running Patong simulation ...')
    286285    cmd = 'python run_model.py > %s' % RUNMODEL_STDOUT
     286    log.debug("run_simulation: doing '%s'" % cmd)
    287287    res = os.system(cmd)
    288     assert res == 0
     288    log.debug("run_simulation: res=%d" % res)
     289
     290    # 'unimport' project.py
     291    del project
     292
     293    # check result
     294    if res != 0:
     295        log.critical('Simulation failed, check log')
     296
     297    return res == 0
    289298
    290299def check_that_output_is_as_expected():
    … …  
    316325        return 1
    317326
     327    log.debug('check_that_output_is_as_expected: output_directory=%s'
     328              % output_directory)
     329   
    318330    # compare SWW files here and there
    319331    new_output_sww = os.path.join(output_directory, OUTPUT_SWW)
    320332    cmd = 'python cmpsww.py %s %s > cmpsww.stdout' % (local_sww, new_output_sww)
     333    log.debug("check_that_output_is_as_expected: doing '%s'" % cmd)
    321334    res = os.system(cmd)
     335    log.debug("check_that_output_is_as_expected: res=%d" % res)
    322336    if res == 0:
    323337        log.info('Simulation results are as expected.')
    … …  
    332346def teardown():
    333347    '''Clean up after validation run.'''
    334    
    335     # clear all data objects from local data directory
    336     for data_object in Local_Data_Objects:
    337         obj_path = os.path.join(Local_Data_Directory, data_object)
    338         if os.path.isfile(obj_path):
    339             os.remove(obj_path)
    340         else:
    341             shutil.rmtree(obj_path, ignore_errors=True)
     348
     349    log.debug('teardown: called')
     350   
     351##    # clear all data objects from local data directory
     352##    for data_object in Local_Data_Objects:
     353##        obj_path = os.path.join(Local_Data_Directory, data_object)
     354##        if os.path.isfile(obj_path):
     355##            os.remove(obj_path)
     356##        else:
     357##            shutil.rmtree(obj_path, ignore_errors=True)
    342358
    343359    # remove remote directory and stdout capture file
    … …  
    355371# set logging levels
    356372log.console_logging_level = log.INFO
     373log.log_logging_level = log.DEBUG
    357374setup()
    358375
    … …  
    377394
    378395# make sure local data is up to date
    379 data_objects = []
    380 for o in Mandatory_Data_Objects:
    381     data_objects.append(o)
    382 for o in Optional_Data_Objects:
    383     data_objects.append(o)
    384    
    385 if not refresh_local_data(data_objects, Local_Data_Directory, MIRRORS):
     396all_objects = Mandatory_Data_Objects + Optional_Data_Objects
     397if not refresh_local_data(all_objects, Local_Data_Directory, MIRRORS):
    386398    if not can_we_run():
    387399        log.critical("Can't refresh via the internet and you don't have the "
    … …  
    395407        sys.exit(10)
    396408
     409# now untar mandatory objects
     410for obj in Mandatory_Data_Objects:
     411    tar_path = os.path.join(Local_Data_Directory, obj)
     412    log.info('Untarring %s in directory %s ...'
     413             % (tar_path, Local_Data_Directory))
     414    untar_file(tar_path, target_dir=Local_Data_Directory)
     415
     416# set required environment variables
     417set_environment()
     418
    397419# now run what simulations we can and check output is as expected
    398420for odo in Optional_Data_Objects:
    399421    (_, vtype, _) = odo.rsplit('.', 2)
    400422    vtype = vtype.lower()
    401     print "Running Patong '%s' validation" % vtype
    402     run_simulation(vtype, odo)
    403     check_that_output_is_as_expected(vtype, odo)
     423    log.critical("Running Patong '%s' validation ..." % vtype)
     424    if run_simulation(vtype, odo):
     425        check_that_output_is_as_expected(vtype, odo)
    404426
    405427# clean up
    406 #teardown()
     428teardown()
Note: See TracChangeset for help on using the changeset viewer.