source: anuga_validation/automated_validation_tests/patong_validation/validate_patong.py @ 6732

Last change on this file since 6732 was 6665, checked in by rwilson, 16 years ago

Final form, for now.

File size: 7.8 KB
RevLine 
[6528]1'''
[6660]2Automatic verification that the ANUGA code runs the Patong simulation
3and produces the expected output.
[6639]4
5Required files are downloaded from the ANUGA servers if they are
6out of date or missing.
[6528]7'''
8
9import os
10import glob
11import unittest
[6649]12import time
[6660]13import shutil
14
[6528]15import project
[6649]16from anuga.utilities.system_tools import get_web_file, untar_file
[6639]17import anuga.utilities.log as log
[6528]18
19
[6660]20# base URL for the remote ANUGA data
21PATONG_DATA_URL = 'http://10.7.64.243/patong_validation_data/'
22
[6639]23# path to the local data directory
24Local_Data_Directory = os.path.join('.', 'local_data')
25
26# path to the remote data directory
27Remote_Data_Directory = os.path.join('.', 'remote_data')
28
29# sequence of required local data objects
[6660]30# first is always the SWW file to be compared
31Local_Data_Objects = ('patong.sww', 'data')
[6639]32
[6660]33# name of stdout catch file for runmodel.py
34RUNMODEL_STDOUT = 'runmodel.stdout'
[6639]35
[6660]36# text at start of 'output dir' line in RUNMODEL_STDOUT file
37OUTDIR_PREFIX = 'Make directory '
38
39# Name of SWW file produced by simulation
40OUTPUT_SWW = 'patong.sww'
41
42
43def setup():
44    '''Prepare for the validation run.
45
46    Check we have required data set in project.py.
47    '''
48   
49    # Check that environment variables are defined.
50    if os.getenv(project.ENV_INUNDATIONHOME) is None:
51        msg = ("Environment variable '%s' is not set in project.py"
52               % project.ENV_INUNDATIONHOME)
53        raise Exception, msg
54
55    if os.getenv(project.ENV_MUXHOME) is None:
56        msg = ("Environment variable '%s' is not set in project.py"
57               % project.ENV_MUXHOME)
58        raise Exception, msg
59
60
[6639]61def update_local_data():
[6660]62    '''Update local data objects from the server.
63
64    These are expected to be *.tgz files/directories.
65    '''
66   
67    # local function to update one data object
[6639]68    def update_object(obj, auth):
[6660]69        '''Update object 'obj' using authentication tuple 'auth'.'''
70       
[6649]71        # Get a unique date+time string to defeat caching.  The idea is to add
72        # this to the end of any URL so proxy sees a different request.
73        cache_defeat = '?' + time.strftime('%Y%m%d%H%M%S')
74
75        # create local and remote paths, URLs, etc.
[6639]76        remote_path = os.path.join(Remote_Data_Directory, obj)
77        remote_digest = remote_path + '.tgz.digest'
78       
79        local_path = os.path.join(Local_Data_Directory, obj)
[6649]80        local_file = local_path + '.tgz'
81        local_digest = local_file + '.digest'
[6639]82       
[6660]83        object_url = PATONG_DATA_URL + obj + '.tgz'
[6649]84        digest_url = object_url + '.digest'
85
[6660]86        # see if missing either digest or object .tgz
[6649]87        if not os.path.exists(local_digest) or not os.path.exists(local_file):
88            # no digest or no object, download both digest and object
89            log.debug('Fetching remote file %s' % digest_url)
90            auth = get_web_file(digest_url+cache_defeat,
91                                local_digest, auth=auth)
[6639]92            log.debug('Fetching remote file %s' % object_url)
[6649]93            auth = get_web_file(object_url+cache_defeat, local_file, auth=auth)
[6639]94        else:
95            # download object digest to remote data directory
96            log.debug('Fetching remote file %s' % object_url)
[6660]97            auth = get_web_file(digest_url+cache_defeat,
[6649]98                                remote_digest, auth=auth)
[6639]99           
[6649]100            # compare remote with local digest
[6639]101            fd = open(local_digest, 'r')
[6660]102            local_data_digest = fd.read()
[6639]103            fd.close()
104
105            fd = open(remote_digest, 'r')
[6660]106            remote_data_digest = fd.read()
[6639]107            fd.close()
108
[6660]109            # if digests differ, refresh object
110            if local_data_digest != remote_data_digest:
[6649]111                log.debug('Local file %s is out of date' % obj)
[6639]112                fd = open(local_digest, 'w')
[6660]113                fd.write(remote_data_digest)
[6639]114                fd.close()
115
116                log.debug('Fetching remote file %s' % object_url)
[6649]117                auth = get_web_file(object_url+cache_defeat,
118                                    local_file, auth=auth)
[6639]119        return auth
120
[6665]121    log.debug('Refreshing local data ...')
[6639]122   
123    # create local data directory if required
124    if not os.path.exists(Local_Data_Directory):
125        os.mkdir(Local_Data_Directory)
126
[6660]127    # clean out remote data copy directory
128    shutil.rmtree(Remote_Data_Directory, ignore_errors=True)
129    os.mkdir(Remote_Data_Directory)
[6639]130
131    # refresh local files
132    auth = None
133    for data_object in Local_Data_Objects:
134        auth = update_object(data_object, auth)
[6649]135
136    # unpack *.tgz files
137    for data_object in Local_Data_Objects:
138        tar_path = os.path.join(Local_Data_Directory, data_object+'.tgz')
139        log.debug('Untarring %s in dir %s' % (tar_path, Local_Data_Directory))
140        untar_file(tar_path, target_dir=Local_Data_Directory)
[6639]141       
[6665]142    log.debug('Local data has been refreshed.')
[6639]143
[6528]144
[6660]145def run_simulation():
146    '''Run the Patong simulation.'''
147   
148    # modify environment so we use the local data
149    # INUNDATIONHOME points into the 'data' local data directory
150    old_inundationhome = os.getenv(project.ENV_INUNDATIONHOME)
151    os.putenv(project.ENV_INUNDATIONHOME,
152              os.path.join(Local_Data_Directory, ''))
153    old_muxhome = os.getenv(project.ENV_MUXHOME)
154    os.putenv(project.ENV_MUXHOME,
155              os.path.join(Local_Data_Directory, 'data'))
156   
157    # run the simulation, produce SWW file
158    log.debug('Running Patong simulation ...')
159    cmd = 'python run_model.py > %s' % RUNMODEL_STDOUT
160    res = os.system(cmd)
161    assert res == 0
[6528]162
[6660]163    # undo environment changes
164    if old_inundationhome:
165        os.putenv(project.ENV_INUNDATIONHOME, old_inundationhome)
166    if old_muxhome:
167        os.putenv(project.ENV_MUXHOME, old_muxhome)
[6609]168
[6660]169   
170def check_that_output_is_as_expected():
171    '''Check that validation output is as required.'''
[6528]172
[6660]173    # get path to expected SWW file
174    log.debug('Checking that simulation results are as expected ...')
175    local_sww = os.path.join(Local_Data_Directory, Local_Data_Objects[0])
[6528]176
[6660]177    # get output directory from stdout capture file
178    try:
179        fd = open(RUNMODEL_STDOUT, 'r')
180    except IOError, e:
181        log.critical("Can't open catch file '%s': %s"
182                     % (RUNMODEL_STDOUT, str(e)))
183        return 1
184    lines = fd.readlines()
185    fd.close
[6649]186
[6660]187    output_directory = None
188    for line in lines:
189        if line.startswith(OUTDIR_PREFIX):
190            output_directory = line.replace(OUTDIR_PREFIX, '', 1)
191            output_directory = output_directory.strip('\n')
192            break
193    if output_directory is None:
194        log.critical("Couldn't find line starting with '%s' in file '%s'"
195                     % (OUTDIR_PREFIX, RUNMODEL_STDOUT))
196        return 1
[6649]197
[6660]198    # compare SWW files here and there
199    new_output_sww = os.path.join(output_directory, OUTPUT_SWW)
200    cmd = 'python cmpsww.py %s %s > cmpsww.stdout' % (local_sww, new_output_sww)
201    res = os.system(cmd)
202    assert res == 0
203    log.debug('Simulation results are as expected.')
204
205
206def teardown():
207    '''Clean up after validation run.'''
208   
209    # clear all data objects from local data directory
210    for data_object in Local_Data_Objects:
[6665]211        obj_path = os.path.join(Local_Data_Directory, data_object)
212        if os.path.isfile(obj_path):
213            os.remove(obj_path)
214        else:
215            shutil.rmtree(obj_path, ignore_errors=True)
216
217    # remove remote directory and stdout capture file
218    shutil.rmtree(Remote_Data_Directory, ignore_errors=True)
219    os.remove(RUNMODEL_STDOUT)
[6660]220           
221
[6528]222################################################################################
[6660]223# Mainline - run the simulation, check output.
224################################################################################
[6528]225
[6660]226# set logging levels
227log.console_logging_level = log.DEBUG
228setup()
[6649]229
[6660]230# make sure local data is up to date
231update_local_data()
232
233# run the simulation
234run_simulation()
235
236# check output is as expected
237check_that_output_is_as_expected()
238
239# clean up
240teardown()
Note: See TracBrowser for help on using the repository browser.