[6528] | 1 | ''' |
---|
[6660] | 2 | Automatic verification that the ANUGA code runs the Patong simulation |
---|
| 3 | and produces the expected output. |
---|
[6639] | 4 | |
---|
| 5 | Required files are downloaded from the ANUGA servers if they are |
---|
| 6 | out of date or missing. |
---|
[6528] | 7 | ''' |
---|
| 8 | |
---|
| 9 | import os |
---|
| 10 | import glob |
---|
| 11 | import unittest |
---|
[6649] | 12 | import time |
---|
[6660] | 13 | import shutil |
---|
| 14 | |
---|
[6528] | 15 | import project |
---|
[6649] | 16 | from anuga.utilities.system_tools import get_web_file, untar_file |
---|
[6639] | 17 | import anuga.utilities.log as log |
---|
[6528] | 18 | |
---|
| 19 | |
---|
[6660] | 20 | # base URL for the remote ANUGA data |
---|
| 21 | PATONG_DATA_URL = 'http://10.7.64.243/patong_validation_data/' |
---|
| 22 | |
---|
[6639] | 23 | # path to the local data directory |
---|
| 24 | Local_Data_Directory = os.path.join('.', 'local_data') |
---|
| 25 | |
---|
| 26 | # path to the remote data directory |
---|
| 27 | Remote_Data_Directory = os.path.join('.', 'remote_data') |
---|
| 28 | |
---|
| 29 | # sequence of required local data objects |
---|
[6660] | 30 | # first is always the SWW file to be compared |
---|
| 31 | Local_Data_Objects = ('patong.sww', 'data') |
---|
[6639] | 32 | |
---|
[6660] | 33 | # name of stdout catch file for runmodel.py |
---|
| 34 | RUNMODEL_STDOUT = 'runmodel.stdout' |
---|
[6639] | 35 | |
---|
[6660] | 36 | # text at start of 'output dir' line in RUNMODEL_STDOUT file |
---|
| 37 | OUTDIR_PREFIX = 'Make directory ' |
---|
| 38 | |
---|
| 39 | # Name of SWW file produced by simulation |
---|
| 40 | OUTPUT_SWW = 'patong.sww' |
---|
| 41 | |
---|
| 42 | |
---|
| 43 | def setup(): |
---|
| 44 | '''Prepare for the validation run. |
---|
| 45 | |
---|
| 46 | Check we have required data set in project.py. |
---|
| 47 | ''' |
---|
| 48 | |
---|
| 49 | # Check that environment variables are defined. |
---|
| 50 | if os.getenv(project.ENV_INUNDATIONHOME) is None: |
---|
| 51 | msg = ("Environment variable '%s' is not set in project.py" |
---|
| 52 | % project.ENV_INUNDATIONHOME) |
---|
| 53 | raise Exception, msg |
---|
| 54 | |
---|
| 55 | if os.getenv(project.ENV_MUXHOME) is None: |
---|
| 56 | msg = ("Environment variable '%s' is not set in project.py" |
---|
| 57 | % project.ENV_MUXHOME) |
---|
| 58 | raise Exception, msg |
---|
| 59 | |
---|
| 60 | |
---|
[6639] | 61 | def update_local_data(): |
---|
[6660] | 62 | '''Update local data objects from the server. |
---|
| 63 | |
---|
| 64 | These are expected to be *.tgz files/directories. |
---|
| 65 | ''' |
---|
| 66 | |
---|
| 67 | # local function to update one data object |
---|
[6639] | 68 | def update_object(obj, auth): |
---|
[6660] | 69 | '''Update object 'obj' using authentication tuple 'auth'.''' |
---|
| 70 | |
---|
[6649] | 71 | # Get a unique date+time string to defeat caching. The idea is to add |
---|
| 72 | # this to the end of any URL so proxy sees a different request. |
---|
| 73 | cache_defeat = '?' + time.strftime('%Y%m%d%H%M%S') |
---|
| 74 | |
---|
| 75 | # create local and remote paths, URLs, etc. |
---|
[6639] | 76 | remote_path = os.path.join(Remote_Data_Directory, obj) |
---|
| 77 | remote_digest = remote_path + '.tgz.digest' |
---|
| 78 | |
---|
| 79 | local_path = os.path.join(Local_Data_Directory, obj) |
---|
[6649] | 80 | local_file = local_path + '.tgz' |
---|
| 81 | local_digest = local_file + '.digest' |
---|
[6639] | 82 | |
---|
[6660] | 83 | object_url = PATONG_DATA_URL + obj + '.tgz' |
---|
[6649] | 84 | digest_url = object_url + '.digest' |
---|
| 85 | |
---|
[6660] | 86 | # see if missing either digest or object .tgz |
---|
[6649] | 87 | if not os.path.exists(local_digest) or not os.path.exists(local_file): |
---|
| 88 | # no digest or no object, download both digest and object |
---|
| 89 | log.debug('Fetching remote file %s' % digest_url) |
---|
| 90 | auth = get_web_file(digest_url+cache_defeat, |
---|
| 91 | local_digest, auth=auth) |
---|
[6639] | 92 | log.debug('Fetching remote file %s' % object_url) |
---|
[6649] | 93 | auth = get_web_file(object_url+cache_defeat, local_file, auth=auth) |
---|
[6639] | 94 | else: |
---|
| 95 | # download object digest to remote data directory |
---|
| 96 | log.debug('Fetching remote file %s' % object_url) |
---|
[6660] | 97 | auth = get_web_file(digest_url+cache_defeat, |
---|
[6649] | 98 | remote_digest, auth=auth) |
---|
[6639] | 99 | |
---|
[6649] | 100 | # compare remote with local digest |
---|
[6639] | 101 | fd = open(local_digest, 'r') |
---|
[6660] | 102 | local_data_digest = fd.read() |
---|
[6639] | 103 | fd.close() |
---|
| 104 | |
---|
| 105 | fd = open(remote_digest, 'r') |
---|
[6660] | 106 | remote_data_digest = fd.read() |
---|
[6639] | 107 | fd.close() |
---|
| 108 | |
---|
[6660] | 109 | # if digests differ, refresh object |
---|
| 110 | if local_data_digest != remote_data_digest: |
---|
[6649] | 111 | log.debug('Local file %s is out of date' % obj) |
---|
[6639] | 112 | fd = open(local_digest, 'w') |
---|
[6660] | 113 | fd.write(remote_data_digest) |
---|
[6639] | 114 | fd.close() |
---|
| 115 | |
---|
| 116 | log.debug('Fetching remote file %s' % object_url) |
---|
[6649] | 117 | auth = get_web_file(object_url+cache_defeat, |
---|
| 118 | local_file, auth=auth) |
---|
[6639] | 119 | return auth |
---|
| 120 | |
---|
[6665] | 121 | log.debug('Refreshing local data ...') |
---|
[6639] | 122 | |
---|
| 123 | # create local data directory if required |
---|
| 124 | if not os.path.exists(Local_Data_Directory): |
---|
| 125 | os.mkdir(Local_Data_Directory) |
---|
| 126 | |
---|
[6660] | 127 | # clean out remote data copy directory |
---|
| 128 | shutil.rmtree(Remote_Data_Directory, ignore_errors=True) |
---|
| 129 | os.mkdir(Remote_Data_Directory) |
---|
[6639] | 130 | |
---|
| 131 | # refresh local files |
---|
| 132 | auth = None |
---|
| 133 | for data_object in Local_Data_Objects: |
---|
| 134 | auth = update_object(data_object, auth) |
---|
[6649] | 135 | |
---|
| 136 | # unpack *.tgz files |
---|
| 137 | for data_object in Local_Data_Objects: |
---|
| 138 | tar_path = os.path.join(Local_Data_Directory, data_object+'.tgz') |
---|
| 139 | log.debug('Untarring %s in dir %s' % (tar_path, Local_Data_Directory)) |
---|
| 140 | untar_file(tar_path, target_dir=Local_Data_Directory) |
---|
[6639] | 141 | |
---|
[6665] | 142 | log.debug('Local data has been refreshed.') |
---|
[6639] | 143 | |
---|
[6528] | 144 | |
---|
[6660] | 145 | def run_simulation(): |
---|
| 146 | '''Run the Patong simulation.''' |
---|
| 147 | |
---|
| 148 | # modify environment so we use the local data |
---|
| 149 | # INUNDATIONHOME points into the 'data' local data directory |
---|
| 150 | old_inundationhome = os.getenv(project.ENV_INUNDATIONHOME) |
---|
| 151 | os.putenv(project.ENV_INUNDATIONHOME, |
---|
| 152 | os.path.join(Local_Data_Directory, '')) |
---|
| 153 | old_muxhome = os.getenv(project.ENV_MUXHOME) |
---|
| 154 | os.putenv(project.ENV_MUXHOME, |
---|
| 155 | os.path.join(Local_Data_Directory, 'data')) |
---|
| 156 | |
---|
| 157 | # run the simulation, produce SWW file |
---|
| 158 | log.debug('Running Patong simulation ...') |
---|
| 159 | cmd = 'python run_model.py > %s' % RUNMODEL_STDOUT |
---|
| 160 | res = os.system(cmd) |
---|
| 161 | assert res == 0 |
---|
[6528] | 162 | |
---|
[6660] | 163 | # undo environment changes |
---|
| 164 | if old_inundationhome: |
---|
| 165 | os.putenv(project.ENV_INUNDATIONHOME, old_inundationhome) |
---|
| 166 | if old_muxhome: |
---|
| 167 | os.putenv(project.ENV_MUXHOME, old_muxhome) |
---|
[6609] | 168 | |
---|
[6660] | 169 | |
---|
| 170 | def check_that_output_is_as_expected(): |
---|
| 171 | '''Check that validation output is as required.''' |
---|
[6528] | 172 | |
---|
[6660] | 173 | # get path to expected SWW file |
---|
| 174 | log.debug('Checking that simulation results are as expected ...') |
---|
| 175 | local_sww = os.path.join(Local_Data_Directory, Local_Data_Objects[0]) |
---|
[6528] | 176 | |
---|
[6660] | 177 | # get output directory from stdout capture file |
---|
| 178 | try: |
---|
| 179 | fd = open(RUNMODEL_STDOUT, 'r') |
---|
| 180 | except IOError, e: |
---|
| 181 | log.critical("Can't open catch file '%s': %s" |
---|
| 182 | % (RUNMODEL_STDOUT, str(e))) |
---|
| 183 | return 1 |
---|
| 184 | lines = fd.readlines() |
---|
| 185 | fd.close |
---|
[6649] | 186 | |
---|
[6660] | 187 | output_directory = None |
---|
| 188 | for line in lines: |
---|
| 189 | if line.startswith(OUTDIR_PREFIX): |
---|
| 190 | output_directory = line.replace(OUTDIR_PREFIX, '', 1) |
---|
| 191 | output_directory = output_directory.strip('\n') |
---|
| 192 | break |
---|
| 193 | if output_directory is None: |
---|
| 194 | log.critical("Couldn't find line starting with '%s' in file '%s'" |
---|
| 195 | % (OUTDIR_PREFIX, RUNMODEL_STDOUT)) |
---|
| 196 | return 1 |
---|
[6649] | 197 | |
---|
[6660] | 198 | # compare SWW files here and there |
---|
| 199 | new_output_sww = os.path.join(output_directory, OUTPUT_SWW) |
---|
| 200 | cmd = 'python cmpsww.py %s %s > cmpsww.stdout' % (local_sww, new_output_sww) |
---|
| 201 | res = os.system(cmd) |
---|
| 202 | assert res == 0 |
---|
| 203 | log.debug('Simulation results are as expected.') |
---|
| 204 | |
---|
| 205 | |
---|
| 206 | def teardown(): |
---|
| 207 | '''Clean up after validation run.''' |
---|
| 208 | |
---|
| 209 | # clear all data objects from local data directory |
---|
| 210 | for data_object in Local_Data_Objects: |
---|
[6665] | 211 | obj_path = os.path.join(Local_Data_Directory, data_object) |
---|
| 212 | if os.path.isfile(obj_path): |
---|
| 213 | os.remove(obj_path) |
---|
| 214 | else: |
---|
| 215 | shutil.rmtree(obj_path, ignore_errors=True) |
---|
| 216 | |
---|
| 217 | # remove remote directory and stdout capture file |
---|
| 218 | shutil.rmtree(Remote_Data_Directory, ignore_errors=True) |
---|
| 219 | os.remove(RUNMODEL_STDOUT) |
---|
[6660] | 220 | |
---|
| 221 | |
---|
[6528] | 222 | ################################################################################ |
---|
[6660] | 223 | # Mainline - run the simulation, check output. |
---|
| 224 | ################################################################################ |
---|
[6528] | 225 | |
---|
[6660] | 226 | # set logging levels |
---|
| 227 | log.console_logging_level = log.DEBUG |
---|
| 228 | setup() |
---|
[6649] | 229 | |
---|
[6660] | 230 | # make sure local data is up to date |
---|
| 231 | update_local_data() |
---|
| 232 | |
---|
| 233 | # run the simulation |
---|
| 234 | run_simulation() |
---|
| 235 | |
---|
| 236 | # check output is as expected |
---|
| 237 | check_that_output_is_as_expected() |
---|
| 238 | |
---|
| 239 | # clean up |
---|
| 240 | teardown() |
---|