[6844] | 1 | #!/usr/bin/env python |
---|
| 2 | |
---|
| 3 | ''' |
---|
| 4 | A program to compare two SWW files for "equality". |
---|
| 5 | |
---|
| 6 | This program makes lots of assumptions about the structure of the SWW files, |
---|
| 7 | so if that structure changes, this program must change. |
---|
| 8 | ''' |
---|
| 9 | |
---|
| 10 | import sys |
---|
| 11 | import os |
---|
| 12 | import os.path |
---|
| 13 | import getopt |
---|
| 14 | from Scientific.IO.NetCDF import NetCDFFile |
---|
| 15 | import Numeric as num |
---|
| 16 | from anuga.config import netcdf_mode_r |
---|
| 17 | |
---|
| 18 | |
---|
| 19 | ##### |
---|
| 20 | # Various constants. |
---|
| 21 | ##### |
---|
| 22 | |
---|
[6918] | 23 | # allowable 'slop' when testing two float values |
---|
| 24 | epsilon = 1.0e-9 |
---|
| 25 | |
---|
[6844] | 26 | # Global attributes that should exist and be same in both files |
---|
| 27 | # Don't have to have all of these, and we don't care about others. |
---|
| 28 | expect_global_attributes = ['smoothing', 'vertices_are_stored_uniquely', |
---|
[6918] | 29 | 'order', 'starttime', |
---|
[6844] | 30 | 'xllcorner', 'yllcorner', |
---|
| 31 | 'zone', 'false_easting', 'false_northing', |
---|
| 32 | 'datum', 'projection', 'units'] |
---|
| 33 | |
---|
| 34 | # dimensions expected, with expected values (None means unknown) |
---|
| 35 | expected_dimensions = {'number_of_volumes': None, |
---|
| 36 | 'number_of_vertices': 3, |
---|
| 37 | 'numbers_in_range': 2, |
---|
| 38 | 'number_of_points': None, |
---|
| 39 | 'number_of_timesteps': None} |
---|
| 40 | |
---|
| 41 | # Variables expected, with expected dimensions. |
---|
| 42 | # Don't have to have all of these, and we don't care about others. |
---|
| 43 | expected_variables = {'x': ('number_of_points',), |
---|
| 44 | 'y': ('number_of_points',), |
---|
| 45 | 'elevation': ('number_of_points',), |
---|
| 46 | 'elevation_range': ('numbers_in_range',), |
---|
| 47 | 'z': ('number_of_points',), |
---|
| 48 | 'volumes': ('number_of_volumes', 'number of vertices'), |
---|
| 49 | 'time': ('number_of_timesteps',), |
---|
[6918] | 50 | 'stage': ('number_of_timesteps', 'numbers_of_points',), |
---|
[6844] | 51 | 'stage_range': ('numbers_in_range',), |
---|
| 52 | 'xmomentum': ('number_of_timesteps', 'number_of_points'), |
---|
| 53 | 'xmomentum_range': ('numbers_in_range'), |
---|
| 54 | 'ymomentum': ('number_of_timesteps', 'number_of_points'), |
---|
| 55 | 'ymomentum_range': ('numbers_in_range')} |
---|
| 56 | |
---|
| 57 | ## |
---|
| 58 | # @brief An exception to inform user of usage problems. |
---|
| 59 | class Usage(Exception): |
---|
| 60 | def __init__(self, msg): |
---|
| 61 | self.msg = msg |
---|
| 62 | |
---|
| 63 | |
---|
| 64 | ## |
---|
| 65 | # @brief Compare two SWW files. |
---|
| 66 | # @param files A tuple of two filenames. |
---|
| 67 | # @param globals A list of global attribute names to compare. |
---|
| 68 | # @param timesteps A list of timesteps to compare at. |
---|
| 69 | # @param variables A list of variable names to compare. |
---|
| 70 | # @return Returns if files 'equal', else raises RuntimeError. |
---|
| 71 | def files_are_the_same(files, globals=None, timesteps=None, variables=None): |
---|
| 72 | # split out the filenames and check they exist |
---|
| 73 | (file1, file2) = files |
---|
[6918] | 74 | filename1 = os.path.basename(file1) |
---|
| 75 | filename2 = os.path.basename(file2) |
---|
[7040] | 76 | # width = max(len(filename1), len(filename2)) |
---|
| 77 | # filename1 = filename1.rjust(width) |
---|
| 78 | # filename2 = filename2.rjust(width) |
---|
[6844] | 79 | |
---|
| 80 | error = False |
---|
| 81 | error_msg = '' |
---|
| 82 | |
---|
| 83 | try: |
---|
| 84 | fid1 = NetCDFFile(file1, netcdf_mode_r) |
---|
| 85 | except: |
---|
[6918] | 86 | error_msg += "\nFile '%s' can't be opened?\n" % file1 |
---|
[6844] | 87 | error = True |
---|
| 88 | |
---|
| 89 | try: |
---|
| 90 | fid2 = NetCDFFile(file2, netcdf_mode_r) |
---|
| 91 | except: |
---|
[6918] | 92 | error_msg += "\nFile '%s' can't be opened?\n" % file2 |
---|
[6844] | 93 | error = True |
---|
| 94 | fid1.close() |
---|
| 95 | |
---|
| 96 | if error: |
---|
| 97 | raise RuntimeError, error_msg |
---|
| 98 | |
---|
[6918] | 99 | if globals is None: |
---|
| 100 | globals = expect_global_attributes |
---|
| 101 | |
---|
[6844] | 102 | ##### |
---|
| 103 | # First, check that files have the required structure |
---|
| 104 | ##### |
---|
| 105 | |
---|
| 106 | # dimensions - only check expected dimensions |
---|
| 107 | for key in expected_dimensions: |
---|
| 108 | if key not in fid1.dimensions.keys(): |
---|
[6918] | 109 | error_msg += ("\nFile %s doesn't contain dimension '%s'\n" |
---|
| 110 | % (filename1, key)) |
---|
[6844] | 111 | error = True |
---|
| 112 | if key not in fid2.dimensions.keys(): |
---|
[6918] | 113 | error_msg += ("\nFile %s doesn't contain dimension '%s'\n" |
---|
| 114 | % (filename2, key)) |
---|
[6844] | 115 | error = True |
---|
| 116 | |
---|
| 117 | # now check that dimensions are the same length |
---|
| 118 | # NOTE: DOESN'T CHECK 'UNLIMITED' DIMENSIONS YET! (get None at the moment) |
---|
| 119 | for dim in expected_dimensions: |
---|
| 120 | dim1_shape = fid1.dimensions.get(dim, None) |
---|
| 121 | dim2_shape = fid2.dimensions.get(dim, None) |
---|
[6918] | 122 | if dim1_shape != dim2_shape: |
---|
| 123 | error_msg += ("\nFile %s has '%s' dimension of size %s,\n" |
---|
| 124 | "file %s has that dimension of size %s\n" |
---|
| 125 | % (filename1, dim, str(dim1_shape), |
---|
| 126 | filename2, str(dim2_shape))) |
---|
[6844] | 127 | error = True |
---|
| 128 | |
---|
| 129 | # check that we have the required globals |
---|
| 130 | if globals: |
---|
| 131 | for glob in globals: |
---|
| 132 | if glob not in dir(fid1): |
---|
[6918] | 133 | error_msg += ("\nGlobal attribute '%s' isn't in file %s\n" |
---|
| 134 | % (glob, filename1)) |
---|
[6844] | 135 | error = True |
---|
| 136 | if glob not in dir(fid2): |
---|
[6918] | 137 | error_msg += ("\nGlobal attribute '%s' isn't in file %s\n" |
---|
| 138 | % (glob, filename2)) |
---|
[6844] | 139 | error = True |
---|
| 140 | else: |
---|
| 141 | # get list of global attributes |
---|
| 142 | glob_vars1 = [] |
---|
| 143 | glob_vars2 = [] |
---|
| 144 | for glob in expect_global_attributes: |
---|
| 145 | if glob in dir(fid1): |
---|
| 146 | glob_vars1.append(glob) |
---|
| 147 | if glob in dir(fid2): |
---|
| 148 | glob_vars2.append(glob) |
---|
| 149 | |
---|
| 150 | # now check attribute lists are same |
---|
| 151 | if glob_vars1 != glob_vars2: |
---|
[6918] | 152 | error_msg = ('\nFiles differ in global attributes:\n' |
---|
[6844] | 153 | '%s: %s,\n' |
---|
[7001] | 154 | '%s: %s\n' |
---|
| 155 | % (filename1, str(glob_vars1), |
---|
| 156 | filename2, str(glob_vars2))) |
---|
[6844] | 157 | error = True |
---|
| 158 | |
---|
| 159 | # get variables to test |
---|
| 160 | if variables: |
---|
| 161 | for var in variables: |
---|
| 162 | if var not in fid1.variables.keys(): |
---|
[6918] | 163 | error_msg += ("\nVariable '%s' isn't in file %s\n" |
---|
| 164 | % (var, filename1)) |
---|
[6844] | 165 | error = True |
---|
| 166 | if var not in fid2.variables.keys(): |
---|
[6918] | 167 | error_msg += ("\nVariable '%s' isn't in file %s\n" |
---|
| 168 | % (var, filename2)) |
---|
[6844] | 169 | error = True |
---|
| 170 | else: |
---|
| 171 | # check that variables are as expected in both files |
---|
| 172 | var_names1 = [] |
---|
| 173 | var_names2 = [] |
---|
| 174 | for var_name in expected_variables: |
---|
| 175 | if fid1.variables.has_key(var_name): |
---|
| 176 | var_names1.append(var_name) |
---|
| 177 | if fid2.variables.has_key(var_name): |
---|
| 178 | var_names2.append(var_name) |
---|
| 179 | |
---|
| 180 | if var_names1 != var_names2: |
---|
[6918] | 181 | error_msg += ('\nVariables are not the same between files:\n' |
---|
[7001] | 182 | '%s variables=%s,\n' |
---|
| 183 | '%s variables=%s\n' |
---|
[6918] | 184 | % (filename1, str(var_names1), filename2, str(var_names2))) |
---|
[6844] | 185 | error = True |
---|
| 186 | variables = var_names1 |
---|
| 187 | |
---|
| 188 | # get size of time dimension |
---|
| 189 | num_timesteps1 = fid1.variables['time'].shape |
---|
| 190 | num_timesteps2 = fid2.variables['time'].shape |
---|
| 191 | if num_timesteps1 != num_timesteps2: |
---|
[6918] | 192 | error_msg += ('Files have different number of timesteps:\n' |
---|
[7001] | 193 | '%s=%s,\n' |
---|
| 194 | '%s=%s\n' |
---|
| 195 | % (filename1, str(num_timesteps1), |
---|
| 196 | filename2, str(num_timesteps2))) |
---|
[6844] | 197 | error = True |
---|
| 198 | |
---|
| 199 | num_timesteps = num_timesteps1[0] |
---|
| 200 | |
---|
| 201 | # variable shapes same? |
---|
| 202 | for var_name in variables: |
---|
| 203 | var1 = fid1.variables[var_name] |
---|
| 204 | var2 = fid2.variables[var_name] |
---|
| 205 | var1_shape = var1.shape |
---|
| 206 | var2_shape = var2.shape |
---|
| 207 | if var1_shape != var2_shape: |
---|
| 208 | error_msg += ('Files differ in variable %s shape:\n' |
---|
| 209 | '%s: %s,\n' |
---|
| 210 | '%s: %s\n' |
---|
[6918] | 211 | % (var_name, filename1, str(var1_shape), |
---|
[7001] | 212 | filename2, str(var2_shape))) |
---|
[6844] | 213 | error = True |
---|
| 214 | |
---|
| 215 | if error: |
---|
| 216 | fid1.close() |
---|
| 217 | fid2.close() |
---|
| 218 | raise RuntimeError, error_msg |
---|
| 219 | |
---|
| 220 | ##### |
---|
| 221 | # Now check that actual data values are the same |
---|
| 222 | ##### |
---|
| 223 | |
---|
| 224 | # check values of global attributes |
---|
| 225 | for glob_name in globals: |
---|
[6918] | 226 | if getattr(fid1, glob_name) != getattr(fid2, glob_name): |
---|
| 227 | error_msg += ("\nFiles differ in global '%s':\n" |
---|
| 228 | "%s: '%s',\n" |
---|
[6844] | 229 | "%s: '%s'\n" |
---|
[6918] | 230 | % (glob_name, filename1, str(g1), filename2, str(g2))) |
---|
[6844] | 231 | error = True |
---|
| 232 | |
---|
| 233 | # check data variables, be clever with time series data |
---|
| 234 | for var_name in variables: |
---|
| 235 | var_dims = expected_variables[var_name] |
---|
| 236 | if (len(var_dims) > 1) and (var_dims[0] == 'number_of_timesteps'): |
---|
| 237 | # time series, check by timestep block |
---|
[6918] | 238 | for t in xrange(num_timesteps): |
---|
| 239 | var1 = num.array(fid1.variables[var_name][t,:]) |
---|
| 240 | var2 = num.array(fid2.variables[var_name][t,:]) |
---|
| 241 | if var1 != var2: |
---|
| 242 | for i in xrange(len(var1)): |
---|
| 243 | if var1[i] != var2[i]: |
---|
| 244 | error_msg += ('\nFiles differ in variable ' |
---|
[7001] | 245 | '%s[%d,%d]:\n' |
---|
| 246 | '%s: %f\n' |
---|
| 247 | '%s: %f\n' |
---|
| 248 | 'difference=%f\n' |
---|
[6918] | 249 | % (var_name, t, i, |
---|
| 250 | filename1, var1[i], |
---|
[7001] | 251 | filename2, var2[i], |
---|
| 252 | var1[i]-var2[i])) |
---|
[6918] | 253 | break |
---|
[6844] | 254 | error = True |
---|
| 255 | else: |
---|
| 256 | # simple data, check whole thing at once |
---|
| 257 | var1 = num.array(fid1.variables[var_name][:]) |
---|
| 258 | var2 = num.array(fid2.variables[var_name][:]) |
---|
| 259 | if not num.allclose(var1, var2): |
---|
[6918] | 260 | for j in xrange(len(var1)): |
---|
| 261 | if abs(var1[j] - var2[j]) > epsilon: |
---|
| 262 | error_msg += ('\nFiles differ in variable ' |
---|
| 263 | '%s[%d]:\n' |
---|
[7001] | 264 | '%s: %f\n' |
---|
| 265 | '%s: %f\n' |
---|
| 266 | 'difference=%f\n' |
---|
[6918] | 267 | % (var_name, j, |
---|
| 268 | filename1, var1[j], |
---|
[7001] | 269 | filename2, var2[j], |
---|
| 270 | var1[j]-var2[j])) |
---|
[6918] | 271 | break |
---|
[6844] | 272 | error = True |
---|
| 273 | |
---|
| 274 | ##### |
---|
[6918] | 275 | # close files and signal OK or ERROR |
---|
[6844] | 276 | ##### |
---|
| 277 | |
---|
| 278 | fid1.close() |
---|
| 279 | fid2.close() |
---|
| 280 | |
---|
[6918] | 281 | if error: |
---|
| 282 | raise RuntimeError, error_msg |
---|
| 283 | |
---|
[6844] | 284 | return |
---|
| 285 | |
---|
| 286 | |
---|
| 287 | ## |
---|
| 288 | # @brief Return a usage string. |
---|
| 289 | def usage(): |
---|
| 290 | result = [] |
---|
| 291 | a = result.append |
---|
| 292 | a('Usage: %s <options> <file1> <file2>\n' % ProgName) |
---|
| 293 | a('where <options> is zero or more of:\n') |
---|
| 294 | a(' -h print this help\n') |
---|
| 295 | a(" -a <val> set absolute threshold of 'equivalent'\n") |
---|
| 296 | a(" -r <val> set relative threshold of 'equivalent'\n") |
---|
| 297 | a(' -g <arg> check only global attributes specified\n') |
---|
| 298 | a(' <arg> has the form <globname>[,<globname>[,...]]\n') |
---|
| 299 | a(' -t <arg> check only timesteps specified\n') |
---|
| 300 | a(' <arg> has the form <starttime>[,<stoptime>[,<step>]]\n') |
---|
| 301 | a(' -v <arg> check only the named variables\n') |
---|
| 302 | a(' <arg> has the form <varname>[,<varname>[,...]]\n') |
---|
| 303 | a('and <file1> and <file2> are two SWW files to compare.\n') |
---|
| 304 | a('\n') |
---|
| 305 | a('The program exit status is one of:\n') |
---|
| 306 | a(' 0 the two files are equivalent\n') |
---|
| 307 | a(' else the files are not equivalent.') |
---|
| 308 | return ''.join(result) |
---|
| 309 | |
---|
| 310 | ## |
---|
| 311 | # @brief Print a message to stderr. |
---|
| 312 | def warn(msg): |
---|
| 313 | print >>sys.stderr, msg |
---|
| 314 | |
---|
| 315 | |
---|
| 316 | ## |
---|
| 317 | # @brief |
---|
| 318 | # @param argv |
---|
| 319 | # @return The status code the program will exit with. |
---|
| 320 | def main(argv=None): |
---|
| 321 | if argv is None: |
---|
| 322 | argv = sys.argv |
---|
| 323 | |
---|
| 324 | try: |
---|
| 325 | try: |
---|
| 326 | opts, args = getopt.getopt(argv[1:], 'hg:t:v:', |
---|
| 327 | ['help', 'globals', |
---|
| 328 | 'variables', 'timesteps']) |
---|
| 329 | except getopt.error, msg: |
---|
| 330 | raise Usage(msg) |
---|
| 331 | except Usage, err: |
---|
| 332 | print >>sys.stderr, err.msg |
---|
| 333 | print >>sys.stderr, "for help use --help" |
---|
| 334 | return 2 |
---|
| 335 | |
---|
| 336 | # process options |
---|
| 337 | globals = None |
---|
| 338 | timesteps = None |
---|
| 339 | variables = None |
---|
| 340 | for opt, arg in opts: |
---|
| 341 | if opt in ('-h', '--help'): |
---|
| 342 | print usage() |
---|
| 343 | sys.exit(0) |
---|
| 344 | elif opt in ('-g', '--globals'): |
---|
| 345 | globals = arg.split(',') |
---|
| 346 | elif opt in ('-t', '--timesteps'): |
---|
| 347 | timesteps = arg.split(',') |
---|
| 348 | elif opt in ('-v', '--variables'): |
---|
| 349 | variables = arg.split(',') |
---|
| 350 | |
---|
| 351 | # process arguments |
---|
| 352 | if len(args) != 2: |
---|
| 353 | msg = usage() |
---|
| 354 | print 'msg=%s' % msg |
---|
| 355 | raise Usage(msg) |
---|
| 356 | |
---|
| 357 | try: |
---|
| 358 | files_are_the_same(args, globals=globals, |
---|
| 359 | timesteps=timesteps, variables=variables) |
---|
| 360 | except RuntimeError, msg: |
---|
| 361 | print msg |
---|
| 362 | return 10 |
---|
| 363 | |
---|
| 364 | |
---|
| 365 | if __name__ == "__main__": |
---|
| 366 | global ProgName |
---|
| 367 | |
---|
| 368 | ProgName = os.path.basename(sys.argv[0]) |
---|
| 369 | |
---|
| 370 | sys.exit(main()) |
---|
| 371 | |
---|