1 | #!/usr/bin/env python |
---|
2 | |
---|
3 | ''' |
---|
4 | A program to compare two SWW files for "equality". |
---|
5 | |
---|
6 | This program makes lots of assumptions about the structure of the SWW files, |
---|
7 | so if that structure changes, this program must change. |
---|
8 | ''' |
---|
9 | |
---|
10 | import sys |
---|
11 | import os |
---|
12 | import os.path |
---|
13 | import getopt |
---|
14 | from Scientific.IO.NetCDF import NetCDFFile |
---|
15 | import numpy as num |
---|
16 | from anuga.config import netcdf_mode_r |
---|
17 | |
---|
18 | |
---|
19 | ##### |
---|
20 | # Various constants. |
---|
21 | ##### |
---|
22 | |
---|
23 | # allowable 'slop' when testing two float values |
---|
24 | epsilon = 1.0e-9 |
---|
25 | |
---|
26 | # Global attributes that should exist and be same in both files |
---|
27 | # Don't have to have all of these, and we don't care about others. |
---|
28 | expect_global_attributes = ['smoothing', 'vertices_are_stored_uniquely', |
---|
29 | 'order', 'starttime', |
---|
30 | 'xllcorner', 'yllcorner', |
---|
31 | 'zone', 'false_easting', 'false_northing', |
---|
32 | 'datum', 'projection', 'units'] |
---|
33 | |
---|
34 | # dimensions expected, with expected values (None means unknown) |
---|
35 | expected_dimensions = {'number_of_volumes': None, |
---|
36 | 'number_of_vertices': 3, |
---|
37 | 'numbers_in_range': 2, |
---|
38 | 'number_of_points': None, |
---|
39 | 'number_of_timesteps': None} |
---|
40 | |
---|
41 | # Variables expected, with expected dimensions. |
---|
42 | # Don't have to have all of these, and we don't care about others. |
---|
43 | expected_variables = {'x': ('number_of_points',), |
---|
44 | 'y': ('number_of_points',), |
---|
45 | 'elevation': ('number_of_points',), |
---|
46 | 'elevation_range': ('numbers_in_range',), |
---|
47 | 'z': ('number_of_points',), |
---|
48 | 'volumes': ('number_of_volumes', 'number of vertices'), |
---|
49 | 'time': ('number_of_timesteps',), |
---|
50 | 'stage': ('number_of_timesteps', 'numbers_of_points',), |
---|
51 | 'stage_range': ('numbers_in_range',), |
---|
52 | 'xmomentum': ('number_of_timesteps', 'number_of_points'), |
---|
53 | 'xmomentum_range': ('numbers_in_range'), |
---|
54 | 'ymomentum': ('number_of_timesteps', 'number_of_points'), |
---|
55 | 'ymomentum_range': ('numbers_in_range')} |
---|
56 | |
---|
57 | ## |
---|
58 | # @brief An exception to inform user of usage problems. |
---|
59 | class Usage(Exception): |
---|
60 | def __init__(self, msg): |
---|
61 | self.msg = msg |
---|
62 | |
---|
63 | |
---|
64 | ## |
---|
65 | # @brief Compare two SWW files. |
---|
66 | # @param files A tuple of two filenames. |
---|
67 | # @param globals A list of global attribute names to compare. |
---|
68 | # @param timesteps A list of timesteps to compare at. |
---|
69 | # @param variables A list of variable names to compare. |
---|
70 | # @return Returns if files 'equal', else raises RuntimeError. |
---|
71 | def files_are_the_same(files, globals=None, timesteps=None, variables=None): |
---|
72 | # split out the filenames and check they exist |
---|
73 | (file1, file2) = files |
---|
74 | filename1 = os.path.basename(file1) |
---|
75 | filename2 = os.path.basename(file2) |
---|
76 | # width = max(len(filename1), len(filename2)) |
---|
77 | # filename1 = filename1.rjust(width) |
---|
78 | # filename2 = filename2.rjust(width) |
---|
79 | |
---|
80 | error = False |
---|
81 | error_msg = '' |
---|
82 | |
---|
83 | try: |
---|
84 | fid1 = NetCDFFile(file1, netcdf_mode_r) |
---|
85 | except: |
---|
86 | error_msg += "\nFile '%s' can't be opened?\n" % file1 |
---|
87 | error = True |
---|
88 | |
---|
89 | try: |
---|
90 | fid2 = NetCDFFile(file2, netcdf_mode_r) |
---|
91 | except: |
---|
92 | error_msg += "\nFile '%s' can't be opened?\n" % file2 |
---|
93 | error = True |
---|
94 | fid1.close() |
---|
95 | |
---|
96 | if error: |
---|
97 | raise RuntimeError, error_msg |
---|
98 | |
---|
99 | if globals is None: |
---|
100 | globals = expect_global_attributes |
---|
101 | |
---|
102 | ##### |
---|
103 | # First, check that files have the required structure |
---|
104 | ##### |
---|
105 | |
---|
106 | # dimensions - only check expected dimensions |
---|
107 | for key in expected_dimensions: |
---|
108 | if key not in fid1.dimensions.keys(): |
---|
109 | error_msg += ("\nFile %s doesn't contain dimension '%s'\n" |
---|
110 | % (filename1, key)) |
---|
111 | error = True |
---|
112 | if key not in fid2.dimensions.keys(): |
---|
113 | error_msg += ("\nFile %s doesn't contain dimension '%s'\n" |
---|
114 | % (filename2, key)) |
---|
115 | error = True |
---|
116 | |
---|
117 | # now check that dimensions are the same length |
---|
118 | # NOTE: DOESN'T CHECK 'UNLIMITED' DIMENSIONS YET! (get None at the moment) |
---|
119 | for dim in expected_dimensions: |
---|
120 | dim1_shape = fid1.dimensions.get(dim, None) |
---|
121 | dim2_shape = fid2.dimensions.get(dim, None) |
---|
122 | if dim1_shape != dim2_shape: |
---|
123 | error_msg += ("\nFile %s has '%s' dimension of size %s,\n" |
---|
124 | "file %s has that dimension of size %s\n" |
---|
125 | % (filename1, dim, str(dim1_shape), |
---|
126 | filename2, str(dim2_shape))) |
---|
127 | error = True |
---|
128 | |
---|
129 | # check that we have the required globals |
---|
130 | if globals: |
---|
131 | for glob in globals: |
---|
132 | if glob not in dir(fid1): |
---|
133 | error_msg += ("\nGlobal attribute '%s' isn't in file %s\n" |
---|
134 | % (glob, filename1)) |
---|
135 | error = True |
---|
136 | if glob not in dir(fid2): |
---|
137 | error_msg += ("\nGlobal attribute '%s' isn't in file %s\n" |
---|
138 | % (glob, filename2)) |
---|
139 | error = True |
---|
140 | else: |
---|
141 | # get list of global attributes |
---|
142 | glob_vars1 = [] |
---|
143 | glob_vars2 = [] |
---|
144 | for glob in expect_global_attributes: |
---|
145 | if glob in dir(fid1): |
---|
146 | glob_vars1.append(glob) |
---|
147 | if glob in dir(fid2): |
---|
148 | glob_vars2.append(glob) |
---|
149 | |
---|
150 | # now check attribute lists are same |
---|
151 | if glob_vars1 != glob_vars2: |
---|
152 | error_msg = ('\nFiles differ in global attributes:\n' |
---|
153 | '%s: %s,\n' |
---|
154 | '%s: %s\n' |
---|
155 | % (filename1, str(glob_vars1), |
---|
156 | filename2, str(glob_vars2))) |
---|
157 | error = True |
---|
158 | |
---|
159 | # get variables to test |
---|
160 | if variables: |
---|
161 | for var in variables: |
---|
162 | if var not in fid1.variables.keys(): |
---|
163 | error_msg += ("\nVariable '%s' isn't in file %s\n" |
---|
164 | % (var, filename1)) |
---|
165 | error = True |
---|
166 | if var not in fid2.variables.keys(): |
---|
167 | error_msg += ("\nVariable '%s' isn't in file %s\n" |
---|
168 | % (var, filename2)) |
---|
169 | error = True |
---|
170 | else: |
---|
171 | # check that variables are as expected in both files |
---|
172 | var_names1 = [] |
---|
173 | var_names2 = [] |
---|
174 | for var_name in expected_variables: |
---|
175 | if fid1.variables.has_key(var_name): |
---|
176 | var_names1.append(var_name) |
---|
177 | if fid2.variables.has_key(var_name): |
---|
178 | var_names2.append(var_name) |
---|
179 | |
---|
180 | if var_names1 != var_names2: |
---|
181 | error_msg += ('\nVariables are not the same between files:\n' |
---|
182 | '%s variables=%s,\n' |
---|
183 | '%s variables=%s\n' |
---|
184 | % (filename1, str(var_names1), filename2, str(var_names2))) |
---|
185 | error = True |
---|
186 | variables = var_names1 |
---|
187 | |
---|
188 | # get size of time dimension |
---|
189 | num_timesteps1 = fid1.variables['time'].shape |
---|
190 | num_timesteps2 = fid2.variables['time'].shape |
---|
191 | if num_timesteps1 != num_timesteps2: |
---|
192 | error_msg += ('Files have different number of timesteps:\n' |
---|
193 | '%s=%s,\n' |
---|
194 | '%s=%s\n' |
---|
195 | % (filename1, str(num_timesteps1), |
---|
196 | filename2, str(num_timesteps2))) |
---|
197 | error = True |
---|
198 | |
---|
199 | num_timesteps = num_timesteps1[0] |
---|
200 | |
---|
201 | # variable shapes same? |
---|
202 | for var_name in variables: |
---|
203 | var1 = fid1.variables[var_name] |
---|
204 | var2 = fid2.variables[var_name] |
---|
205 | var1_shape = var1.shape |
---|
206 | var2_shape = var2.shape |
---|
207 | if var1_shape != var2_shape: |
---|
208 | error_msg += ('Files differ in variable %s shape:\n' |
---|
209 | '%s: %s,\n' |
---|
210 | '%s: %s\n' |
---|
211 | % (var_name, filename1, str(var1_shape), |
---|
212 | filename2, str(var2_shape))) |
---|
213 | error = True |
---|
214 | |
---|
215 | if error: |
---|
216 | fid1.close() |
---|
217 | fid2.close() |
---|
218 | raise RuntimeError, error_msg |
---|
219 | |
---|
220 | ##### |
---|
221 | # Now check that actual data values are the same |
---|
222 | ##### |
---|
223 | |
---|
224 | # check values of global attributes |
---|
225 | for glob_name in globals: |
---|
226 | if getattr(fid1, glob_name) != getattr(fid2, glob_name): |
---|
227 | error_msg += ("\nFiles differ in global '%s':\n" |
---|
228 | "%s: '%s',\n" |
---|
229 | "%s: '%s'\n" |
---|
230 | % (glob_name, filename1, str(g1), filename2, str(g2))) |
---|
231 | error = True |
---|
232 | |
---|
233 | # check data variables, be clever with time series data |
---|
234 | for var_name in variables: |
---|
235 | var_dims = expected_variables[var_name] |
---|
236 | if (len(var_dims) > 1) and (var_dims[0] == 'number_of_timesteps'): |
---|
237 | # time series, check by timestep block |
---|
238 | for t in xrange(num_timesteps): |
---|
239 | var1 = num.array(fid1.variables[var_name][t,:]) |
---|
240 | var2 = num.array(fid2.variables[var_name][t,:]) |
---|
241 | if var1 != var2: |
---|
242 | for i in xrange(len(var1)): |
---|
243 | if var1[i] != var2[i]: |
---|
244 | error_msg += ('\nFiles differ in variable ' |
---|
245 | '%s[%d,%d]:\n' |
---|
246 | '%s: %f\n' |
---|
247 | '%s: %f\n' |
---|
248 | 'difference=%f\n' |
---|
249 | % (var_name, t, i, |
---|
250 | filename1, var1[i], |
---|
251 | filename2, var2[i], |
---|
252 | var1[i]-var2[i])) |
---|
253 | break |
---|
254 | error = True |
---|
255 | else: |
---|
256 | # simple data, check whole thing at once |
---|
257 | var1 = num.array(fid1.variables[var_name][:]) |
---|
258 | var2 = num.array(fid2.variables[var_name][:]) |
---|
259 | if not num.allclose(var1, var2): |
---|
260 | for j in xrange(len(var1)): |
---|
261 | if abs(var1[j] - var2[j]) > epsilon: |
---|
262 | error_msg += ('\nFiles differ in variable ' |
---|
263 | '%s[%d]:\n' |
---|
264 | '%s: %f\n' |
---|
265 | '%s: %f\n' |
---|
266 | 'difference=%f\n' |
---|
267 | % (var_name, j, |
---|
268 | filename1, var1[j], |
---|
269 | filename2, var2[j], |
---|
270 | var1[j]-var2[j])) |
---|
271 | break |
---|
272 | error = True |
---|
273 | |
---|
274 | ##### |
---|
275 | # close files and signal OK or ERROR |
---|
276 | ##### |
---|
277 | |
---|
278 | fid1.close() |
---|
279 | fid2.close() |
---|
280 | |
---|
281 | if error: |
---|
282 | raise RuntimeError, error_msg |
---|
283 | |
---|
284 | return |
---|
285 | |
---|
286 | |
---|
287 | ## |
---|
288 | # @brief Return a usage string. |
---|
289 | def usage(): |
---|
290 | result = [] |
---|
291 | a = result.append |
---|
292 | a('Usage: %s <options> <file1> <file2>\n' % ProgName) |
---|
293 | a('where <options> is zero or more of:\n') |
---|
294 | a(' -h print this help\n') |
---|
295 | a(" -a <val> set absolute threshold of 'equivalent'\n") |
---|
296 | a(" -r <val> set relative threshold of 'equivalent'\n") |
---|
297 | a(' -g <arg> check only global attributes specified\n') |
---|
298 | a(' <arg> has the form <globname>[,<globname>[,...]]\n') |
---|
299 | a(' -t <arg> check only timesteps specified\n') |
---|
300 | a(' <arg> has the form <starttime>[,<stoptime>[,<step>]]\n') |
---|
301 | a(' -v <arg> check only the named variables\n') |
---|
302 | a(' <arg> has the form <varname>[,<varname>[,...]]\n') |
---|
303 | a('and <file1> and <file2> are two SWW files to compare.\n') |
---|
304 | a('\n') |
---|
305 | a('The program exit status is one of:\n') |
---|
306 | a(' 0 the two files are equivalent\n') |
---|
307 | a(' else the files are not equivalent.') |
---|
308 | return ''.join(result) |
---|
309 | |
---|
310 | ## |
---|
311 | # @brief Print a message to stderr. |
---|
312 | def warn(msg): |
---|
313 | print >>sys.stderr, msg |
---|
314 | |
---|
315 | |
---|
316 | ## |
---|
317 | # @brief |
---|
318 | # @param argv |
---|
319 | # @return The status code the program will exit with. |
---|
320 | def main(argv=None): |
---|
321 | if argv is None: |
---|
322 | argv = sys.argv |
---|
323 | |
---|
324 | try: |
---|
325 | try: |
---|
326 | opts, args = getopt.getopt(argv[1:], 'hg:t:v:', |
---|
327 | ['help', 'globals', |
---|
328 | 'variables', 'timesteps']) |
---|
329 | except getopt.error, msg: |
---|
330 | raise Usage(msg) |
---|
331 | except Usage, err: |
---|
332 | print >>sys.stderr, err.msg |
---|
333 | print >>sys.stderr, "for help use --help" |
---|
334 | return 2 |
---|
335 | |
---|
336 | # process options |
---|
337 | globals = None |
---|
338 | timesteps = None |
---|
339 | variables = None |
---|
340 | for opt, arg in opts: |
---|
341 | if opt in ('-h', '--help'): |
---|
342 | print usage() |
---|
343 | sys.exit(0) |
---|
344 | elif opt in ('-g', '--globals'): |
---|
345 | globals = arg.split(',') |
---|
346 | elif opt in ('-t', '--timesteps'): |
---|
347 | timesteps = arg.split(',') |
---|
348 | elif opt in ('-v', '--variables'): |
---|
349 | variables = arg.split(',') |
---|
350 | |
---|
351 | # process arguments |
---|
352 | if len(args) != 2: |
---|
353 | msg = usage() |
---|
354 | print 'msg=%s' % msg |
---|
355 | raise Usage(msg) |
---|
356 | |
---|
357 | try: |
---|
358 | files_are_the_same(args, globals=globals, |
---|
359 | timesteps=timesteps, variables=variables) |
---|
360 | except RuntimeError, msg: |
---|
361 | print msg |
---|
362 | return 10 |
---|
363 | |
---|
364 | |
---|
365 | if __name__ == "__main__": |
---|
366 | global ProgName |
---|
367 | |
---|
368 | ProgName = os.path.basename(sys.argv[0]) |
---|
369 | |
---|
370 | sys.exit(main()) |
---|
371 | |
---|