[1758] | 1 | """This module contains various auxiliary function used by pyvolution. |
---|
| 2 | |
---|
| 3 | It is also a clearing house for functions that may later earn a module |
---|
| 4 | of their own. |
---|
| 5 | """ |
---|
| 6 | |
---|
| 7 | |
---|
[1911] | 8 | import utilities.polygon |
---|
[1932] | 9 | from warnings import warn |
---|
[1911] | 10 | |
---|
| 11 | |
---|
[1758] | 12 | |
---|
| 13 | def file_function(filename, |
---|
| 14 | domain = None, |
---|
| 15 | quantities = None, |
---|
[1986] | 16 | interpolation_points = None, |
---|
| 17 | verbose = False, |
---|
| 18 | use_cache = False): |
---|
[1835] | 19 | """Read time history of spatial data from NetCDF file and return |
---|
| 20 | a callable object. |
---|
[1758] | 21 | |
---|
[1905] | 22 | Input variables: |
---|
| 23 | |
---|
| 24 | filename - Name of sww or tms file |
---|
| 25 | |
---|
| 26 | If the file has extension 'sww' then it is assumed to be spatio-temporal |
---|
| 27 | or temporal and the callable object will have the form f(t,x,y) or f(t) |
---|
| 28 | depending on whether the file contains spatial data |
---|
[1758] | 29 | |
---|
[1905] | 30 | If the file has extension 'tms' then it is assumed to be temporal only |
---|
| 31 | and the callable object will have the form f(t) |
---|
[1835] | 32 | |
---|
[1905] | 33 | Either form will return interpolated values based on the input file |
---|
| 34 | using the underlying interpolation_function. |
---|
[1835] | 35 | |
---|
[1905] | 36 | domain - Associated domain object |
---|
| 37 | If domain is specified, model time (domain.starttime) |
---|
| 38 | will be checked and possibly modified. |
---|
[1835] | 39 | |
---|
[1905] | 40 | All times are assumed to be in UTC |
---|
| 41 | |
---|
| 42 | All spatial information is assumed to be in absolute UTM coordinates. |
---|
[1835] | 43 | |
---|
[1905] | 44 | quantities - the name of the quantity to be interpolated or a |
---|
| 45 | list of quantity names. The resulting function will return |
---|
| 46 | a tuple of values - one for each quantity |
---|
[1835] | 47 | |
---|
[1905] | 48 | interpolation_points - list of absolute UTM coordinates for points at |
---|
| 49 | which values are sought |
---|
| 50 | |
---|
[1986] | 51 | use_cache: True means that caching of intermediate result of |
---|
| 52 | Interpolation_function is attempted |
---|
[1905] | 53 | |
---|
| 54 | |
---|
[1835] | 55 | See Interpolation function for further documentation |
---|
[1758] | 56 | """ |
---|
[1986] | 57 | |
---|
| 58 | |
---|
| 59 | if use_cache is True: |
---|
| 60 | try: |
---|
| 61 | from caching import cache |
---|
| 62 | except: |
---|
| 63 | msg = 'Caching was requested, but caching module'+\ |
---|
| 64 | 'could not be imported' |
---|
| 65 | raise msg |
---|
| 66 | |
---|
| 67 | |
---|
| 68 | f = cache(_file_function, |
---|
| 69 | filename, |
---|
| 70 | {'domain': domain, |
---|
| 71 | 'quantities': quantities, |
---|
| 72 | 'interpolation_points': interpolation_points, |
---|
| 73 | 'verbose': verbose}, |
---|
| 74 | dependencies = [filename], |
---|
| 75 | compression = False, |
---|
| 76 | verbose = verbose) |
---|
| 77 | #FIXME (Ole): Pass cache arguments, such as compression, in some sort of |
---|
| 78 | #structure |
---|
| 79 | |
---|
| 80 | else: |
---|
| 81 | f = _file_function(filename, |
---|
| 82 | domain, |
---|
| 83 | quantities, |
---|
| 84 | interpolation_points, |
---|
| 85 | verbose) |
---|
| 86 | |
---|
| 87 | return f |
---|
| 88 | |
---|
| 89 | |
---|
| 90 | |
---|
| 91 | def _file_function(filename, |
---|
| 92 | domain = None, |
---|
| 93 | quantities = None, |
---|
| 94 | interpolation_points = None, |
---|
| 95 | verbose = False): |
---|
| 96 | """Internal function |
---|
[1835] | 97 | |
---|
[1986] | 98 | See file_function for documentatiton |
---|
| 99 | """ |
---|
| 100 | |
---|
[1758] | 101 | |
---|
| 102 | #FIXME (OLE): Should check origin of domain against that of file |
---|
| 103 | #In fact, this is where origin should be converted to that of domain |
---|
| 104 | #Also, check that file covers domain fully. |
---|
[1859] | 105 | #If we use the suggested Point_set class for interpolation points |
---|
| 106 | #here it would be easier |
---|
[1758] | 107 | |
---|
[1859] | 108 | #Take into account: |
---|
| 109 | #- domain's georef |
---|
| 110 | #- sww file's georef |
---|
[1884] | 111 | #- interpolation points as absolute UTM coordinates |
---|
[1859] | 112 | |
---|
| 113 | |
---|
[1758] | 114 | assert type(filename) == type(''),\ |
---|
| 115 | 'First argument to File_function must be a string' |
---|
| 116 | |
---|
| 117 | try: |
---|
| 118 | fid = open(filename) |
---|
| 119 | except Exception, e: |
---|
| 120 | msg = 'File "%s" could not be opened: Error="%s"'\ |
---|
| 121 | %(filename, e) |
---|
| 122 | raise msg |
---|
| 123 | |
---|
| 124 | line = fid.readline() |
---|
| 125 | fid.close() |
---|
| 126 | |
---|
| 127 | if quantities is None: |
---|
| 128 | if domain is not None: |
---|
| 129 | quantities = domain.conserved_quantities |
---|
| 130 | |
---|
| 131 | |
---|
| 132 | |
---|
| 133 | if line[:3] == 'CDF': |
---|
| 134 | return get_netcdf_file_function(filename, domain, quantities, |
---|
[1986] | 135 | interpolation_points, |
---|
| 136 | verbose = verbose) |
---|
[1758] | 137 | else: |
---|
| 138 | raise 'Must be a NetCDF File' |
---|
| 139 | |
---|
| 140 | |
---|
| 141 | |
---|
[1986] | 142 | def get_netcdf_file_function(filename, |
---|
| 143 | domain=None, |
---|
| 144 | quantity_names=None, |
---|
| 145 | interpolation_points=None, |
---|
| 146 | verbose = False): |
---|
[1758] | 147 | """Read time history of spatial data from NetCDF sww file and |
---|
| 148 | return a callable object f(t,x,y) |
---|
| 149 | which will return interpolated values based on the input file. |
---|
| 150 | |
---|
| 151 | If domain is specified, model time (domain.starttime) |
---|
| 152 | will be checked and possibly modified |
---|
| 153 | |
---|
| 154 | All times are assumed to be in UTC |
---|
| 155 | |
---|
| 156 | See Interpolation function for further documetation |
---|
| 157 | |
---|
| 158 | """ |
---|
| 159 | |
---|
| 160 | |
---|
| 161 | #FIXME: Check that model origin is the same as file's origin |
---|
| 162 | #(both in UTM coordinates) |
---|
| 163 | #If not - modify those from file to match domain |
---|
| 164 | #Take this code from e.g. dem2pts in data_manager.py |
---|
| 165 | #FIXME: Use geo_reference to read and write xllcorner... |
---|
| 166 | |
---|
| 167 | |
---|
[1986] | 168 | #FIXME: Maybe move caching out to this level rather than at the |
---|
| 169 | #Interpolation_function level (below) |
---|
| 170 | |
---|
[1758] | 171 | import time, calendar, types |
---|
| 172 | from config import time_format |
---|
| 173 | from Scientific.IO.NetCDF import NetCDFFile |
---|
[1884] | 174 | from Numeric import array, zeros, Float, alltrue, concatenate, reshape |
---|
[2516] | 175 | from utilities.numerical_tools import ensure_numeric |
---|
[1758] | 176 | |
---|
| 177 | #Open NetCDF file |
---|
| 178 | if verbose: print 'Reading', filename |
---|
| 179 | fid = NetCDFFile(filename, 'r') |
---|
| 180 | |
---|
| 181 | if type(quantity_names) == types.StringType: |
---|
| 182 | quantity_names = [quantity_names] |
---|
| 183 | |
---|
| 184 | if quantity_names is None or len(quantity_names) < 1: |
---|
[1819] | 185 | #If no quantities are specified get quantities from file |
---|
| 186 | #x, y, time are assumed as the independent variables so |
---|
| 187 | #they are excluded as potentiol quantities |
---|
[1758] | 188 | quantity_names = [] |
---|
| 189 | for name in fid.variables.keys(): |
---|
| 190 | if name not in ['x', 'y', 'time']: |
---|
| 191 | quantity_names.append(name) |
---|
| 192 | |
---|
[1819] | 193 | if len(quantity_names) < 1: |
---|
| 194 | msg = 'ERROR: At least one independent value must be specified' |
---|
| 195 | raise msg |
---|
| 196 | |
---|
| 197 | |
---|
[1884] | 198 | if interpolation_points is not None: |
---|
[1961] | 199 | interpolation_points = ensure_numeric(interpolation_points, Float) |
---|
[1884] | 200 | |
---|
| 201 | |
---|
| 202 | |
---|
[1819] | 203 | #Now assert that requested quantitites (and the independent ones) |
---|
| 204 | #are present in file |
---|
[1758] | 205 | missing = [] |
---|
[1835] | 206 | for quantity in ['time'] + quantity_names: |
---|
[1758] | 207 | if not fid.variables.has_key(quantity): |
---|
| 208 | missing.append(quantity) |
---|
| 209 | |
---|
| 210 | if len(missing) > 0: |
---|
| 211 | msg = 'Quantities %s could not be found in file %s'\ |
---|
| 212 | %(str(missing), filename) |
---|
| 213 | fid.close() |
---|
| 214 | raise msg |
---|
| 215 | |
---|
| 216 | #Decide whether this data has a spatial dimension |
---|
| 217 | spatial = True |
---|
| 218 | for quantity in ['x', 'y']: |
---|
| 219 | if not fid.variables.has_key(quantity): |
---|
| 220 | spatial = False |
---|
| 221 | |
---|
[1835] | 222 | if filename[-3:] == 'tms' and spatial is True: |
---|
| 223 | msg = 'Files of type tms must not contain spatial information' |
---|
| 224 | raise msg |
---|
[1758] | 225 | |
---|
[1835] | 226 | if filename[-3:] == 'sww' and spatial is False: |
---|
| 227 | msg = 'Files of type sww must contain spatial information' |
---|
| 228 | raise msg |
---|
[1758] | 229 | |
---|
| 230 | #Get first timestep |
---|
| 231 | try: |
---|
| 232 | starttime = fid.starttime[0] |
---|
| 233 | except ValueError: |
---|
| 234 | msg = 'Could not read starttime from file %s' %filename |
---|
| 235 | raise msg |
---|
| 236 | |
---|
| 237 | #Get variables |
---|
| 238 | if verbose: print 'Get variables' |
---|
| 239 | time = fid.variables['time'][:] |
---|
| 240 | |
---|
| 241 | if spatial: |
---|
| 242 | #Get origin |
---|
| 243 | xllcorner = fid.xllcorner[0] |
---|
| 244 | yllcorner = fid.yllcorner[0] |
---|
[1859] | 245 | zone = fid.zone[0] |
---|
[1758] | 246 | |
---|
| 247 | x = fid.variables['x'][:] |
---|
| 248 | y = fid.variables['y'][:] |
---|
| 249 | triangles = fid.variables['volumes'][:] |
---|
| 250 | |
---|
| 251 | x = reshape(x, (len(x),1)) |
---|
| 252 | y = reshape(y, (len(y),1)) |
---|
| 253 | vertex_coordinates = concatenate((x,y), axis=1) #m x 2 array |
---|
| 254 | |
---|
[1884] | 255 | if interpolation_points is not None: |
---|
| 256 | #Adjust for georef |
---|
| 257 | interpolation_points[:,0] -= xllcorner |
---|
| 258 | interpolation_points[:,1] -= yllcorner |
---|
| 259 | |
---|
[1758] | 260 | |
---|
[1884] | 261 | |
---|
| 262 | |
---|
[1758] | 263 | if domain is not None: |
---|
| 264 | #Update domain.startime if it is *earlier* than starttime |
---|
| 265 | if starttime > domain.starttime: |
---|
| 266 | msg = 'WARNING: Start time as specified in domain (%f)'\ |
---|
| 267 | %domain.starttime |
---|
| 268 | msg += ' is earlier than the starttime of file %s (%f).'\ |
---|
| 269 | %(filename, starttime) |
---|
| 270 | msg += ' Modifying domain starttime accordingly.' |
---|
| 271 | |
---|
| 272 | if verbose: print msg |
---|
| 273 | domain.starttime = starttime #Modifying model time |
---|
| 274 | if verbose: print 'Domain starttime is now set to %f'\ |
---|
| 275 | %domain.starttime |
---|
| 276 | |
---|
| 277 | |
---|
| 278 | #If domain.startime is *later* than starttime, |
---|
| 279 | #move time back - relative to domain's time |
---|
| 280 | if domain.starttime > starttime: |
---|
| 281 | time = time - domain.starttime + starttime |
---|
| 282 | |
---|
[1859] | 283 | #FIXME Use method in geo to reconcile |
---|
| 284 | #if spatial: |
---|
| 285 | #assert domain.geo_reference.xllcorner == xllcorner |
---|
| 286 | #assert domain.geo_reference.yllcorner == yllcorner |
---|
| 287 | #assert domain.geo_reference.zone == zone |
---|
| 288 | |
---|
[1758] | 289 | if verbose: |
---|
| 290 | print 'File_function data obtained from: %s' %filename |
---|
| 291 | print ' References:' |
---|
| 292 | #print ' Datum ....' #FIXME |
---|
| 293 | if spatial: |
---|
| 294 | print ' Lower left corner: [%f, %f]'\ |
---|
| 295 | %(xllcorner, yllcorner) |
---|
| 296 | print ' Start time: %f' %starttime |
---|
| 297 | |
---|
| 298 | |
---|
| 299 | #Produce values for desired data points at |
---|
| 300 | #each timestep |
---|
| 301 | |
---|
| 302 | quantities = {} |
---|
| 303 | for i, name in enumerate(quantity_names): |
---|
| 304 | quantities[name] = fid.variables[name][:] |
---|
| 305 | fid.close() |
---|
| 306 | |
---|
| 307 | |
---|
| 308 | from least_squares import Interpolation_function |
---|
| 309 | |
---|
| 310 | if not spatial: |
---|
| 311 | vertex_coordinates = triangles = interpolation_points = None |
---|
[1986] | 312 | |
---|
| 313 | |
---|
[1758] | 314 | return Interpolation_function(time, |
---|
| 315 | quantities, |
---|
| 316 | quantity_names, |
---|
| 317 | vertex_coordinates, |
---|
| 318 | triangles, |
---|
| 319 | interpolation_points, |
---|
| 320 | verbose = verbose) |
---|
| 321 | |
---|
| 322 | |
---|
| 323 | |
---|
| 324 | |
---|
| 325 | |
---|
[1927] | 326 | def multiple_replace(text, dictionary): |
---|
| 327 | """Multiple replace of words in text |
---|
[1919] | 328 | |
---|
[1927] | 329 | text: String to be modified |
---|
| 330 | dictionary: Mapping of words that are to be substituted |
---|
[1919] | 331 | |
---|
[1927] | 332 | Python Cookbook 3.14 page 88 and page 90 |
---|
| 333 | """ |
---|
[1919] | 334 | |
---|
[1927] | 335 | import re |
---|
| 336 | |
---|
| 337 | #Create a regular expression from all of the dictionary keys |
---|
| 338 | #matching only entire words |
---|
| 339 | regex = re.compile(r'\b'+ \ |
---|
| 340 | r'\b|\b'.join(map(re.escape, dictionary.keys()))+ \ |
---|
| 341 | r'\b' ) |
---|
[1919] | 342 | |
---|
[1927] | 343 | #For each match, lookup the corresponding value in the dictionary |
---|
| 344 | return regex.sub(lambda match: dictionary[match.group(0)], text) |
---|
[1919] | 345 | |
---|
| 346 | |
---|
[1927] | 347 | |
---|
| 348 | |
---|
| 349 | def apply_expression_to_dictionary(expression, dictionary):#dictionary): |
---|
| 350 | """Apply arbitrary expression to values of dictionary |
---|
| 351 | |
---|
[1919] | 352 | Given an expression in terms of the keys, replace key by the |
---|
| 353 | corresponding values and evaluate. |
---|
[1927] | 354 | |
---|
[1919] | 355 | |
---|
[1927] | 356 | expression: Arbitrary, e.g. arithmetric, expression relating keys |
---|
| 357 | from dictionary. |
---|
| 358 | |
---|
| 359 | dictionary: Mapping between symbols in expression and objects that |
---|
| 360 | will be evaluated by expression. |
---|
| 361 | Values in dictionary must support operators given in |
---|
[2314] | 362 | expression e.g. by overloading |
---|
| 363 | |
---|
| 364 | due to a limitation with Numeric, this can not evaluate 0/0 |
---|
| 365 | In general, the user can fix by adding 1e-30 to the numerator. |
---|
| 366 | SciPy core can handle this situation. |
---|
[1919] | 367 | """ |
---|
| 368 | |
---|
| 369 | import types |
---|
[1924] | 370 | import re |
---|
[1919] | 371 | |
---|
| 372 | assert isinstance(expression, basestring) |
---|
[1927] | 373 | assert type(dictionary) == types.DictType |
---|
[1919] | 374 | |
---|
[1927] | 375 | #Convert dictionary values to textual representations suitable for eval |
---|
| 376 | D = {} |
---|
| 377 | for key in dictionary: |
---|
| 378 | D[key] = 'dictionary["%s"]' %key |
---|
[1924] | 379 | |
---|
[1927] | 380 | #Perform substitution of variables |
---|
| 381 | expression = multiple_replace(expression, D) |
---|
[1924] | 382 | |
---|
[1919] | 383 | #Evaluate and return |
---|
| 384 | try: |
---|
| 385 | return eval(expression) |
---|
| 386 | except NameError, e: |
---|
| 387 | msg = 'Expression "%s" could not be evaluated: %s' %(expression, e) |
---|
| 388 | raise NameError, msg |
---|
[2314] | 389 | except ValueError, e: |
---|
| 390 | msg = 'Expression "%s" could not be evaluated: %s' %(expression, e) |
---|
| 391 | raise ValueError, msg |
---|
[1919] | 392 | |
---|
| 393 | |
---|
| 394 | |
---|
| 395 | |
---|
[2516] | 396 | #################################### |
---|
| 397 | ####OBSOLETE STUFF |
---|
[1919] | 398 | |
---|
| 399 | |
---|
[2526] | 400 | def angle(v): |
---|
| 401 | """Temporary Interface to new location""" |
---|
| 402 | |
---|
| 403 | import utilities.numerical_tools as NT |
---|
| 404 | |
---|
| 405 | msg = 'angle has moved from util.py. ' |
---|
| 406 | msg += 'Please use "from utilities.numerical_tools import angle"' |
---|
| 407 | warn(msg, DeprecationWarning) |
---|
| 408 | |
---|
| 409 | return NT.angle(v) |
---|
| 410 | |
---|
| 411 | def anglediff(v0, v1): |
---|
| 412 | """Temporary Interface to new location""" |
---|
| 413 | |
---|
| 414 | import utilities.numerical_tools as NT |
---|
| 415 | |
---|
| 416 | msg = 'anglediff has moved from util.py. ' |
---|
| 417 | msg += 'Please use "from utilities.numerical_tools import anglediff"' |
---|
| 418 | warn(msg, DeprecationWarning) |
---|
| 419 | |
---|
| 420 | return NT.anglediff(v0, v1) |
---|
| 421 | |
---|
| 422 | |
---|
| 423 | def mean(x): |
---|
| 424 | """Temporary Interface to new location""" |
---|
| 425 | |
---|
| 426 | import utilities.numerical_tools as NT |
---|
| 427 | |
---|
| 428 | msg = 'mean has moved from util.py. ' |
---|
| 429 | msg += 'Please use "from utilities.numerical_tools import mean"' |
---|
| 430 | warn(msg, DeprecationWarning) |
---|
| 431 | |
---|
| 432 | return NT.mean(x) |
---|
| 433 | |
---|
[2516] | 434 | def point_on_line(*args, **kwargs): |
---|
| 435 | """Temporary Interface to new location""" |
---|
[1919] | 436 | |
---|
[2516] | 437 | msg = 'point_on_line has moved from util.py. ' |
---|
| 438 | msg += 'Please use "from utilities.polygon import point_on_line"' |
---|
| 439 | warn(msg, DeprecationWarning) |
---|
[1919] | 440 | |
---|
[2516] | 441 | return utilities.polygon.point_on_line(*args, **kwargs) |
---|
| 442 | |
---|
| 443 | def inside_polygon(*args, **kwargs): |
---|
| 444 | """Temporary Interface to new location""" |
---|
[1758] | 445 | |
---|
[2516] | 446 | print 'inside_polygon has moved from util.py. ', |
---|
| 447 | print 'Please use "from utilities.polygon import inside_polygon"' |
---|
[1758] | 448 | |
---|
[2516] | 449 | return utilities.polygon.inside_polygon(*args, **kwargs) |
---|
| 450 | |
---|
| 451 | def outside_polygon(*args, **kwargs): |
---|
| 452 | """Temporary Interface to new location""" |
---|
[1758] | 453 | |
---|
[2516] | 454 | print 'outside_polygon has moved from util.py. ', |
---|
| 455 | print 'Please use "from utilities.polygon import outside_polygon"' |
---|
| 456 | |
---|
| 457 | return utilities.polygon.outside_polygon(*args, **kwargs) |
---|
| 458 | |
---|
| 459 | |
---|
| 460 | def separate_points_by_polygon(*args, **kwargs): |
---|
| 461 | """Temporary Interface to new location""" |
---|
| 462 | |
---|
| 463 | print 'separate_points_by_polygon has moved from util.py. ', |
---|
| 464 | print 'Please use "from utilities.polygon import separate_points_by_polygon"' |
---|
| 465 | |
---|
| 466 | return utilities.polygon.separate_points_by_polygon(*args, **kwargs) |
---|
| 467 | |
---|
| 468 | |
---|
| 469 | |
---|
| 470 | def read_polygon(*args, **kwargs): |
---|
| 471 | """Temporary Interface to new location""" |
---|
| 472 | |
---|
| 473 | print 'read_polygon has moved from util.py. ', |
---|
| 474 | print 'Please use "from utilities.polygon import read_polygon"' |
---|
| 475 | |
---|
| 476 | return utilities.polygon.read_polygon(*args, **kwargs) |
---|
| 477 | |
---|
| 478 | |
---|
| 479 | def populate_polygon(*args, **kwargs): |
---|
| 480 | """Temporary Interface to new location""" |
---|
| 481 | |
---|
| 482 | print 'populate_polygon has moved from util.py. ', |
---|
| 483 | print 'Please use "from utilities.polygon import populate_polygon"' |
---|
| 484 | |
---|
| 485 | return utilities.polygon.populate_polygon(*args, **kwargs) |
---|
| 486 | |
---|
| 487 | def read_xya(filename): |
---|
| 488 | """Read simple xya file, possibly with a header in the first line, just |
---|
| 489 | x y [attributes] |
---|
| 490 | separated by whitespace |
---|
| 491 | |
---|
| 492 | Format can be either ASCII or NetCDF |
---|
| 493 | |
---|
| 494 | Return list of points, list of attributes and |
---|
| 495 | attribute names if present otherwise None |
---|
| 496 | """ |
---|
| 497 | print "read_xya is obsolete. Use import_points_file from load_mesh.loadASCII" |
---|
| 498 | #FIXME: Probably obsoleted by similar function in load_ASCII |
---|
| 499 | #FIXME: Name read_data_points (or see 'load' in pylab) |
---|
| 500 | |
---|
| 501 | |
---|
| 502 | from load_mesh.loadASCII import import_points_file |
---|
| 503 | |
---|
| 504 | points_dict = import_points_file(filename) |
---|
| 505 | return points_dict['pointlist'], points_dict['attributelist'] |
---|
| 506 | |
---|
| 507 | |
---|
| 508 | |
---|
| 509 | |
---|