Changeset 6232


Ignore:
Timestamp:
Jan 23, 2009, 2:44:41 AM (16 years ago)
Author:
ole
Message:

Improved test_caching_of_set_quantity - it still needs to be moved to validation suite.

Files:
4 edited

Legend:

Unmodified
Added
Removed

  • anuga_core/source/anuga/caching/caching.py

    r6230 r6232  
    4747import os
    4848if os.name in ['nt', 'dos', 'win32', 'what else?']:
    49   unix = 0
     49  unix = False
    5050else:
    51   unix = 1
     51  unix = True
    5252
    5353import Numeric as num
    … …  
    8282  'cachedir': cachedir,  # Default cache directory
    8383  'maxfiles': 1000000,   # Maximum number of cached files
    84   'savestat': 1,         # Log caching info to stats file
    85   'verbose': 1,          # Write messages to standard output
    86   'bin': 1,              # Use binary format (more efficient)
    87   'compression': 1,      # Use zlib compression
    88   'bytecode': 1,         # Recompute if bytecode has changed
    89   'expire': 0            # Automatically remove files that have been accessed
     84  'savestat': True,      # Log caching info to stats file
     85  'verbose': True,       # Write messages to standard output
     86  'bin': True,           # Use binary format (more efficient)
     87  'compression': True,   # Use zlib compression
     88  'bytecode': True,      # Recompute if bytecode has changed
     89  'expire': False        # Automatically remove files that have been accessed
    9090                         # least recently
    9191}
    … …  
    116116# Function cache - the main routine
    117117
    118 def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
    119           verbose=None, compression=None, evaluate=0, test=0, clear=0,
    120           return_filename=0):
    121   """Supervised caching of function results.
     118def cache(func,
     119          args=(),
     120          kwargs={},
     121          dependencies=None,
     122          cachedir=None,
     123          verbose=None,
     124          compression=None,
     125          evaluate=False,
     126          test=False,
     127          clear=False,
     128          return_filename=False):
     129  """Supervised caching of function results. Also known as memoization.
    122130
    123131  USAGE:
    … …  
    134142                       (Default: options['verbose'])
    135143    compression --     Flag zlib compression (Default: options['compression'])
    136     evaluate --        Flag forced evaluation of func (Default: 0)
    137     test --            Flag test for cached results (Default: 0)
    138     clear --           Flag delete cached results (Default: 0)   
    139     return_filename -- Flag return of cache filename (Default: 0)   
     144    evaluate --        Flag forced evaluation of func (Default: False)
     145    test --            Flag test for cached results (Default: False)
     146    clear --           Flag delete cached results (Default: False)   
     147    return_filename -- Flag return of cache filename (Default: False)   
    140148
    141149  DESCRIPTION:
    … …  
    190198  Explicit dependencies:
    191199    The call
    192       cache(func,(arg1,...,argn),dependencies = <list of filenames>)
     200      cache(func,(arg1,...,argn), dependencies = <list of filenames>)
    193201    Checks the size, creation time and modification time of each listed file.
    194202    If any file has changed the function is recomputed and the results stored
    … …  
    204212  Silent operation:
    205213    The call
    206       cache(func,(arg1,...,argn),verbose=0)
     214      cache(func,(arg1,...,argn), verbose=False)
    207215    suppresses messages to standard output.
    208216
    209217  Compression:
    210218    The call
    211       cache(func,(arg1,...,argn),compression=0)
    212     disables compression. (Default: compression=1). If the requested compressed
     219      cache(func,(arg1,...,argn), compression=False)
     220    disables compression. (Default: compression=True). If the requested compressed
    213221    or uncompressed file is not there, it'll try the other version.
    214222
    215223  Forced evaluation:
    216224    The call
    217       cache(func,(arg1,...,argn),evaluate=1)
     225      cache(func,(arg1,...,argn), evaluate=True)
    218226    forces the function to evaluate even though cached data may exist.
    219227
    220228  Testing for presence of cached result:
    221229    The call
    222       cache(func,(arg1,...,argn),test=1)
     230      cache(func,(arg1,...,argn), test=True)
    223231    retrieves cached result if it exists, otherwise None. The function will not
    224232    be evaluated. If both evaluate and test are switched on, evaluate takes
    … …  
    226234    ??NOTE: In case of hash collisions, this may return the wrong result as
    227235    ??it only checks if *a* cached result is present.
     236    # I think this was due to the bytecode option being False for some reason. (23/1/2009).
    228237   
    229238  Obtain cache filenames:
    230239    The call   
    231       cache(func,(arg1,...,argn),return_filename=1)
     240      cache(func,(arg1,...,argn), return_filename=True)
    232241    returns the hashed base filename under which this function and its
    233242    arguments would be cached
    … …  
    244253
    245254    New form of clear:
    246       cache(func,(arg1,...,argn),clear=1)
     255      cache(func,(arg1,...,argn), clear=True)
    247256    clears cached data for particular combination func and args
    248257     
    … …  
    325334 
    326335  # Check if previous computation has been cached
    327   if evaluate:
     336  if evaluate is True:
    328337    Retrieved = None  # Force evaluation of func regardless of caching status.
    329338    reason = 5
    … …  
    341350      T = None
    342351    else:  # Evaluate function and save to cache
    343       if verbose:
     352      if verbose is True:
    344353       
    345354        msg1(funcname, args, kwargs,reason)
    … …  
    359368      comptime = time.time()-t0
    360369
    361       if verbose:
     370      if verbose is True:
    362371        msg2(funcname,args,kwargs,comptime,reason)
    363372
    … …  
    365374      loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
    366375                                       funcname, dependencies, compression)
    367       if verbose:
     376      if verbose is True:
    368377        msg3(loadtime, CD, FN, deps, compression)
    369378      compressed = compression
    … …  
    413422# What remains here includes example of the
    414423# cache statistics form.
    415 def test(cachedir=None,verbose=0,compression=None):
     424def test(cachedir=None, verbose=False, compression=None):
    416425  """Test the functionality of caching.
    417426
    … …  
    420429
    421430  ARGUMENTS:
    422     verbose --     Flag whether caching will output its statistics (default=0)
     431    verbose --     Flag whether caching will output its statistics (default=False)
    423432    cachedir --    Directory for cache files (Default: options['cachedir'])
    424433    compression -- Flag zlib compression (Default: options['compression'])
    … …  
    11151124# -----------------------------------------------------------------------------
    11161125
    1117 def load_from_cache(CD,FN,compression):
     1126def load_from_cache(CD, FN, compression):
    11181127  """Load previously cached data from file FN
    11191128
    … …  
    11351144# -----------------------------------------------------------------------------
    11361145
    1137 def myopen(FN,mode,compression=1):
     1146def myopen(FN, mode, compression=True):
    11381147  """Open file FN using given mode
    11391148
    11401149  USAGE:
    1141     myopen(FN,mode,compression=1)
     1150    myopen(FN, mode, compression=True)
    11421151
    11431152  ARGUMENTS:
    … …  
    12591268# -----------------------------------------------------------------------------
    12601269
    1261 def mysave(T,file,compression):
     1270def mysave(T, file, compression):
    12621271  """Save data T to file
    12631272
    12641273  USAGE:
    1265     mysave(T,file,compression)
     1274    mysave(T, file, compression)
    12661275
    12671276  """
    … …  
    17241733# -----------------------------------------------------------------------------
    17251734
    1726 def checkdir(CD,verbose=None, warn=False):
     1735def checkdir(CD, verbose=None, warn=False):
    17271736  """Check or create caching directory
    17281737
    … …  
    17741783#==============================================================================
    17751784
    1776 def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
     1785def addstatsline(CD, funcname, FN, Retrieved, reason, comptime, loadtime,
    17771786                 compression):
    17781787  """Add stats entry
    … …  
    18521861# FIXME: should take cachedir as an optional arg
    18531862#
    1854 def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
     1863def __cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
    18551864  """  List caching statistics.
    18561865
    … …  
    19151924    print 'Reading file ', SD+FN
    19161925
    1917     while 1:
     1926    while True:
    19181927      A = input.readlines(blocksize)
    19191928      if len(A) == 0: break
    … …  
    20622071#==============================================================================
    20632072
    2064 def UpdateDict(Dict,key,info):
     2073def UpdateDict(Dict, key, info):
    20652074  """Update dictionary by adding new values to existing.
    20662075
    … …  
    20812090# -----------------------------------------------------------------------------
    20822091
    2083 def SortDict(Dict,sortidx=0):
     2092def SortDict(Dict, sortidx=0):
    20842093  """Sort dictionary
    20852094
    … …  
    21352144#==============================================================================
    21362145
    2137 def msg1(funcname,args,kwargs,reason):
     2146def msg1(funcname, args, kwargs, reason):
    21382147  """Message 1
    21392148
    21402149  USAGE:
    2141     msg1(funcname,args,kwargs,reason):
     2150    msg1(funcname, args, kwargs, reason):
    21422151  """
    21432152
    … …  
    21842193# -----------------------------------------------------------------------------
    21852194
    2186 def msg2(funcname,args,kwargs,comptime,reason):
     2195def msg2(funcname, args, kwargs, comptime, reason):
    21872196  """Message 2
    21882197
    21892198  USAGE:
    2190     msg2(funcname,args,kwargs,comptime,reason)
     2199    msg2(funcname, args, kwargs, comptime, reason)
    21912200  """
    21922201
    … …  
    22082217# -----------------------------------------------------------------------------
    22092218
    2210 def msg3(savetime, CD, FN, deps,compression):
     2219def msg3(savetime, CD, FN, deps, compression):
    22112220  """Message 3
    22122221
    22132222  USAGE:
    2214     msg3(savetime, CD, FN, deps,compression)
     2223    msg3(savetime, CD, FN, deps, compression)
    22152224  """
    22162225
    … …  
    22222231# -----------------------------------------------------------------------------
    22232232
    2224 def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
     2233def msg4(funcname, args, kwargs, deps, comptime, loadtime, CD, FN, compression):
    22252234  """Message 4
    22262235
    22272236  USAGE:
    2228     msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
     2237    msg4(funcname, args, kwargs, deps, comptime, loadtime, CD, FN, compression)
    22292238  """
    22302239
    … …  
    22422251# -----------------------------------------------------------------------------
    22432252
    2244 def msg5(CD,FN,deps,compression):
     2253def msg5(CD, FN, deps, compression):
    22452254  """Message 5
    22462255
    22472256  USAGE:
    2248     msg5(CD,FN,deps,compression)
     2257    msg5(CD, FN, deps, compression)
    22492258
    22502259  DESCRIPTION:
    … …  
    23042313# -----------------------------------------------------------------------------
    23052314
    2306 def msg6(funcname,args,kwargs):
     2315def msg6(funcname, args, kwargs):
    23072316  """Message 6
    23082317
    23092318  USAGE:
    2310     msg6(funcname,args,kwargs)
     2319    msg6(funcname, args, kwargs)
    23112320  """
    23122321
    … …  
    25072516  print
    25082517 
    2509   #import sys
    2510   #sys.exit()
    25112518  raise StandardError
    25122519

  • anuga_core/source/anuga/caching/test_caching.py

    r6231 r6232  
    433433        res1 = cache(f1, x, test=True, verbose=False)               
    434434        assert num.allclose(res1, ref1)               
     435       
     436        # Test that f2(x) is still clear
     437        cache(f2, x, clear=True, verbose=False)
     438        flag = cache(f2, x, test=True, verbose=False)       
     439        assert flag is None               
    435440       
    436441        # Run f2(x) and test result

  • anuga_core/source/anuga/fit_interpolate/fit.py

    r6191 r6232  
    457457                alpha=DEFAULT_ALPHA,
    458458                verbose=False,
    459                 acceptable_overshoot=1.01,
     459                acceptable_overshoot=1.01, # FIXME: Move to config - this value is assumed in caching test
    460460                mesh_origin=None,
    461461                data_origin=None,
    … …  
    478478              'data_origin': data_origin,
    479479              'max_read_lines': max_read_lines,
    480               'attribute_name': attribute_name,
    481               'use_cache': use_cache
     480              'attribute_name': attribute_name
    482481              }
    483482
    … …  
    494493        #from caching import myhash
    495494        #import copy
     495        #print args
    496496        #print kwargs
    497497        #print 'hashing:'
    … …  
    534534                 data_origin=None,
    535535                 max_read_lines=None,
    536                  attribute_name=None,
    537                  use_cache = False):
     536                 attribute_name=None):
    538537    """
    539538    Fit a smooth surface to a triangulation,

  • anuga_validation/okushiri_2005/test_caching_of_set_quantity.py

    r6196 r6232  
    2626# Module imports
    2727from anuga.shallow_water import Domain
     28from anuga.caching import cache
     29from anuga.fit_interpolate.fit import _fit_to_mesh
    2830import project
     31import Numeric as num
    2932
     33internal_verbose = False # Verbose used in set_quantity and passed into fit_to_mesh
     34
     35filename=project.bathymetry_filename
     36alpha=0.02
     37from anuga.config import points_file_block_line_size as max_read_lines
    3038
    3139#-------------------------
    3240# Create Domain from mesh
    3341#-------------------------
    34 domain = Domain(project.mesh_filename, use_cache=True, verbose=True)
     42domain = cache(Domain, (project.mesh_filename, {'verbose': True}), verbose=False)
     43
     44# Clear caching of underlying function                                           
     45args = (filename, )
     46kwargs = {'vertex_coordinates': None,
     47          'triangles': None,
     48          'mesh': domain.mesh,
     49          'point_attributes': None,
     50          'alpha': alpha,
     51          'verbose': internal_verbose,
     52          'acceptable_overshoot': 1.01, # This is the default value in _fit_to_mesh
     53          'mesh_origin': None,
     54          'data_origin': None,
     55          'max_read_lines': max_read_lines,
     56          'attribute_name': None
     57          }
     58
     59
     60cache(_fit_to_mesh,
     61      args,
     62      kwargs,
     63      verbose=False,
     64      dependencies=[filename],
     65      clear=True)
     66
     67# Check that cache is empty     
     68flag = cache(_fit_to_mesh,
     69             args,
     70             kwargs,
     71             verbose=False,
     72             dependencies=[filename],
     73             test=True)
     74assert flag is None
     75
    3576
    3677
    … …  
    3879# Initial Conditions
    3980#-------------------------
    40 print 'Set elevation and cache'
     81#print 'Set elevation and cache'
    4182domain.set_quantity('elevation',
    42                     filename=project.bathymetry_filename,
     83                    filename=filename,
    4384                    alpha=0.02,                   
    44                     verbose=True,
     85                    verbose=internal_verbose,
    4586                    use_cache=True)
     87                   
     88ref = domain.get_quantity('elevation').get_values()                   
    4689
    47 print 'Try to read in via cache'
     90# Check that cache is now present (and correct)
     91flag = cache(_fit_to_mesh,
     92             args,
     93             kwargs,
     94             verbose=False,
     95             dependencies=[filename],
     96             test=True)
     97assert flag is not None
     98res = domain.get_quantity('elevation').get_values()                                       
     99assert num.allclose(res, ref)
     100
     101# Now check this using the high level call
     102#print 'Try to read in via cache'
    48103domain.set_quantity('elevation',
    49                     filename=project.bathymetry_filename,
     104                    filename=filename,
    50105                    alpha=0.02,                   
    51                     verbose=True,
     106                    verbose=internal_verbose,
    52107                    use_cache=True)
     108                   
     109res = domain.get_quantity('elevation').get_values()                                       
     110assert num.allclose(res, ref)
Note: See TracChangeset for help on using the changeset viewer.