Changeset 5855


Ignore:
Timestamp:
Oct 22, 2008, 3:07:29 PM (16 years ago)
Author:
ole
Message:

Much more work on caching and fitting.
Caching of set_quantity as per Okushiri example now works again.

Moreover, caching can now handle instances and ever circular references.

This should address ticket:244 and ticket:302

Location:
anuga_core/source/anuga
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • anuga_core/source/anuga/abstract_2d_finite_volumes/quantity.py

    r5847 r5855  
    856856            raise msg
    857857
    858         if True: 
     858        if False: 
     859            # FIXME (Ole)
     860            # Use mesh as defined by domain
     861            # This causes problems for caching
     862            # due to quantities changing, but
     863            # it would work if we only passed in
     864            # the appropriate Mesh object.
    859865            vertex_attributes = fit_to_mesh(filename,
    860866                                            mesh=self.domain, 
    … …  
    865871                                            max_read_lines=max_read_lines)
    866872        else:
    867        
     873            # This variant will cause Mesh object to be recreated
     874            # in fit_to_mesh thus doubling up on the neighbour structure
    868875            coordinates = self.domain.get_nodes(absolute=True)
    869876            triangles = self.domain.triangles      #FIXME
    870877            vertex_attributes = fit_to_mesh(filename,
    871878                                            coordinates, triangles,
     879                                            mesh=None,                                           
    872880                                            alpha=alpha,
    873881                                            attribute_name=attribute_name,

  • anuga_core/source/anuga/caching/caching.py

    r5854 r5855  
    281281    raise TypeError   
    282282   
    283   #print 'hashing' #FIXME: make faster hashing function
    284    
    285283  # Hash arguments (and keyword args) to integer
    286   arghash = myhash((args,kwargs))
     284  arghash = myhash((args, kwargs))
    287285 
    288286  # Get sizes and timestamps for files listed in dependencies.
    … …  
    326324    reason = 5
    327325  else:
    328     (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
    329       CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
    330                   dependencies)
     326    T, FN, Retrieved, reason, comptime, loadtime, compressed = \
     327        CacheLookup(CD, FN, func,
     328                    args, kwargs,
     329                    deps,
     330                    verbose,
     331                    compression,
     332                    dependencies)
    331333
    332334  if not Retrieved:
    … …  
    838840  # Retrieve arguments and adm. info
    839841  #
    840   R, reason = myload(argsfile,compressed)  # The original arguments
     842  R, reason = myload(argsfile, compressed)  # The original arguments
    841843  argsfile.close()
    842844   
    843   ##if R == None and reason > 0:
    844845  if reason > 0:
    845     return(None,FN,None,reason,None,None,None) #Recompute using same filename
     846      # Recompute using same filename   
     847      return(None, FN, None, reason, None, None, None)
    846848  else:   
    847     (argsref, kwargsref) = R
    848 
    849   R, reason = myload(admfile,compressed)
     849      (argsref, kwargsref) = R
     850
     851  R, reason = myload(admfile, compressed)
    850852  admfile.close() 
    851   ##if R == None and reason > 0:
     853
    852854  if reason > 0:
    853855    return(None,FN,None,reason,None,None,None) #Recompute using same filename
    … …  
    861863  # Check if dependencies have changed
    862864  #
    863   if dependencies and not compare(depsref,deps):
     865  if dependencies and not compare(depsref, deps):
    864866    if verbose:
    865867      print 'MESSAGE (caching.py): Dependencies', dependencies, \
    … …  
    867869    # Don't use cached file - recompute
    868870    reason = 2
    869     return(None,FN,None,reason,None,None,None)
     871    return(None, FN, None, reason, None, None, None)
    870872
    871873  # Get bytecode from func
    … …  
    873875  bytecode = get_bytecode(func)
    874876
    875   #print compare(argsref,args),
     877  #print 'Diags'
     878  #print argsref
     879  #print args
     880  #print compare(argsref,args),   
     881 
    876882  #print compare(kwargsref,kwargs),
    877883  #print compare(bytecode,coderef)
    878884
    879885  # Check if arguments or bytecode have changed
    880   if compare(argsref,args) and compare(kwargsref,kwargs) and \
    881      (not options['bytecode'] or compare(bytecode,coderef)):
     886  if compare(argsref, args) and compare(kwargsref, kwargs) and \
     887     (not options['bytecode'] or compare(bytecode, coderef)):
    882888
    883889    # Arguments and dependencies match. Get cached results
    884     T, loadtime, compressed, reason = load_from_cache(CD,FN,compressed)
     890    T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
    885891    if reason > 0:
    886       return(None,FN,None,reason,None,None,None) # Recompute using same FN
     892        # Recompute using same FN     
     893        return(None, FN, None, reason, None, None, None)
    887894
    888895    Retrieved = 1
    … …  
    902909    # until either a matching or an unused filename is found.
    903910    #
    904     (T,FN,Retrieved,reason,comptime,loadtime,compressed) = \
    905        CacheLookup(CD,FN+'x',func,args,kwargs,deps,verbose,compression, \
    906                    dependencies)
     911    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
     912        CacheLookup(CD, FN+'x', func, args, kwargs, deps,
     913                    verbose, compression, dependencies)
    907914
    908915    # DEBUGGING
    … …  
    915922    # Not that the recursive seach has found an unused filename
    916923    if not Retrieved:
    917       if not compare(bytecode,coderef):
     924      if not compare(bytecode, coderef):
    918925        reason = 4 # Bytecode has changed
    919926      else:   
    … …  
    12251232     
    12261233      del RsC  # Free up some space
    1227       R   = pickler.loads(Rs)
     1234      R = pickler.loads(Rs)
    12281235    else:
    12291236      try:
    … …  
    13201327# -----------------------------------------------------------------------------
    13211328
     1329
     1330def Xmyhash(T, ids=None):
     1331    import pickle as pickler # Use non-C version here
     1332    return hash(pickler.dumps(T,0))
     1333
     1334   
    13221335def myhash(T, ids=None):
    13231336  """Compute hashed integer from a range of inputs.
    … …  
    13331346
    13341347  from types import TupleType, ListType, DictType, InstanceType 
    1335   from Numeric import ArrayType
    1336  
    1337   if type(T) in [TupleType, ListType, DictType, InstanceType]: 
    1338  
     1348  from Numeric import ArrayType, average
     1349 
     1350  #if type(T) in [TupleType, ListType, DictType, InstanceType]: 
     1351  #if type(T) in [ListType, DictType, InstanceType]: 
     1352  if type(T) == InstanceType: 
     1353  #if False:
    13391354      # Keep track of unique id's to protect against infinite recursion
    13401355      if ids is None: ids = []
    … …  
    13441359 
    13451360      if i in ids:
     1361          # T has been hashed already     
     1362     
    13461363          # FIXME (Ole): It seems that different objects get the same id
    1347           # T has been hashed already
    1348        
    1349           #print 'T has already been hashed:', T, id(T)
     1364          # For example id(D.items()) is the same as id(D.values()).
     1365
     1366          #print 'T has already been hashed:', mkargstr(T, 132), id(T), type(T)
    13501367          return 0
    13511368      else:
    … …  
    13731390      val = hash(tuple(hvals))
    13741391  elif type(T) == DictType:
    1375       val = myhash(T.items(), ids)
     1392      # Make dictionary ordering unique 
     1393      I = T.items()
     1394      I.sort()   
     1395      val = myhash(I, ids)
    13761396  elif type(T) == ArrayType:
    1377       val = myhash(tuple(T), ids)
     1397      val = hash(average(T.flat))  # Use mean value for efficiency
     1398      #val = myhash(tuple(T), ids)
    13781399  elif type(T) == InstanceType:
    13791400      val = myhash(T.__dict__, ids)
     1401     
     1402      #print 'Hashed instance:'
     1403      #print T.__dict__.keys()
     1404      #print mkargstr(T.__dict__, 132)
     1405      #print 'with value', val
     1406
    13801407  else:
    13811408      try:
    … …  
    13881415# -----------------------------------------------------------------------------
    13891416
     1417
    13901418def compare(A, B, ids=None):
     1419    """Safe comparison of general objects
     1420
     1421    USAGE:
     1422      compare(A,B)
     1423
     1424    DESCRIPTION:
     1425      Return 1 if A and B they are identical, 0 otherwise
     1426    """
     1427
     1428    if A == B:
     1429        identical = True
     1430    else:
     1431        # Use pickle to compare data
     1432        # The native pickler must be used
     1433        # since the faster cPickle does not
     1434        # guarantee a unique translation
     1435       
     1436        import pickle as pickler # Use non-C version here
     1437        identical = (pickler.dumps(A,0) == pickler.dumps(B,0))
     1438
     1439    return(identical)
     1440
     1441
     1442def Xcompare(A, B, ids=None):
    13911443    """Safe comparison of general objects
    13921444
    … …  
    14591511            identical = (A == B)
    14601512        except:
     1513            import pickle # Use non-C version here
    14611514            try:
    1462                 identical = (pickler.dumps(A) == pickler.dumps(B))
     1515                identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
    14631516            except:
    14641517                identical = 0

  • anuga_core/source/anuga/caching/dummy_classes_for_testing.py

    r5854 r5855  
    88        self.another = another
    99   
    10     def copy(self):
    11         return Dummy(self.value, self.another)
    12        
    1310    def __repr__(self):
    1411        return str(self.value) + ', ' + str(self.another)

  • anuga_core/source/anuga/caching/test_caching.py

    r5854 r5855  
    33from Numeric import arange, array
    44
     5from copy import deepcopy
     6       
    57from anuga.caching import *
    68from anuga.caching.dummy_classes_for_testing import Dummy, Dummy_memorytest
    … …  
    161163           
    162164
     165    def test_caching_of_dictionaries(self):
     166        """test_caching_of_dictionaries
     167       
     168        Real example from ANUGA that caused some
     169        hashing problems
     170        """
     171   
     172
     173        verbose = False #True
     174       
     175        D = {'point_attributes': None,
     176             'use_cache': True,
     177             'vertex_coordinates': None,
     178             'verbose': True,
     179             'max_read_lines': 500,
     180             'acceptable_overshoot': 1.01,
     181             'mesh': None,
     182             'data_origin': None,
     183             'alpha': 0.02,
     184             'mesh_origin': None,
     185             'attribute_name': None,
     186             'triangles': None}         
     187       
     188        DD = deepcopy(D) # Mangles the dictionary ordering
     189       
     190        assert myhash(DD) == myhash(D)
     191
     192        # Also test caching now that we are at it
     193        comprange = 2
     194        for comp in range(comprange):
     195 
     196            # Evaluate and store using D
     197            T1 = cache(f_generic, D, evaluate=1,
     198                       compression=comp, verbose=verbose)
     199
     200            # Retrieve using copy (DD)
     201            T2 = cache(f_generic, DD,
     202                       compression=comp, test=1, verbose=verbose)
     203                       
     204            # Check for presence of cached result
     205            msg = 'Cached object was not found'           
     206            assert T2 is not None, msg
     207
     208            # Reference result
     209            T3 = f_generic(D) # Compute without caching
     210
     211           
     212            msg = 'Cached result does not match computed result'
     213           
     214            # Compare dictionaries
     215            for key in T1:
     216                assert T1[key] == T2[key]
     217                assert T2[key] == T3[key]               
     218               
     219           
     220           
     221           
    163222
    164223    def test_caching_of_objects(self):
    … …  
    176235        B0 = Dummy(2.2, -5)
    177236       
    178         A1 = A0.copy()
    179         B1 = B0.copy()
     237        A0.new_attribute = 'x' 
     238        B0.new_attribute = 'x'       
     239       
     240        A1 = deepcopy(A0)
     241        B1 = deepcopy(B0)       
    180242       
    181243        # Check that their ids are different
    … …  
    193255
    194256            # Retrieve
    195             #T2 = cache(f_object, (A1, B1),
    196             #           compression=comp, verbose=verbose)                       
    197                        
    198             # Retrieve
    199257            T2 = cache(f_object, (A1, B1),
    200                        compression=comp, test=1, verbose=verbose)
     258                       compression=comp,
     259                       test=1,
     260                       verbose=verbose)
    201261                       
    202262            # Check for presence of cached result
    … …  
    226286        C = [B, 15]
    227287        A.value = C # Make it circular
    228        
     288        A.x = [1,2,C,5,A] # More circular and self referential
     289       
     290        AA = deepcopy(A)
    229291
    230292        # Test caching
    … …  
    233295 
    234296            # Evaluate and store
    235             T1 = cache(f_generic, A, evaluate=1,
     297            T1 = cache(f_generic, A,
     298                       evaluate=1,
    236299                       compression=comp, verbose=verbose)
    237300
    238301            # Retrieve
    239             T2 = cache(f_generic, A,
    240                        compression=comp, test=1, verbose=verbose)
     302            T2 = cache(f_generic, AA,
     303                       compression=comp,
     304                       test=1, verbose=verbose)
    241305                       
    242306            # Check for presence of cached result
    … …  
    251315            assert str(T1) == str(T2), msg
    252316            assert str(T2) == str(T3), msg
    253            
    254317                                   
    255318           
    256            
    257            
     319    def Xtest_caching_of_complex_circular_structure(self):
     320        """test_caching_of_complex_circular_structure
     321       
     322        Test that Caching can handle a realistic
     323        complex structure. This one is inspired by
     324        ANUGA's Okushiri example, although reduced in size.
     325        """
     326       
     327        pass
     328
     329       
     330    def test_uniqueness_of_hash_values(self):
     331        """test_uniqueness_of_hash_values(self):
     332       
     333        Test that Caching can handle a realistic
     334        complex structure by hashing it consistently and
     335        uniquely.
     336        """
     337       
     338        verbose = False
     339       
     340        # Create input argument
     341        A = Dummy(5, 7)
     342        B = {'x': 10, 'A': A}
     343        C = [B, array([1.2, 3, 5, 0.1])]
     344        A.value = C # Make it circular
     345
     346        # Create identical but separate object   
     347        AA = Dummy(None, None)
     348        BB = {'A': AA, 'x': 10}
     349        CC = [BB, array([1.200, 3.000, 5.00, 1.0/10])]
     350        AA.value = CC # Make it circular
     351        AA.another = 3+4       
     352       
     353       
     354        assert myhash(A) == myhash(AA)     
     355           
     356           
     357       
     358        # Also test caching now that we are at it
     359        comprange = 2
     360        for comp in range(comprange):
     361 
     362            # Evaluate and store using A
     363            T1 = cache(f_generic, A, evaluate=1,
     364                       compression=comp, verbose=verbose)
     365
     366            # Retrieve using copy (AA)
     367            T2 = cache(f_generic, AA,
     368                       compression=comp, test=1, verbose=verbose)
     369                       
     370            # Check for presence of cached result
     371            msg = 'Cached object was not found'           
     372            assert T2 is not None, msg
     373
     374            # Reference result
     375            T3 = f_generic(A) # Compute without caching
     376
     377           
     378            msg = 'Cached result does not match computed result'
     379            assert str(T1) == str(T2), msg
     380            assert str(T2) == str(T3), msg
     381           
     382           
    258383
    259384    def XXtest_caching_of_simple_circular_structures(self):
    … …  
    338463        """Test 'test' function when cache is present
    339464        """
    340         N = 5000  #Make N fairly small here
     465       
     466        verbose = False
     467       
     468        N = 5 
    341469
    342470        a = [1,2]
    … …  
    347475       
    348476
    349         T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1)
    350        
    351         T4 = cache(f,(a,b,c,N), {'x':x, 'y':y}, test=1)
    352         assert T1 == T4, "Option 'test' when cache file present failed"     
     477        T1 = cache(f, (a,b,c,N), {'x':x, 'y':y},
     478                   evaluate=1,
     479                   verbose=verbose)
     480       
     481        T2 = cache(f, (a,b,c,N), {'x':x, 'y':y},
     482                   test=1,
     483                   verbose=verbose)
     484                   
     485                   
     486        # Check for presence of cached result
     487        msg = 'Different objects with same attributes were not recognised'
     488        assert T2 is not None, msg                   
     489                   
     490        assert T1 == T2, "Option 'test' when cache file present failed"     
    353491
    354492
    … …  
    595733
    596734
    597     # NOTE (Ole): This test has been commented out because, although the test will pass
     735    # NOTE (Ole): This test has been commented out because,
     736    #             although the test will pass (not anymore!)
    598737    #             inside the caching dir and also at the anuga_core level,
    599738    #             it won't pass at the anuga_core/source/anuga level.
    600739    # It may have to do with the comments above.
    601740    #
    602     # But this is probably not so important, really
     741    # But this is a very nice test to run occasionally within the caching
     742    # area
    603743    def Xtest_objects_are_created_memory(self):
    604744      """
    … …  
    611751      """
    612752
    613       verbose = False
     753      verbose = True #False
    614754
    615755      # Redefine class Dummy_memorytest

  • anuga_core/source/anuga/fit_interpolate/fit.py

    r5852 r5855  
    490490            dep = None
    491491
     492           
     493        #from caching import myhash
     494        #import copy
     495        #print kwargs
     496        #print 'hashing:'
     497        #print 'args', myhash( (args, kwargs) )
     498        #print 'again', myhash( copy.deepcopy( (args, kwargs)) )       
     499       
     500        #print 'mesh hash', myhash( kwargs['mesh'] )       
     501       
     502        #print '-------------------------'
     503        #print 'vertices hash', myhash( kwargs['mesh'].nodes )
     504        #print 'triangles hash', myhash( kwargs['mesh'].triangles )
     505        #print '-------------------------'       
     506       
     507        #for key in mesh.__dict__:
     508        #    print key, myhash(mesh.__dict__[key])
     509       
     510        #for key in mesh.quantities.keys():
     511        #    print key, myhash(mesh.quantities[key])
     512       
     513        #import sys; sys.exit()
     514           
    492515        return cache(_fit_to_mesh,
    493516                     args, kwargs,

  • anuga_core/source/anuga/fit_interpolate/interpolate.py

    r5791 r5855  
    109109   
    110110    from anuga.caching import cache
     111   
    111112
    112113    # Create interpolation object with matrix
    113     args = (vertex_coordinates, triangles)
     114    args = (ensure_numeric(vertex_coordinates, Float),
     115            ensure_numeric(triangles))
    114116    kwargs = {'mesh_origin': mesh_origin,
    115117              'max_vertices_per_cell': max_vertices_per_cell,
Note: See TracChangeset for help on using the changeset viewer.