Changeset 5854

Timestamp:

Oct 21, 2008, 6:15:06 PM (16 years ago)

Author:

ole

Message:

Work on caching allowing for instances.
Progress made, but Okushiri example still doesn't cache properly.

Location:

anuga_core/source/anuga/caching

Files:

• caching.py (modified) (21 diffs)
• dummy_classes_for_testing.py (modified) (1 diff)
• test_caching.py (modified) (6 diffs)

anuga_core/source/anuga/caching/caching.py

@@ -43 +43 @@
 #
 from os import getenv
+import types
 
 import os
@@ -258 +259 @@
 
   # Create cache directory if needed
-  #
   CD = checkdir(cachedir,verbose)
 
   # Handle the case cache('clear')
-  #
   if type(func) == types.StringType:
     if string.lower(func) == 'clear':
@@ -269 +268 @@
 
   # Handle the case cache(func, 'clear')
-  #
   if type(args) == types.StringType:
     if string.lower(args) == 'clear':
@@ -276 +274 @@
 
   # Force singleton arg into a tuple
-  #
   if type(args) != types.TupleType:
     args = tuple([args])
   
   # Check that kwargs is a dictionary
-  #
   if type(kwargs) != types.DictType:
     raise TypeError    
@@ -288 +284 @@
    
   # Hash arguments (and keyword args) to integer
-  #
   arghash = myhash((args,kwargs))
-
+  
   # Get sizes and timestamps for files listed in dependencies.
   # Force singletons into a tuple.
-  #
-
   if dependencies and type(dependencies) != types.TupleType \
                   and type(dependencies) != types.ListType:
@@ -301 +294 @@
 
   # Extract function name from func object
-  #
   funcname = get_funcname(func)
 
   # Create cache filename
-  #
   FN = funcname+'['+`arghash`+']'  # The symbol '(' does not work under unix
 
@@ -331 +322 @@
   
   # Check if previous computation has been cached
-  #
   if evaluate:
     Retrieved = None  # Force evaluation of func regardless of caching status.
-    reason = 4
+    reason = 5
   else:
     (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
@@ -349 +339 @@
 
       # Remove expired files automatically
-      #
       if options['expire']:
         DeleteOldFiles(CD,verbose)
@@ -355 +344 @@
       # Save args before function is evaluated in case
       # they are modified by function
-      #
       save_args_to_cache(CD,FN,args,kwargs,compression)
 
       # Execute and time function with supplied arguments
-      #
       t0 = time.time()
       T = apply(func,args,kwargs)
@@ -369 +356 @@
 
       # Save results and estimated loading time to cache
-      #
       loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
                                        funcname, dependencies, compression)
@@ -770 +756 @@
               'No cached result', 
               'Dependencies have changed', 
-              'Byte code or arguments have changed',
+              'Arguments have changed',
+              'Bytecode has changed',
               'Recomputation was requested by caller',
               'Cached file was unreadable']              
@@ -803 +790 @@
                      1: No cached result, 
                      2: Dependencies have changed, 
-                     3: Arguments or Bytecode have changed
-                     4: Recomputation was forced
+                     3: Arguments have changed
+                     4: Bytecode has changed
+                     5: Recomputation was forced
+                     6: Unreadable file
     comptime --      Number of seconds it took to computed cachged result
     loadtime --      Number of seconds it took to load cached result
@@ -889 +878 @@
 
   # Check if arguments or bytecode have changed
-  #
   if compare(argsref,args) and compare(kwargsref,kwargs) and \
      (not options['bytecode'] or compare(bytecode,coderef)):
 
     # Arguments and dependencies match. Get cached results
-    #
     T, loadtime, compressed, reason = load_from_cache(CD,FN,compressed)
-    ###if T == None and reason > 0:  #This doesn't work if T is a numeric array
     if reason > 0:
-      return(None,FN,None,reason,None,None,None) #Recompute using same FN 
+      return(None,FN,None,reason,None,None,None) # Recompute using same FN 
 
     Retrieved = 1
@@ -925 +911 @@
     # else:
     #   print 'Match found !'
+    
+    # The real reason is that args or bytecodes have changed.
+    # Not that the recursive seach has found an unused filename
     if not Retrieved:
-      reason = 3     #The real reason is that args or bytecodes have changed.
-                     #Not that the recursive seach has found an unused filename
+      if not compare(bytecode,coderef):
+        reason = 4 # Bytecode has changed
+      else:   
+        reason = 3 # Arguments have changed 
+        
     
   return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
@@ -1125 +1117 @@
   t0 = time.time()
   T, reason = myload(datafile,compressed)
-  #loadtime = round(time.time()-t0,2)
+
   loadtime = time.time()-t0
   datafile.close() 
@@ -1228 +1220 @@
         #print 'ERROR (caching): Could not decompress ', file.name
         #raise Exception
-        reason = 5  #(Unreadable file)
+        reason = 6  # Unreadable file
         return None, reason  
       
@@ -1240 +1232 @@
       except:
         #Catch e.g., file with 0 length or corrupted
-        reason = 5  #(Unreadable file)
+        reason = 6  # Unreadable file
         return None, reason
       
@@ -1285 +1277 @@
       raise MemoryError, msg
     else:  
-      #Compressed pickling      
+      # Compressed pickling      
       TsC = zlib.compress(Ts, comp_level)
       file.write(TsC)
   else:
-      #Uncompressed pickling
+      # Uncompressed pickling
       pickler.dump(T, file, bin)
 
@@ -1328 +1320 @@
 # -----------------------------------------------------------------------------
 
-def myhash(T):
-  """Compute hashed integer from hashable values of tuple T
+def myhash(T, ids=None):
+  """Compute hashed integer from a range of inputs.
+  If T is not hashable being e.g. a tuple T, myhash will recursively 
+  hash the values individually
 
   USAGE:
@@ -1335 +1329 @@
 
   ARGUMENTS:
-    T -- Tuple
-  """
-
-  import types
-
+    T -- Anything
+  """
+
+  from types import TupleType, ListType, DictType, InstanceType  
+  from Numeric import ArrayType
+  
+  if type(T) in [TupleType, ListType, DictType, InstanceType]:  
+  
+      # Keep track of unique id's to protect against infinite recursion
+      if ids is None: ids = []
+
+      # Check if T has already been encountered
+      i = id(T) 
+  
+      if i in ids:
+          # FIXME (Ole): It seems that different objects get the same id
+          # T has been hashed already
+        
+          #print 'T has already been hashed:', T, id(T)
+          return 0
+      else:
+          #print 'Appending', T, id(T)
+          ids.append(i)
+    
+
+  # Start hashing  
+  
+  
   # On some architectures None, False and True gets different hash values
   if T is None:
-    return(-1)
+      return(-1)
   if T is False:
-    return(0)
+      return(0)
   if T is True:
-    return(1)
-
-  # Get hash vals for hashable entries
-  #
-  if type(T) == types.TupleType or type(T) == types.ListType:
-    hvals = []
-    for k in range(len(T)):
-      h = myhash(T[k])
-      hvals.append(h)
-    val = hash(tuple(hvals))
-  elif type(T) == types.DictType:
-    val = dicthash(T)
+      return(1)
+
+  # Get hash values for hashable entries
+  if type(T) in [TupleType, ListType]:
+      hvals = []
+      for t in T:
+          h = myhash(t, ids)
+          hvals.append(h)
+      val = hash(tuple(hvals))
+  elif type(T) == DictType:
+      val = myhash(T.items(), ids)
+  elif type(T) == ArrayType:
+      val = myhash(tuple(T), ids)
+  elif type(T) == InstanceType:
+      val = myhash(T.__dict__, ids)
   else:
-    try:
-      val = hash(T)
-    except:
-      val = 1
       try:
-        import Numeric
-        if type(T) == Numeric.ArrayType:
-          hvals = []        
-          for e in T:
-            h = myhash(e)
-            hvals.append(h)          
-          val = hash(tuple(hvals))
+          val = hash(T)
+      except:
+          val = 1
+
+  return(val)
+
+# -----------------------------------------------------------------------------
+
+def compare(A, B, ids=None):
+    """Safe comparison of general objects
+
+    USAGE:
+      compare(A,B)
+
+    DESCRIPTION:
+      Return 1 if A and B they are identical, 0 otherwise
+    """
+
+    from types import TupleType, ListType, DictType, InstanceType
+    
+    
+    # Keep track of unique id's to protect against infinite recursion
+    if ids is None: ids = {}
+
+
+    # Check if T has already been encountered
+    iA = id(A) 
+    iB = id(B)     
+    
+    if ids.has_key((iA, iB)):
+        # A and B have been compared already
+        #print 'Found', (iA, iB), A, B
+        return ids[(iA, iB)]
+    else:
+        ids[(iA, iB)] = True
+    
+    
+    #print 'Comparing', A, B, (iA, iB)
+    #print ids
+    #raw_input()
+    
+    if type(A) in [TupleType, ListType] and type(B) in [TupleType, ListType]:
+        N = len(A)
+        if len(B) != N: 
+            identical = False
         else:
-          val = 1  #Could implement other Numeric types here 
-      except:    
-        pass
-
-  return(val)
-
-# -----------------------------------------------------------------------------
-
-def dicthash(D):
-  """Compute hashed integer from hashable values of dictionary D
-
-  USAGE:
-    dicthash(D)
-  """
-
-  keys = D.keys()
-
-  # Get hash values for hashable entries
-  #
-  hvals = []
-  for k in range(len(keys)):
-    try:
-      h = myhash(D[keys[k]])
-      hvals.append(h)
-    except:
-      pass
-
-  # Hash obtained values into one value
-  #
-  return(hash(tuple(hvals)))
-
-# -----------------------------------------------------------------------------
-
-def compare(A,B):
-  """Safe comparison of general objects
-
-  USAGE:
-    compare(A,B)
-
-  DESCRIPTION:
-    Return 1 if A and B they are identical, 0 otherwise
-  """
-
-  try:
-    identical = (A == B)
-  except:
-    try:
-      identical = (pickler.dumps(A) == pickler.dumps(B))
-    except:
-      identical = 0
-
-  return(identical)
+            identical = True
+            for i in range(N):
+                if not compare(A[i], B[i], ids): 
+                    identical = False; break
+                
+    elif type(A) == DictType and type(B) == DictType:
+        if len(A) != len(B):
+            identical = False
+        else:    
+            identical = True
+            for key in A.keys():
+                if not B.has_key(key):
+                    identical = False; break
+          
+                if not compare(A[key], B[key], ids): 
+                    identical = False; break      
+    
+    elif type(A) == type(B) == types.InstanceType:    
+        # Take care of special case where elements are instances            
+        # Base comparison on attributes 
+                
+        a = A.__dict__                 
+        b = B.__dict__                             
+                
+        identical = compare(a, b, ids)                
+
+    
+        
+    else:       
+        # Fall back to general code
+        try:
+            identical = (A == B)
+        except:
+            try:
+                identical = (pickler.dumps(A) == pickler.dumps(B))
+            except:
+                identical = 0
+
+    # Record result of comparison and return            
+    ids[(iA, iB)] = identical
+    
+    return(identical)
 
 # -----------------------------------------------------------------------------
@@ -2339 +2383 @@
       argstr = argstr + "'"+str(args)+"'"
     else:
-      #Truncate large Numeric arrays before using str()
+      # Truncate large Numeric arrays before using str()
       import Numeric
       if type(args) == Numeric.ArrayType:

anuga_core/source/anuga/caching/dummy_classes_for_testing.py

@@ -10 +10 @@
     def copy(self):
         return Dummy(self.value, self.another)
+        
+    def __repr__(self):
+        return str(self.value) + ', ' + str(self.another)
     
 

anuga_core/source/anuga/caching/test_caching.py

@@ -35 +35 @@
   return A.value+B.value, A.another+B.another 
   
-  
+
+def f_generic(A):
+  return A 
   
 def clear_and_create_cache(Dummy, verbose=False):
@@ -114 +116 @@
             
 
+            
     def test_caching_of_numeric_arrays(self):
         """test_caching_of_numeric_arrays
@@ -190 +193 @@
 
             # Retrieve
+            #T2 = cache(f_object, (A1, B1), 
+            #           compression=comp, verbose=verbose)                        
+                       
+            # Retrieve
             T2 = cache(f_object, (A1, B1), 
                        compression=comp, test=1, verbose=verbose) 
@@ -205 +212 @@
             
 
-                        
+    def test_caching_of_circular_structures(self):
+        """test_caching_of_circular_structures
+        
+        Test that Caching doesn't recurse infinitely in case of
+        circular or self-referencing structures
+        """
+        
+        verbose = False
+        
+        # Create input argument
+        A = Dummy(5, 7)
+        B = {'x': 10, 'A': A}
+        C = [B, 15]
+        A.value = C # Make it circular
+        
+
+        # Test caching
+        comprange = 2
+        for comp in range(comprange):
+  
+            # Evaluate and store
+            T1 = cache(f_generic, A, evaluate=1,
+                       compression=comp, verbose=verbose)
+
+            # Retrieve
+            T2 = cache(f_generic, A, 
+                       compression=comp, test=1, verbose=verbose) 
+                       
+            # Check for presence of cached result 
+            msg = 'Cached object was not found'            
+            assert T2 is not None, msg
+
+            # Reference result
+            T3 = f_generic(A) # Compute without caching
+
+            
+            msg = 'Cached result does not match computed result' 
+            assert str(T1) == str(T2), msg
+            assert str(T2) == str(T3), msg
+            
+                                    
+            
+            
+            
+
+    def XXtest_caching_of_simple_circular_structures(self):
+    
+        # FIXME (Ole): This one recurses infinitly on
+        # arg strings. 
+        
+        """test_caching_of_circular_structures
+        
+        Test that Caching doesn't recurse infinitely in case of
+        circular or self-referencing structures
+        """
+        
+        verbose = True
+        
+        # Create input argument
+        A = {'x': 10, 'B': None}
+        B = [A, 15]
+        A['B'] = B # Make it circular
+        
+        print A
+
+        # Test caching
+        comprange = 2
+        for comp in range(comprange):
+  
+            # Evaluate and store
+            T1 = cache(f_generic, A, evaluate=1,
+                       compression=comp, verbose=verbose)
+                       
+            import sys; sys.exit() 
+                       
+
+            # Retrieve
+            T2 = cache(f_generic, A, 
+                       compression=comp, test=1, verbose=verbose) 
+                       
+            # Check for presence of cached result 
+            msg = 'Cached object was not found'            
+            assert T2 is not None, msg
+
+            # Reference result
+            T3 = f_generic(A) # Compute without caching
+
+
+            assert T1 == T2, 'Cached result does not match computed result'
+            assert T2 == T3, 'Cached result does not match computed result'
+            
+                                    
             
             
@@ -429 +527 @@
       This test shows how instances can't be effectively cached.
       myhash uses hash which uses id which uses the memory address. 
+      
+      This will be a NIL problem once caching can handle instances with different id's and 
+      identical content.
+      
+      The test is disabled.
       """
+      
+
+      
       verbose = True
       #verbose = False
@@ -526 +632 @@
           
 # Cache created for use with 'test_objects_are_created_memory'
-initial_addr = `Dummy_memorytest`
-clear_and_create_cache(Dummy_memorytest, verbose=False)
-  
-      
-
-
-
-     
-        
+#initial_addr = `Dummy_memorytest`
+#clear_and_create_cache(Dummy_memorytest, verbose=False)
+  
 
 
 #-------------------------------------------------------------
 if __name__ == "__main__":
-    suite = unittest.makeSuite(Test_Caching,'test')
+    suite = unittest.makeSuite(Test_Caching, 'test')
     runner = unittest.TextTestRunner()
     runner.run(suite)