source: anuga_core/source/anuga/caching/caching.py @ 6232

Last change on this file since 6232 was 6232, checked in by ole, 15 years ago

Improved test_caching_of_set_quantity - it still needs to be moved to validation suite.
File size: 70.8 KB
Line 
1# =============================================================================
2# caching.py - Supervised caching of function results.
3# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4# Australian National University (1999-2003)
5# Geoscience Australia (2003-present)
6#
7#    This program is free software; you can redistribute it and/or modify
8#    it under the terms of the GNU General Public License as published by
9#    the Free Software Foundation; either version 2 of the License, or
10#    (at your option) any later version.
11#
12#    This program is distributed in the hope that it will be useful,
13#    but WITHOUT ANY WARRANTY; without even the implied warranty of
14#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15#    GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16#    for more details.
17#
18#    You should have received a copy of the GNU General Public License
19#    along with this program; if not, write to the Free Software
20#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
21#
22#
23# Contact address: Ole.Nielsen@ga.gov.au
24#
25# Version 1.5.6 February 2002
26# =============================================================================
27 
28"""Module caching.py - Supervised caching of function results.
29
30Public functions:
31
32cache(func,args) -- Cache values returned from func given args.
33cachestat() --      Reports statistics about cache hits and time saved.
34test() --       Conducts a basic test of the caching functionality.
35
36See doc strings of individual functions for detailed documentation.
37"""
38
39# -----------------------------------------------------------------------------
40# Initialisation code
41
42# Determine platform
43#
44from os import getenv
45import types
46
47import os
48if os.name in ['nt', 'dos', 'win32', 'what else?']:
49  unix = False
50else:
51  unix = True
52
53import Numeric as num
54
55
56cache_dir = '.python_cache'
57# Make default caching directory name
58#
59if unix:
60  homedir = getenv('INUNDATIONHOME')
61  if homedir is None:
62    homedir = '~'
63  else:
64    homedir = homedir + os.sep + '.cache'
65    # Since homedir will be a group area, individually label the caches
66    user = getenv('LOGNAME')
67    if user is not None:
68      cache_dir += '_' + user
69   
70  CR = '\n'
71else:
72  homedir = 'c:'
73  CR = '\r\n'  #FIXME: Not tested under windows
74 
75cachedir = homedir + os.sep + cache_dir + os.sep
76
77# -----------------------------------------------------------------------------
78# Options directory with default values - to be set by user
79#
80
81options = { 
82  'cachedir': cachedir,  # Default cache directory
83  'maxfiles': 1000000,   # Maximum number of cached files
84  'savestat': True,      # Log caching info to stats file
85  'verbose': True,       # Write messages to standard output
86  'bin': True,           # Use binary format (more efficient)
87  'compression': True,   # Use zlib compression
88  'bytecode': True,      # Recompute if bytecode has changed
89  'expire': False        # Automatically remove files that have been accessed
90                         # least recently
91}
92
93# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
94
95def set_option(key, value):
96  """Function to set values in the options directory.
97
98  USAGE:
99    set_option(key, value)
100
101  ARGUMENTS:
102    key --   Key in options dictionary. (Required)
103    value -- New value for key. (Required)
104
105  DESCRIPTION:
106    Function to set values in the options directory.
107    Raises an exception if key is not in options.
108  """
109
110  if options.has_key(key):
111    options[key] = value
112  else:
113    raise KeyError(key)  # Key not found, raise an exception
114
115# -----------------------------------------------------------------------------
116# Function cache - the main routine
117
118def cache(func, 
119          args=(), 
120          kwargs={}, 
121          dependencies=None, 
122          cachedir=None,
123          verbose=None, 
124          compression=None, 
125          evaluate=False, 
126          test=False, 
127          clear=False,
128          return_filename=False):
129  """Supervised caching of function results. Also known as memoization.
130
131  USAGE:
132    result = cache(func, args, kwargs, dependencies, cachedir, verbose,
133                   compression, evaluate, test, return_filename)
134
135  ARGUMENTS:
136    func --            Function object (Required)
137    args --            Arguments to func (Default: ())
138    kwargs --          Keyword arguments to func (Default: {})   
139    dependencies --    Filenames that func depends on (Default: None)
140    cachedir --        Directory for cache files (Default: options['cachedir'])
141    verbose --         Flag verbose output to stdout
142                       (Default: options['verbose'])
143    compression --     Flag zlib compression (Default: options['compression'])
144    evaluate --        Flag forced evaluation of func (Default: False)
145    test --            Flag test for cached results (Default: False)
146    clear --           Flag delete cached results (Default: False)   
147    return_filename -- Flag return of cache filename (Default: False)   
148
149  DESCRIPTION:
150    A Python function call of the form
151
152      result = func(arg1,...,argn)
153
154    can be replaced by
155
156      from caching import cache
157      result = cache(func,(arg1,...,argn))
158
159  The latter form returns the same output as the former but reuses cached
160  results if the function has been computed previously in the same context.
161  'result' and the arguments can be simple types, tuples, list, dictionaries or
162  objects, but not unhashable types such as functions or open file objects.
163  The function 'func' may be a member function of an object or a module.
164
165  This type of caching is particularly useful for computationally intensive
166  functions with few frequently used combinations of input arguments. Note that
167  if the inputs or output are very large caching might not save time because
168  disc access may dominate the execution time.
169
170  If the function definition changes after a result has been cached it will be
171  detected by examining the functions bytecode (co_code, co_consts,
172  func_defaults, co_argcount) and it will be recomputed.
173
174  LIMITATIONS:
175    1 Caching uses the apply function and will work with anything that can be
176      pickled, so any limitation in apply or pickle extends to caching.
177    2 A function to be cached should not depend on global variables
178      as wrong results may occur if globals are changed after a result has
179      been cached.
180
181  -----------------------------------------------------------------------------
182  Additional functionality:
183
184  Keyword args
185    Keyword arguments (kwargs) can be added as a dictionary of keyword: value
186    pairs, following the syntax of the built-in function apply().
187    A Python function call of the form
188   
189      result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)   
190
191    is then cached as follows
192
193      from caching import cache
194      result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
195   
196    The default value of kwargs is {} 
197
198  Explicit dependencies:
199    The call
200      cache(func,(arg1,...,argn), dependencies = <list of filenames>)
201    Checks the size, creation time and modification time of each listed file.
202    If any file has changed the function is recomputed and the results stored
203    again.
204
205  Specify caching directory:
206    The call
207      cache(func,(arg1,...,argn), cachedir = <cachedir>)
208    designates <cachedir> where cached data are stored. Use ~ to indicate users
209    home directory - not $HOME. The default is ~/.python_cache on a UNIX
210    platform and c:/.python_cache on a Win platform.
211
212  Silent operation:
213    The call
214      cache(func,(arg1,...,argn), verbose=False)
215    suppresses messages to standard output.
216
217  Compression:
218    The call
219      cache(func,(arg1,...,argn), compression=False)
220    disables compression. (Default: compression=True). If the requested compressed
221    or uncompressed file is not there, it'll try the other version.
222
223  Forced evaluation:
224    The call
225      cache(func,(arg1,...,argn), evaluate=True)
226    forces the function to evaluate even though cached data may exist.
227
228  Testing for presence of cached result:
229    The call
230      cache(func,(arg1,...,argn), test=True)
231    retrieves cached result if it exists, otherwise None. The function will not
232    be evaluated. If both evaluate and test are switched on, evaluate takes
233    precedence.
234    ??NOTE: In case of hash collisions, this may return the wrong result as
235    ??it only checks if *a* cached result is present.
236    # I think this was due to the bytecode option being False for some reason. (23/1/2009).
237   
238  Obtain cache filenames:
239    The call   
240      cache(func,(arg1,...,argn), return_filename=True)
241    returns the hashed base filename under which this function and its
242    arguments would be cached
243
244  Clearing cached results:
245    The call
246      cache(func,'clear')
247    clears all cached data for 'func' and
248      cache('clear')
249    clears all cached data.
250 
251    NOTE: The string 'clear' can be passed an *argument* to func using
252      cache(func,('clear',)) or cache(func,tuple(['clear'])).
253
254    New form of clear:
255      cache(func,(arg1,...,argn), clear=True)
256    clears cached data for particular combination func and args
257     
258  """
259
260  # Imports and input checks
261  #
262  import types, time, string
263
264  if not cachedir:
265    cachedir = options['cachedir']
266
267  if verbose == None:  # Do NOT write 'if not verbose:', it could be zero.
268    verbose = options['verbose']
269
270  if compression == None:  # Do NOT write 'if not compression:',
271                           # it could be zero.
272    compression = options['compression']
273
274  # Create cache directory if needed
275  CD = checkdir(cachedir,verbose)
276
277  # Handle the case cache('clear')
278  if type(func) == types.StringType:
279    if string.lower(func) == 'clear':
280      clear_cache(CD,verbose=verbose)
281      return
282
283  # Handle the case cache(func, 'clear')
284  if type(args) == types.StringType:
285    if string.lower(args) == 'clear':
286      clear_cache(CD,func,verbose=verbose)
287      return
288
289  # Force singleton arg into a tuple
290  if type(args) != types.TupleType:
291    args = tuple([args])
292 
293  # Check that kwargs is a dictionary
294  if type(kwargs) != types.DictType:
295    raise TypeError   
296   
297  # Hash arguments (and keyword args) to integer
298  arghash = myhash((args, kwargs))
299 
300  # Get sizes and timestamps for files listed in dependencies.
301  # Force singletons into a tuple.
302  if dependencies and type(dependencies) != types.TupleType \
303                  and type(dependencies) != types.ListType:
304    dependencies = tuple([dependencies])
305  deps = get_depstats(dependencies)
306
307  # Extract function name from func object
308  funcname = get_funcname(func)
309
310  # Create cache filename
311  FN = funcname+'['+`arghash`+']'  # The symbol '(' does not work under unix
312
313  if return_filename:
314    return(FN)
315
316  if clear:
317    for file_type in file_types:
318      file_name = CD+FN+'_'+file_type
319      for fn in [file_name, file_name + '.z']:
320        if os.access(fn, os.F_OK):             
321          if unix:
322            os.remove(fn)
323          else:
324            # FIXME: os.remove doesn't work under windows       
325            os.system('del '+fn)
326          if verbose is True:
327            print 'MESSAGE (caching): File %s deleted' %fn
328        ##else:
329        ##  print '%s was not accessed' %fn
330    return None
331
332
333  #-------------------------------------------------------------------       
334 
335  # Check if previous computation has been cached
336  if evaluate is True:
337    Retrieved = None  # Force evaluation of func regardless of caching status.
338    reason = 5
339  else:
340    T, FN, Retrieved, reason, comptime, loadtime, compressed = \
341        CacheLookup(CD, FN, func, 
342                    args, kwargs, 
343                    deps, 
344                    verbose, 
345                    compression,
346                    dependencies)
347
348  if not Retrieved:
349    if test:  # Do not attempt to evaluate function
350      T = None
351    else:  # Evaluate function and save to cache
352      if verbose is True:
353       
354        msg1(funcname, args, kwargs,reason)
355
356      # Remove expired files automatically
357      if options['expire']:
358        DeleteOldFiles(CD,verbose)
359       
360      # Save args before function is evaluated in case
361      # they are modified by function
362      save_args_to_cache(CD,FN,args,kwargs,compression)
363
364      # Execute and time function with supplied arguments
365      t0 = time.time()
366      T = apply(func,args,kwargs)
367      #comptime = round(time.time()-t0)
368      comptime = time.time()-t0
369
370      if verbose is True:
371        msg2(funcname,args,kwargs,comptime,reason)
372
373      # Save results and estimated loading time to cache
374      loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
375                                       funcname, dependencies, compression)
376      if verbose is True:
377        msg3(loadtime, CD, FN, deps, compression)
378      compressed = compression
379
380  if options['savestat'] and (not test or Retrieved):
381  ##if options['savestat']:
382    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
383
384  return(T)  # Return results in all cases
385
386# -----------------------------------------------------------------------------
387
388def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
389  """Generate statistics of caching efficiency.
390
391  USAGE:
392    cachestat(sortidx, period, showuser, cachedir)
393
394  ARGUMENTS:
395    sortidx --  Index of field by which lists are (default: 4)
396                Legal values are
397                 0: 'Name'
398                 1: 'Hits'
399                 2: 'CPU'
400                 3: 'Time Saved'
401                 4: 'Gain(%)'
402                 5: 'Size'
403    period --   If set to -1 all available caching history is used.
404                If set 0 only the current month is used (default -1).
405    showuser -- Flag for additional table showing user statistics
406                (default: None).
407    cachedir -- Directory for cache files (default: options['cachedir']).
408
409  DESCRIPTION:
410    Logged caching statistics is converted into summaries of the form
411    --------------------------------------------------------------------------
412    Function Name   Hits   Exec(s)  Cache(s)  Saved(s)   Gain(%)      Size
413    --------------------------------------------------------------------------
414  """
415
416  __cachestat(sortidx, period, showuser, cachedir)
417  return
418
419# -----------------------------------------------------------------------------
420
421# Has mostly been moved to proper unit test.
422# What remains here includes example of the
423# cache statistics form.
424def test(cachedir=None, verbose=False, compression=None):
425  """Test the functionality of caching.
426
427  USAGE:
428    test(verbose)
429
430  ARGUMENTS:
431    verbose --     Flag whether caching will output its statistics (default=False)
432    cachedir --    Directory for cache files (Default: options['cachedir'])
433    compression -- Flag zlib compression (Default: options['compression'])
434  """
435   
436  import string, time
437
438  # Initialise
439  #
440  #import caching
441  #reload(caching)
442
443  if not cachedir:
444    cachedir = options['cachedir']
445
446  if verbose is None:  # Do NOT write 'if not verbose:', it could be zero.
447    verbose = options['verbose']
448 
449  if compression == None:  # Do NOT write 'if not compression:',
450                           # it could be zero.
451    compression = options['compression']
452  else:
453    try:
454      set_option('compression', compression)
455    except:
456      test_error('Set option failed')     
457
458  try:
459    import zlib
460  except:
461    print
462    print '*** Could not find zlib, default to no-compression      ***'
463    print '*** Installing zlib will improve performance of caching ***'
464    print
465    compression = 0       
466    set_option('compression', compression)   
467 
468  print 
469  print_header_box('Testing caching module - please stand by')
470  print   
471
472  # Define a test function to be cached
473  #
474  def f(a,b,c,N,x=0,y='abcdefg'):
475    """f(a,b,c,N)
476       Do something time consuming and produce a complex result.
477    """
478
479    import string
480
481    B = []
482    for n in range(N):
483      s = str(n+2.0/(n + 4.0))+'.a'*10
484      B.append((a,b,c,s,n,x,y))
485    return(B)
486   
487  # Check that default cachedir is OK
488  #     
489  CD = checkdir(cachedir,verbose)   
490   
491   
492  # Make a dependency file
493  #   
494  try:
495    DepFN = CD + 'testfile.tmp'
496    DepFN_wildcard = CD + 'test*.tmp'
497    Depfile = open(DepFN,'w')
498    Depfile.write('We are the knights who say NI!')
499    Depfile.close()
500    test_OK('Wrote file %s' %DepFN)
501  except:
502    test_error('Could not open file %s for writing - check your environment' \
503               % DepFN)
504
505  # Check set_option (and switch stats off
506  #   
507  try:
508    set_option('savestat',0)
509    assert(options['savestat'] == 0)
510    test_OK('Set option')
511  except:
512    test_error('Set option failed')   
513   
514  # Make some test input arguments
515  #
516  N = 5000  #Make N fairly small here
517
518  a = [1,2]
519  b = ('Thou shalt count the number three',4)
520  c = {'Five is right out': 6, (7,8): 9}
521  x = 3
522  y = 'holy hand granate'
523
524  # Test caching
525  #
526  if compression:
527    comprange = 2
528  else:
529    comprange = 1
530
531  for comp in range(comprange):
532 
533    # Evaluate and store
534    #
535    try:
536      T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
537                   verbose=verbose, compression=comp)
538      if comp:                   
539        test_OK('Caching evaluation with compression')
540      else:     
541        test_OK('Caching evaluation without compression')     
542    except:
543      if comp:
544        test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
545      else:
546        test_error('Caching evaluation failed - try caching.test(verbose=1)')
547
548    # Retrieve
549    #                           
550    try:                         
551      T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
552                   compression=comp) 
553
554      if comp:                   
555        test_OK('Caching retrieval with compression')
556      else:     
557        test_OK('Caching retrieval without compression')     
558    except:
559      if comp:
560        test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
561      else:                                     
562        test_error('Caching retrieval failed - try caching.test(verbose=1)')
563
564    # Reference result
565    #   
566    T3 = f(a,b,c,N,x=x,y=y)  # Compute without caching
567   
568    if T1 == T2 and T2 == T3:
569      if comp:
570        test_OK('Basic caching functionality (with compression)')
571      else:
572        test_OK('Basic caching functionality (without compression)')
573    else:
574      test_error('Cached result does not match computed result')
575
576
577  # Test return_filename
578  #   
579  try:
580    FN = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
581                 return_filename=1)   
582    assert(FN[:2] == 'f[')
583    test_OK('Return of cache filename')
584  except:
585    test_error('Return of cache filename failed')
586
587  # Test existence of cachefiles
588  # 
589  try:
590    (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
591    (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
592    (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
593    test_OK('Presence of cache files')
594    datafile.close()
595    argsfile.close()
596    admfile.close()
597  except:
598    test_error('Expected cache files did not exist') 
599             
600  # Test 'test' function when cache is present
601  #     
602  try:
603    #T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
604    #                   evaluate=1) 
605    T4 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
606    assert(T1 == T4)
607
608    test_OK("Option 'test' when cache file present")
609  except:
610    test_error("Option 'test' when cache file present failed")     
611
612  # Test that 'clear' works
613  #
614  #try:
615  #  cache(f,'clear',verbose=verbose)
616  #  test_OK('Clearing of cache files')
617  #except:
618  #  test_error('Clear does not work')
619  try:
620    cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)   
621    test_OK('Clearing of cache files')
622  except:
623    test_error('Clear does not work') 
624
625 
626
627  # Test 'test' function when cache is absent
628  #     
629  try:
630    T4 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
631    assert(T4 is None)
632    test_OK("Option 'test' when cache absent")
633  except:
634    test_error("Option 'test' when cache absent failed")     
635         
636  # Test dependencies
637  #
638  T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
639               dependencies=DepFN) 
640  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
641               dependencies=DepFN)                     
642                       
643  if T1 == T2:
644    test_OK('Basic dependencies functionality')
645  else:
646    test_error('Dependencies do not work')
647
648  # Test basic wildcard dependency
649  #
650  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
651               dependencies=DepFN_wildcard)                     
652   
653  if T1 == T3:
654    test_OK('Basic dependencies with wildcard functionality')
655  else:
656    test_error('Dependencies with wildcards do not work')
657
658
659  # Test that changed timestamp in dependencies triggers recomputation
660 
661  # Modify dependency file
662  Depfile = open(DepFN,'a')
663  Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
664  Depfile.close()
665 
666  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
667               dependencies=DepFN, test = 1)                     
668 
669  if T3 is None:
670    test_OK('Changed dependencies recognised')
671  else:
672    test_error('Changed dependencies not recognised')   
673 
674  # Test recomputation when dependencies have changed
675  #
676  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
677               dependencies=DepFN)                       
678  if T1 == T3:
679    test_OK('Recomputed value with changed dependencies')
680  else:
681    test_error('Recomputed value with changed dependencies failed')
682
683  # Performance test (with statistics)
684  # Don't really rely on this as it will depend on specific computer.
685  #
686
687  set_option('savestat',1)
688
689  N = 20*N   #Should be large on fast computers...
690  tt = time.time()
691  T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
692  t1 = time.time() - tt
693 
694  tt = time.time()
695  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
696  t2 = time.time() - tt
697 
698  if T1 == T2:
699    if t1 > t2:
700      test_OK('Performance test: relative time saved = %s pct' \
701              %str(round((t1-t2)*100/t1,2)))
702    #else:
703    #  print 'WARNING: Performance a bit low - this could be specific to current platform'
704  else:       
705    test_error('Basic caching failed for new problem')
706           
707  # Test presence of statistics file
708  #
709  try: 
710    DIRLIST = os.listdir(CD)
711    SF = []
712    for FN in DIRLIST:
713      if string.find(FN,statsfile) >= 0:
714        fid = open(CD+FN,'r')
715        fid.close()
716    test_OK('Statistics files present') 
717  except:
718    test_OK('Statistics files cannot be opened')         
719     
720  print_header_box('Show sample output of the caching function:')
721 
722  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
723  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
724  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
725 
726  print_header_box('Show sample output of cachestat():')
727  if unix:
728    cachestat()   
729  else:
730    try:
731      import time
732      t = time.strptime('2030','%Y')
733      cachestat()
734    except: 
735      print 'cachestat() does not work here, because it'
736      print 'relies on time.strptime() which is unavailable in Windows'
737     
738  print
739  test_OK('Caching self test completed')   
740     
741           
742  # Test setoption (not yet implemented)
743  #
744
745 
746#==============================================================================
747# Auxiliary functions
748#==============================================================================
749
750# Import pickler
751# cPickle is used by functions mysave, myload, and compare
752#
753import cPickle  # 10 to 100 times faster than pickle
754pickler = cPickle
755
756# Local immutable constants
757#
758comp_level = 1              # Compression level for zlib.
759                            # comp_level = 1 works well.
760textwidth1 = 16             # Text width of key fields in report forms.
761#textwidth2 = 132            # Maximal width of textual representation of
762textwidth2 = 300            # Maximal width of textual representation of
763                            # arguments.
764textwidth3 = 16             # Initial width of separation lines. Is modified.
765textwidth4 = 50             # Text width in test_OK()
766statsfile  = '.cache_stat'  # Basefilename for cached statistics.
767                            # It will reside in the chosen cache directory.
768
769file_types = ['Result',     # File name extension for cached function results.
770              'Args',       # File name extension for stored function args.
771              'Admin']      # File name extension for administrative info.
772
773Reason_msg = ['OK',         # Verbose reasons for recomputation
774              'No cached result', 
775              'Dependencies have changed', 
776              'Arguments have changed',
777              'Bytecode has changed',
778              'Recomputation was requested by caller',
779              'Cached file was unreadable']             
780             
781# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
782
783def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, 
784                dependencies):
785  """Determine whether cached result exists and return info.
786
787  USAGE:
788    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \ 
789    CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
790                dependencies)
791
792  INPUT ARGUMENTS:
793    CD --            Cache Directory
794    FN --            Suggested cache file name
795    func --          Function object
796    args --          Tuple of arguments
797    kwargs --        Dictionary of keyword arguments   
798    deps --          Dependencies time stamps
799    verbose --       Flag text output
800    compression --   Flag zlib compression
801    dependencies --  Given list of dependencies
802   
803  OUTPUT ARGUMENTS:
804    T --             Cached result if present otherwise None
805    FN --            File name under which new results must be saved
806    Retrieved --     True if a valid cached result was found
807    reason --        0: OK (if Retrieved),
808                     1: No cached result,
809                     2: Dependencies have changed,
810                     3: Arguments have changed
811                     4: Bytecode has changed
812                     5: Recomputation was forced
813                     6: Unreadable file
814    comptime --      Number of seconds it took to computed cachged result
815    loadtime --      Number of seconds it took to load cached result
816    compressed --    Flag (0,1) if cached results were compressed or not
817
818  DESCRIPTION:
819    Determine if cached result exists as follows:
820    Load in saved arguments and bytecode stored under hashed filename.
821    If they are identical to current arguments and bytecode and if dependencies
822    have not changed their time stamp, then return cached result.
823
824    Otherwise return filename under which new results should be cached.
825    Hash collisions are handled recursively by calling CacheLookup again with a
826    modified filename.
827  """
828
829  import time, string, types
830
831  # Assess whether cached result exists - compressed or not.
832  #
833  if verbose:
834    print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
835           %(CD+FN, file_types[0], file_types[1], file_types[2])
836  (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
837  (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
838  (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
839
840  if verbose is True and deps is not None:
841    print 'Caching: Dependencies are', deps.keys()
842
843  if not (argsfile and datafile and admfile) or \
844     not (compressed0 == compressed1 and compressed0 == compressed2):
845    # Cached result does not exist or files were compressed differently
846    #
847    # This will ensure that evaluation will take place unless all files are
848    # present.
849
850    reason = 1
851    return(None,FN,None,reason,None,None,None) #Recompute using same filename
852
853  compressed = compressed0  # Remember if compressed files were actually used
854  datafile.close()
855
856  # Retrieve arguments and adm. info
857  #
858  R, reason = myload(argsfile, compressed)  # The original arguments
859  argsfile.close()
860   
861  if reason > 0:
862      # Recompute using same filename   
863      return(None, FN, None, reason, None, None, None)
864  else:   
865      (argsref, kwargsref) = R
866
867  R, reason = myload(admfile, compressed)
868  admfile.close() 
869
870  if reason > 0:
871    return(None,FN,None,reason,None,None,None) #Recompute using same filename
872
873 
874  depsref  = R[0]  # Dependency statistics
875  comptime = R[1]  # The computation time
876  coderef  = R[2]  # The byte code
877  funcname = R[3]  # The function name
878
879  # Check if dependencies have changed
880  #
881  if dependencies and not compare(depsref, deps):
882    if verbose:
883      print 'MESSAGE (caching.py): Dependencies', dependencies, \
884            'have changed - recomputing'
885    # Don't use cached file - recompute
886    reason = 2
887    return(None, FN, None, reason, None, None, None)
888
889  # Get bytecode from func
890  #
891  bytecode = get_bytecode(func)
892
893  #print compare(argsref,args),   
894  #print compare(kwargsref,kwargs),
895  #print compare(bytecode,coderef)
896
897  # Check if arguments or bytecode have changed
898  if compare(argsref, args) and compare(kwargsref, kwargs) and \
899     (not options['bytecode'] or compare(bytecode, coderef)):
900
901    # Arguments and dependencies match. Get cached results
902    T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
903    if reason > 0:
904        # Recompute using same FN     
905        return(None, FN, None, reason, None, None, None)
906
907    Retrieved = 1
908    reason = 0
909
910    if verbose:
911      msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
912
913      if loadtime >= comptime:
914        print 'WARNING (caching.py): Caching did not yield any gain.'
915        print '                      Consider executing function ',
916        print '('+funcname+') without caching.'
917  else:
918
919    # Non matching arguments or bytecodes signify a hash-collision.
920    # This is resolved by recursive search of cache filenames
921    # until either a matching or an unused filename is found.
922    #
923    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
924        CacheLookup(CD, FN+'x', func, args, kwargs, deps, 
925                    verbose, compression, dependencies)
926
927    # DEBUGGING
928    # if not Retrieved:
929    #   print 'Arguments did not match'
930    # else:
931    #   print 'Match found !'
932   
933    # The real reason is that args or bytecodes have changed.
934    # Not that the recursive seach has found an unused filename
935    if not Retrieved:
936      if not compare(bytecode, coderef):
937        reason = 4 # Bytecode has changed
938      else:   
939        reason = 3 # Arguments have changed
940       
941   
942  return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
943
944# -----------------------------------------------------------------------------
945
946def clear_cache(CD,func=None, verbose=None):
947  """Clear cache for func.
948
949  USAGE:
950     clear(CD, func, verbose)
951
952  ARGUMENTS:
953     CD --       Caching directory (required)
954     func --     Function object (default: None)
955     verbose --  Flag verbose output (default: None)
956
957  DESCRIPTION:
958
959    If func == None, clear everything,
960    otherwise clear only files pertaining to func.
961  """
962
963  import os, re
964   
965  if CD[-1] != os.sep:
966    CD = CD+os.sep
967 
968  if verbose == None:
969    verbose = options['verbose']
970
971  # FIXME: Windows version needs to be tested
972
973  if func:
974    funcname = get_funcname(func)
975    if verbose:
976      print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
977
978    file_names = os.listdir(CD)
979    for file_name in file_names:
980      #RE = re.search('^' + funcname,file_name)  #Inefficient
981      #if RE:
982      if file_name[:len(funcname)] == funcname:
983        if unix:
984          os.remove(CD+file_name)
985        else:
986          os.system('del '+CD+file_name)
987          # FIXME: os.remove doesn't work under windows
988  else:
989    file_names = os.listdir(CD)
990    if len(file_names) > 0:
991      if verbose:
992        print 'MESSAGE (caching.py): Remove the following files:'
993        for file_name in file_names:
994            print file_name
995
996        A = raw_input('Delete (Y/N)[N] ?')
997      else:
998        A = 'Y' 
999       
1000      if A == 'Y' or A == 'y':
1001        for file_name in file_names:
1002          if unix:
1003            os.remove(CD+file_name)
1004          else:
1005            os.system('del '+CD+file_name)
1006            # FIXME: os.remove doesn't work under windows
1007          #exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
1008
1009# -----------------------------------------------------------------------------
1010
1011def DeleteOldFiles(CD,verbose=None):
1012  """Remove expired files
1013
1014  USAGE:
1015    DeleteOldFiles(CD,verbose=None)
1016  """
1017
1018  if verbose == None:
1019    verbose = options['verbose']
1020
1021  maxfiles = options['maxfiles']
1022
1023  # FIXME: Windows version
1024
1025  import os
1026  block = 1000  # How many files to delete per invokation
1027  Files = os.listdir(CD)
1028  numfiles = len(Files)
1029  if not unix: return  # FIXME: Windows case ?
1030
1031  if numfiles > maxfiles:
1032    delfiles = numfiles-maxfiles+block
1033    if verbose:
1034      print 'Deleting '+`delfiles`+' expired files:'
1035      os.system('ls -lur '+CD+'* | head -' + `delfiles`)            # List them
1036    os.system('ls -ur '+CD+'* | head -' + `delfiles` + ' | xargs /bin/rm')
1037                                                                  # Delete them
1038    # FIXME: Replace this with os.listdir and os.remove
1039
1040# -----------------------------------------------------------------------------
1041
1042def save_args_to_cache(CD, FN, args, kwargs, compression):
1043  """Save arguments to cache
1044
1045  USAGE:
1046    save_args_to_cache(CD,FN,args,kwargs,compression)
1047  """
1048
1049  import time, os, sys, types
1050
1051  (argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1052
1053  if argsfile is None:
1054    msg = 'ERROR (caching): Could not open argsfile for writing: %s' %FN
1055    raise IOError, msg
1056
1057  mysave((args,kwargs),argsfile,compression)  # Save args and kwargs to cache
1058  argsfile.close()
1059
1060  # Change access rights if possible
1061  #
1062  #if unix:
1063  #  try:
1064  #    exitcode=os.system('chmod 666 '+argsfile.name)
1065  #  except:
1066  #    pass
1067  #else:
1068  #  pass  # FIXME: Take care of access rights under Windows
1069
1070  return
1071
1072# -----------------------------------------------------------------------------
1073
1074def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1075                          dependencies, compression):
1076  """Save computed results T and admin info to cache
1077
1078  USAGE:
1079    save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1080                          dependencies, compression)
1081  """
1082
1083  import time, os, sys, types
1084
1085  (datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1086  (admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1087
1088  if not datafile:
1089    if verbose:
1090      print 'ERROR (caching): Could not open %s' %datafile.name
1091    raise IOError
1092
1093  if not admfile:
1094    if verbose:
1095      print 'ERROR (caching): Could not open %s' %admfile.name
1096    raise IOError
1097
1098  t0 = time.time()
1099
1100  mysave(T,datafile,compression)  # Save data to cache
1101  datafile.close()
1102  #savetime = round(time.time()-t0,2)
1103  savetime = time.time()-t0 
1104
1105  bytecode = get_bytecode(func)  # Get bytecode from function object
1106  admtup = (deps, comptime, bytecode, funcname)  # Gather admin info
1107
1108  mysave(admtup,admfile,compression)  # Save admin info to cache
1109  admfile.close()
1110
1111  # Change access rights if possible
1112  #
1113  #if unix:
1114  #  try:
1115  #    exitcode=os.system('chmod 666 '+datafile.name)
1116  #    exitcode=os.system('chmod 666 '+admfile.name)
1117  #  except:
1118  #    pass
1119  #else:
1120  #  pass  # FIXME: Take care of access rights under Windows
1121
1122  return(savetime)
1123
1124# -----------------------------------------------------------------------------
1125
1126def load_from_cache(CD, FN, compression):
1127  """Load previously cached data from file FN
1128
1129  USAGE:
1130    load_from_cache(CD,FN,compression)
1131  """
1132
1133  import time
1134
1135  (datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1136  t0 = time.time()
1137  T, reason = myload(datafile,compressed)
1138
1139  loadtime = time.time()-t0
1140  datafile.close() 
1141
1142  return T, loadtime, compressed, reason
1143
1144# -----------------------------------------------------------------------------
1145
1146def myopen(FN, mode, compression=True):
1147  """Open file FN using given mode
1148
1149  USAGE:
1150    myopen(FN, mode, compression=True)
1151
1152  ARGUMENTS:
1153    FN --           File name to be opened
1154    mode --         Open mode (as in open)
1155    compression --  Flag zlib compression
1156
1157  DESCRIPTION:
1158     if compression
1159       Attempt first to open FN + '.z'
1160       If this fails try to open FN
1161     else do the opposite
1162     Return file handle plus info about whether it was compressed or not.
1163  """
1164
1165  import string
1166
1167  # Determine if file exists already (if writing was requested)
1168  # This info is only used to determine if access modes should be set
1169  #
1170  if 'w' in mode or 'a' in mode:
1171    try:
1172      file = open(FN+'.z','r')
1173      file.close()
1174      new_file = 0
1175    except:
1176      try:
1177        file = open(FN,'r') 
1178        file.close()
1179        new_file = 0
1180      except:
1181        new_file = 1
1182  else:
1183    new_file = 0 #Assume it exists if mode was not 'w'
1184 
1185
1186  compressed = 0
1187  if compression:
1188    try:
1189      file = open(FN+'.z',mode)
1190      compressed = 1
1191    except:
1192      try:
1193        file = open(FN,mode)
1194      except:
1195        file = None
1196  else:
1197    try:
1198      file = open(FN,mode)
1199    except:
1200      try:
1201        file = open(FN+'.z',mode)
1202        compressed = 1
1203      except:
1204        file = None
1205
1206  # Now set access rights if it is a new file
1207  #
1208  if file and new_file:
1209    if unix:
1210      exitcode=os.system('chmod 666 '+file.name)
1211    else:
1212      pass  # FIXME: Take care of access rights under Windows
1213
1214  return(file,compressed)
1215
1216# -----------------------------------------------------------------------------
1217
1218def myload(file, compressed):
1219  """Load data from file
1220
1221  USAGE:
1222    myload(file, compressed)
1223  """
1224
1225  reason = 0
1226  try:
1227    if compressed:
1228      import zlib
1229
1230      RsC = file.read()
1231      try:
1232        Rs  = zlib.decompress(RsC)
1233      except:
1234        #  File "./caching.py", line 1032, in load_from_cache
1235        #  T = myload(datafile,compressed)
1236        #  File "./caching.py", line 1124, in myload
1237        #  Rs  = zlib.decompress(RsC)
1238        #  zlib.error: Error -5 while decompressing data
1239        #print 'ERROR (caching): Could not decompress ', file.name
1240        #raise Exception
1241        reason = 6  # Unreadable file
1242        return None, reason 
1243     
1244     
1245      del RsC  # Free up some space
1246      R = pickler.loads(Rs)
1247    else:
1248      try:
1249        R = pickler.load(file)
1250      #except EOFError, e:
1251      except:
1252        #Catch e.g., file with 0 length or corrupted
1253        reason = 6  # Unreadable file
1254        return None, reason
1255     
1256  except MemoryError:
1257    import sys
1258    if options['verbose']:
1259      print 'ERROR (caching): Out of memory while loading %s, aborting' \
1260            %(file.name)
1261
1262    # Raise the error again for now
1263    #
1264    raise MemoryError
1265
1266  return R, reason
1267
1268# -----------------------------------------------------------------------------
1269
1270def mysave(T, file, compression):
1271  """Save data T to file
1272
1273  USAGE:
1274    mysave(T, file, compression)
1275
1276  """
1277
1278  bin = options['bin']
1279
1280  if compression:
1281    try:
1282      import zlib
1283    except:
1284      print
1285      print '*** Could not find zlib ***'
1286      print '*** Try to run caching with compression off ***'
1287      print "*** caching.set_option('compression', 0) ***"
1288      raise Exception
1289     
1290
1291    try:
1292      Ts  = pickler.dumps(T, bin)
1293    except MemoryError:
1294      msg = '****WARNING (caching.py): Could not pickle data for compression.'
1295      msg += ' Try using compression = False'
1296      raise MemoryError, msg
1297    else: 
1298      # Compressed pickling     
1299      TsC = zlib.compress(Ts, comp_level)
1300      file.write(TsC)
1301  else:
1302      # Uncompressed pickling
1303      pickler.dump(T, file, bin)
1304
1305      # FIXME: This may not work on Windoze network drives.
1306      # The error msg is IOError: [Errno 22] Invalid argument
1307      # Testing with small files was OK, though.
1308      # I think this is an OS problem.
1309
1310      # Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1311     
1312# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1313#
1314#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1315#
1316#The following extract is taken from the site http://www.python.org:
1317#
1318#---------------------------------------------------------------------------------------------
1319#exception IOError
1320#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1321#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1322#---------------------------------------------------------------------------------------------
1323#
1324#The error code(s) that accompany exceptions are described at:
1325#http://www.python.org/dev/doc/devel//lib/module-errno.html
1326#
1327#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1328#
1329#exceptions.IOError invalid argument Errno 22
1330       
1331      #try:
1332      #  pickler.dump(T,file,bin)
1333      #except IOError, e:
1334      #  print e
1335      #  msg = 'Could not store to %s, bin=%s' %(file, bin)
1336      #  raise msg
1337     
1338
1339# -----------------------------------------------------------------------------
1340
1341   
1342def myhash(T, ids=None):
1343  """Compute hashed integer from a range of inputs.
1344  If T is not hashable being e.g. a tuple T, myhash will recursively
1345  hash the values individually
1346
1347  USAGE:
1348    myhash(T)
1349
1350  ARGUMENTS:
1351    T -- Anything
1352  """
1353
1354  from types import TupleType, ListType, DictType, InstanceType 
1355   
1356  if type(T) in [TupleType, ListType, DictType, InstanceType]: 
1357      # Keep track of unique id's to protect against infinite recursion
1358      if ids is None: ids = []
1359
1360      # Check if T has already been encountered
1361      i = id(T) 
1362 
1363      if i in ids:
1364          return 0 # T has been hashed already     
1365      else:
1366          ids.append(i)
1367   
1368
1369   
1370  # Start hashing 
1371 
1372  # On some architectures None, False and True gets different hash values
1373  if T is None:
1374      return(-1)
1375  if T is False:
1376      return(0)
1377  if T is True:
1378      return(1)
1379
1380  # Get hash values for hashable entries
1381  if type(T) in [TupleType, ListType]:
1382      hvals = []
1383      for t in T:
1384          h = myhash(t, ids)
1385          hvals.append(h)
1386      val = hash(tuple(hvals))
1387  elif type(T) == DictType:
1388      # Make dictionary ordering unique 
1389      I = T.items()
1390      I.sort()   
1391      val = myhash(I, ids)
1392  elif type(T) == num.ArrayType:
1393      T = num.array(T) # Ensure array is contiguous
1394
1395      # Use mean value for efficiency
1396      val = hash(num.average(T.flat))
1397  elif type(T) == InstanceType:
1398      # Use the attribute values
1399      val = myhash(T.__dict__, ids)
1400  else:
1401      try:
1402          val = hash(T)
1403      except:
1404          val = 1
1405
1406  return(val)
1407
1408
1409
1410def compare(A, B, ids=None):
1411    """Safe comparison of general objects
1412
1413    USAGE:
1414      compare(A,B)
1415
1416    DESCRIPTION:
1417      Return 1 if A and B they are identical, 0 otherwise
1418    """
1419
1420    from types import TupleType, ListType, DictType, InstanceType
1421   
1422    # Keep track of unique id's to protect against infinite recursion
1423    if ids is None: ids = {}
1424
1425    # Check if T has already been encountered
1426    iA = id(A) 
1427    iB = id(B)     
1428   
1429    if ids.has_key((iA, iB)):
1430        # A and B have been compared already
1431        #print 'Found', (iA, iB), A, B
1432        return ids[(iA, iB)]
1433    else:
1434        ids[(iA, iB)] = True
1435   
1436   
1437    # Check if arguments are of same type
1438    if type(A) != type(B):
1439        return False
1440       
1441 
1442    # Compare recursively based on argument type
1443    if type(A) in [TupleType, ListType]:
1444        N = len(A)
1445        if len(B) != N: 
1446            identical = False
1447        else:
1448            identical = True
1449            for i in range(N):
1450                if not compare(A[i], B[i], ids): 
1451                    identical = False; break
1452                   
1453    elif type(A) == DictType:
1454        if len(A) != len(B):
1455            identical = False
1456        else:                       
1457            # Make dictionary ordering unique
1458            a = A.items(); a.sort()   
1459            b = B.items(); b.sort()
1460           
1461            identical = compare(a, b, ids)
1462           
1463    elif type(A) == num.ArrayType:
1464        # Use element by element comparison
1465        identical = num.alltrue(A==B)
1466
1467    elif type(A) == InstanceType:
1468        # Take care of special case where elements are instances           
1469        # Base comparison on attributes     
1470        identical = compare(A.__dict__, 
1471                            B.__dict__, 
1472                            ids)
1473    else:       
1474        # Fall back to general code
1475        try:
1476            identical = (A == B)
1477        except:
1478            import pickle
1479            # Use pickle to compare data
1480            # The native pickler must be used
1481            # since the faster cPickle does not
1482            # guarantee a unique translation           
1483            try:
1484                identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
1485            except:
1486                identical = False
1487
1488    # Record result of comparison and return           
1489    ids[(iA, iB)] = identical
1490   
1491    return(identical)
1492
1493   
1494# -----------------------------------------------------------------------------
1495
1496def nospace(s):
1497  """Replace spaces in string s with underscores
1498
1499  USAGE:
1500    nospace(s)
1501
1502  ARGUMENTS:
1503    s -- string
1504  """
1505
1506  import string
1507
1508  newstr = ''
1509  for i in range(len(s)):
1510    if s[i] == ' ':
1511      newstr = newstr+'_'
1512    else:
1513      newstr = newstr+s[i]
1514
1515  return(newstr)
1516
1517# -----------------------------------------------------------------------------
1518
1519def get_funcname(func):
1520  """Retrieve name of function object func (depending on its type)
1521
1522  USAGE:
1523    get_funcname(func)
1524  """
1525
1526  import types, string
1527
1528  if type(func) == types.FunctionType:
1529    funcname = func.func_name
1530  elif type(func) == types.BuiltinFunctionType:
1531    funcname = func.__name__
1532  else:
1533    tab = string.maketrans("<>'","   ")
1534    tmp = string.translate(`func`,tab)
1535    tmp = string.split(tmp)
1536    funcname = string.join(tmp)
1537
1538    # Truncate memory address as in
1539    # class __main__.Dummy at 0x00A915D0
1540    index = funcname.find('at 0x')
1541    if index >= 0:
1542      funcname = funcname[:index+5] # Keep info that there is an address
1543
1544  funcname = nospace(funcname)
1545  return(funcname)
1546
1547# -----------------------------------------------------------------------------
1548
1549def get_bytecode(func):
1550  """ Get bytecode from function object.
1551
1552  USAGE:
1553    get_bytecode(func)
1554  """
1555
1556  import types
1557
1558  if type(func) == types.FunctionType:
1559    return get_func_code_details(func)
1560  elif type(func) == types.MethodType:
1561    return get_func_code_details(func.im_func)
1562  elif type(func) == types.InstanceType:   
1563    if callable(func):
1564      # Get bytecode from __call__ method
1565      bytecode = get_func_code_details(func.__call__.im_func)
1566     
1567      # Add hash value of object to detect attribute changes
1568      return bytecode + (myhash(func),) 
1569    else:
1570      msg = 'Instance %s was passed into caching in the role of a function ' % str(func)
1571      msg = ' but it was not callable.'
1572      raise Exception, msg
1573  elif type(func) in [types.BuiltinFunctionType, types.BuiltinMethodType]:     
1574    # Built-in functions are assumed not to change 
1575    return None, 0, 0, 0
1576  elif type(func) == types.ClassType:
1577      # Get bytecode from __init__ method
1578      bytecode = get_func_code_details(func.__init__.im_func)   
1579      return bytecode     
1580  else:
1581    msg = 'Unknown function type: %s' % type(func)
1582    raise Exception, msg
1583
1584
1585 
1586 
1587def get_func_code_details(func):
1588  """Extract co_code, co_consts, co_argcount, func_defaults
1589  """
1590 
1591  bytecode = func.func_code.co_code
1592  consts = func.func_code.co_consts
1593  argcount = func.func_code.co_argcount   
1594  defaults = func.func_defaults       
1595 
1596  return bytecode, consts, argcount, defaults 
1597
1598# -----------------------------------------------------------------------------
1599
1600def get_depstats(dependencies):
1601  """ Build dictionary of dependency files and their size, mod. time and ctime.
1602
1603  USAGE:
1604    get_depstats(dependencies):
1605  """
1606
1607  import types
1608
1609  #print 'Caching DEBUG: Dependencies are', dependencies
1610  d = {}
1611  if dependencies:
1612
1613    #Expand any wildcards
1614    import glob
1615    expanded_dependencies = []
1616    for FN in dependencies:
1617      expanded_FN = glob.glob(FN)
1618
1619      if expanded_FN == []:
1620        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1621        raise Exception, errmsg     
1622
1623      expanded_dependencies += expanded_FN
1624
1625   
1626    for FN in expanded_dependencies:
1627      if not type(FN) == types.StringType:
1628        errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1629        errmsg += '                   Dependency given: %s' %FN
1630        raise Exception, errmsg     
1631      if not os.access(FN,os.F_OK):
1632        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1633        raise Exception, errmsg
1634      (size,atime,mtime,ctime) = filestat(FN)
1635
1636      # We don't use atime because that would cause recomputation every time.
1637      # We don't use ctime because that is irrelevant and confusing for users.
1638      d.update({FN : (size,mtime)})
1639
1640  return(d)
1641
1642# -----------------------------------------------------------------------------
1643
1644def filestat(FN):
1645  """A safe wrapper using os.stat to get basic file statistics
1646     The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1647
1648  USAGE:
1649    filestat(FN)
1650
1651  DESCRIPTION:
1652     Must compile Python with
1653     CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1654              configure
1655     as given in section 8.1.1 Large File Support in the Libray Reference
1656  """
1657
1658  import os, time
1659
1660  try:
1661    stats = os.stat(FN)
1662    size  = stats[6]
1663    atime = stats[7]
1664    mtime = stats[8]
1665    ctime = stats[9]
1666  except:
1667
1668    # Hack to get the results anyway (works only on Unix at the moment)
1669    #
1670    print 'Hack to get os.stat when files are too large'
1671
1672    if unix:
1673      tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1674      # Unique filename, FIXME: Use random number
1675
1676      # Get size and access time (atime)
1677      #
1678      exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1679      (size,atime) = get_lsline(tmp)
1680
1681      # Get size and modification time (mtime)
1682      #
1683      exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1684      (size,mtime) = get_lsline(tmp)
1685
1686      # Get size and ctime
1687      #
1688      exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1689      (size,ctime) = get_lsline(tmp)
1690
1691      try:
1692        exitcode=os.system('rm '+tmp)
1693        # FIXME: Gives error if file doesn't exist
1694      except:
1695        pass
1696    else:
1697      pass
1698      raise Exception  # FIXME: Windows case
1699
1700  return(long(size),atime,mtime,ctime)
1701
1702# -----------------------------------------------------------------------------
1703
1704def get_lsline(FN):
1705  """get size and time for filename
1706
1707  USAGE:
1708    get_lsline(file_name)
1709
1710  DESCRIPTION:
1711    Read in one line 'ls -la' item from file (generated by filestat) and
1712    convert time to seconds since epoch. Return file size and time.
1713  """
1714
1715  import string, time
1716
1717  f = open(FN,'r')
1718  info = f.read()
1719  info = string.split(info)
1720
1721  size = info[4]
1722  week = info[5]
1723  mon  = info[6]
1724  day  = info[7]
1725  hour = info[8]
1726  year = info[9]
1727
1728  str = week+' '+mon+' '+day+' '+hour+' '+year
1729  timetup = time.strptime(str)
1730  t = time.mktime(timetup)
1731  return(size, t)
1732
1733# -----------------------------------------------------------------------------
1734
1735def checkdir(CD, verbose=None, warn=False):
1736  """Check or create caching directory
1737
1738  USAGE:
1739    checkdir(CD,verbose):
1740
1741  ARGUMENTS:
1742    CD -- Directory
1743    verbose -- Flag verbose output (default: None)
1744
1745  DESCRIPTION:
1746    If CD does not exist it will be created if possible
1747  """
1748
1749  import os
1750  import os.path
1751
1752  if CD[-1] != os.sep: 
1753    CD = CD + os.sep  # Add separator for directories
1754
1755  CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1756  if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1757    try:
1758      exitcode=os.mkdir(CD)
1759
1760      # Change access rights if possible
1761      #
1762      if unix:
1763        exitcode=os.system('chmod 777 '+CD)
1764      else:
1765        pass  # FIXME: What about acces rights under Windows?
1766      if verbose: print 'MESSAGE: Directory', CD, 'created.'
1767    except:
1768      if warn is True:
1769        print 'WARNING: Directory', CD, 'could not be created.'
1770      if unix:
1771        CD = '/tmp/'
1772      else:
1773        CD = 'C:' 
1774      if warn is True:
1775        print 'Using directory %s instead' %CD
1776
1777  return(CD)
1778
1779checkdir(cachedir, warn=True)
1780
1781#==============================================================================
1782# Statistics
1783#==============================================================================
1784
1785def addstatsline(CD, funcname, FN, Retrieved, reason, comptime, loadtime,
1786                 compression):
1787  """Add stats entry
1788
1789  USAGE:
1790    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1791
1792  DESCRIPTION:
1793    Make one entry in the stats file about one cache hit recording time saved
1794    and other statistics. The data are used by the function cachestat.
1795  """
1796
1797  import os, time
1798
1799  try:
1800    TimeTuple = time.localtime(time.time())
1801    extension = time.strftime('%b%Y',TimeTuple)
1802    SFN = CD+statsfile+'.'+extension
1803    #statfile = open(SFN,'a')
1804    (statfile, dummy) = myopen(SFN,'a',compression=0)
1805
1806    # Change access rights if possible
1807    #
1808    #if unix:
1809    #  try:
1810    #    exitcode=os.system('chmod 666 '+SFN)
1811    #  except:
1812    #    pass
1813  except:
1814    print 'Warning: Stat file could not be opened'
1815
1816  try:
1817    if os.environ.has_key('USER'):
1818      user = os.environ['USER']
1819    else:
1820      user = 'Nobody'
1821
1822    date = time.asctime(TimeTuple)
1823
1824    if Retrieved:
1825      hit = '1'
1826    else:
1827      hit = '0'
1828
1829    # Get size of result file
1830    #   
1831    if compression:
1832      stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1833    else:
1834      stats = os.stat(CD+FN+'_'+file_types[0])
1835 
1836    if stats: 
1837      size = stats[6]
1838    else:
1839      size = -1  # Error condition, but don't crash. This is just statistics 
1840
1841    # Build entry
1842    # 
1843    entry = date             + ',' +\
1844            user             + ',' +\
1845            FN               + ',' +\
1846            str(int(size))   + ',' +\
1847            str(compression) + ',' +\
1848            hit              + ',' +\
1849            str(reason)      + ',' +\
1850            str(round(comptime,4)) + ',' +\
1851            str(round(loadtime,4)) +\
1852            CR
1853           
1854    statfile.write(entry)
1855    statfile.close()
1856  except:
1857    print 'Warning: Writing of stat file failed'
1858
1859# -----------------------------------------------------------------------------
1860
1861# FIXME: should take cachedir as an optional arg
1862#
1863def __cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
1864  """  List caching statistics.
1865
1866  USAGE:
1867    __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1868
1869      Generate statistics of caching efficiency.
1870      The parameter sortidx determines by what field lists are sorted.
1871      If the optional keyword period is set to -1,
1872      all available caching history is used.
1873      If it is 0 only the current month is used.
1874      Future versions will include more than one month....
1875      OMN 20/8/2000
1876  """
1877
1878  import os
1879  import os.path
1880  from string import split, rstrip, find
1881  from time import strptime, localtime, strftime, mktime, ctime
1882
1883  # sortidx = 4    # Index into Fields[1:]. What to sort by.
1884
1885  Fields = ['Name', 'Hits', 'Exec(s)', \
1886            'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1887  Widths = [25,7,9,9,9,9,13]
1888  #Types = ['s','d','d','d','d','.2f','d']
1889  Types = ['s','d','.2f','.2f','.2f','.2f','d'] 
1890
1891  Dictnames = ['Function', 'User']
1892
1893  if not cachedir:
1894    cachedir = checkdir(options['cachedir'])
1895
1896  SD = os.path.expanduser(cachedir)  # Expand ~ or ~user in pathname
1897
1898  if period == -1:  # Take all available stats
1899    SFILENAME = statsfile
1900  else:  # Only stats from current month 
1901       # MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1902    TimeTuple = localtime(time())
1903    extension = strftime('%b%Y',TimeTuple)
1904    SFILENAME = statsfile+'.'+extension
1905
1906  DIRLIST = os.listdir(SD)
1907  SF = []
1908  for FN in DIRLIST:
1909    if find(FN,SFILENAME) >= 0:
1910      SF.append(FN)
1911
1912  blocksize = 15000000
1913  total_read = 0
1914  total_hits = 0
1915  total_discarded = 0
1916  firstday = mktime(strptime('2030','%Y'))
1917             # FIXME: strptime don't exist in WINDOWS ?
1918  lastday = 0
1919
1920  FuncDict = {}
1921  UserDict = {}
1922  for FN in SF:
1923    input = open(SD+FN,'r')
1924    print 'Reading file ', SD+FN
1925
1926    while True:
1927      A = input.readlines(blocksize)
1928      if len(A) == 0: break
1929      total_read = total_read + len(A)
1930      for record in A:
1931        record = tuple(split(rstrip(record),','))
1932        #print record, len(record)
1933
1934        if len(record) == 9:
1935          timestamp = record[0]
1936       
1937          try:
1938            t = mktime(strptime(timestamp))
1939          except:
1940            total_discarded = total_discarded + 1         
1941            continue   
1942             
1943          if t > lastday:
1944            lastday = t
1945          if t < firstday:
1946            firstday = t
1947
1948          user     = record[1]
1949          func     = record[2]
1950
1951          # Strip hash-stamp off
1952          #
1953          i = find(func,'[')
1954          func = func[:i]
1955
1956          size        = float(record[3])
1957
1958          # Compression kepword can be Boolean
1959          if record[4] in ['True', '1']:
1960            compression = 1
1961          elif record[4] in ['False', '0']: 
1962            compression = 0
1963          else:
1964            print 'Unknown value of compression', record[4]
1965            print record
1966            total_discarded = total_discarded + 1           
1967            continue
1968
1969          #compression = int(record[4]) # Can be Boolean
1970          hit         = int(record[5])
1971          reason      = int(record[6])   # Not used here   
1972          cputime     = float(record[7])
1973          loadtime    = float(record[8])
1974
1975          if hit:
1976            total_hits = total_hits + 1
1977            saving = cputime-loadtime
1978
1979            if cputime != 0:
1980              rel_saving = round(100.0*saving/cputime,2)
1981            else:
1982              #rel_saving = round(1.0*saving,2)
1983              rel_saving = 100.0 - round(1.0*saving,2)  # A bit of a hack
1984
1985            info = [1,cputime,loadtime,saving,rel_saving,size]
1986
1987            UpdateDict(UserDict,user,info)
1988            UpdateDict(FuncDict,func,info)
1989          else:
1990            pass #Stats on recomputations and their reasons could go in here
1991             
1992        else:
1993          #print 'Record discarded'
1994          #print record
1995          total_discarded = total_discarded + 1
1996
1997    input.close()
1998
1999  # Compute averages of all sums and write list
2000  #
2001
2002  if total_read == 0:
2003    printline(Widths,'=')
2004    print 'CACHING STATISTICS: No valid records read'
2005    printline(Widths,'=')
2006    return
2007
2008  print
2009  printline(Widths,'=')
2010  print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
2011  printline(Widths,'=')
2012  #print '  Period:', ctime(firstday), 'to', ctime(lastday)
2013  print '  Total number of valid records', total_read
2014  print '  Total number of discarded records', total_discarded
2015  print '  Total number of hits', total_hits
2016  print
2017
2018  print '  Fields', Fields[2:], 'are averaged over number of hits'
2019  print '  Time is measured in seconds and size in bytes'
2020  print '  Tables are sorted by', Fields[1:][sortidx]
2021
2022  # printline(Widths,'-')
2023
2024  if showuser:
2025    Dictionaries = [FuncDict, UserDict]
2026  else:
2027    Dictionaries = [FuncDict]
2028
2029  i = 0
2030  for Dict in Dictionaries:
2031    for key in Dict.keys():
2032      rec = Dict[key]
2033      for n in range(len(rec)):
2034        if n > 0:
2035          rec[n] = round(1.0*rec[n]/rec[0],2)
2036      Dict[key] = rec
2037
2038    # Sort and output
2039    #
2040    keylist = SortDict(Dict,sortidx)
2041
2042    # Write Header
2043    #
2044    print
2045    #print Dictnames[i], 'statistics:'; i=i+1
2046    printline(Widths,'-')
2047    n = 0
2048    for s in Fields:
2049      if s == Fields[0]:  # Left justify
2050        s = Dictnames[i] + ' ' + s; i=i+1
2051        exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
2052      else:
2053        exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
2054    print
2055    printline(Widths,'-')
2056
2057    # Output Values
2058    #
2059    for key in keylist:
2060      rec = Dict[key]
2061      n = 0
2062      if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
2063      exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
2064      for val in rec:
2065        exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
2066      print
2067    print
2068
2069#==============================================================================
2070# Auxiliary stats functions
2071#==============================================================================
2072
2073def UpdateDict(Dict, key, info):
2074  """Update dictionary by adding new values to existing.
2075
2076  USAGE:
2077    UpdateDict(Dict,key,info)
2078  """
2079
2080  if Dict.has_key(key):
2081    dinfo = Dict[key]
2082    for n in range(len(dinfo)):
2083      dinfo[n] = info[n] + dinfo[n]
2084  else:
2085    dinfo = info[:]  # Make a copy of info list
2086
2087  Dict[key] = dinfo
2088  return Dict
2089
2090# -----------------------------------------------------------------------------
2091
2092def SortDict(Dict, sortidx=0):
2093  """Sort dictionary
2094
2095  USAGE:
2096    SortDict(Dict,sortidx):
2097
2098  DESCRIPTION:
2099    Sort dictionary of lists according field number 'sortidx'
2100  """
2101
2102  import types
2103
2104  sortlist  = []
2105  keylist = Dict.keys()
2106  for key in keylist:
2107    rec = Dict[key]
2108    if not type(rec) in [types.ListType, types.TupleType]:
2109      rec = [rec]
2110
2111    if sortidx > len(rec)-1:
2112      if options['verbose']:
2113        print 'ERROR: Sorting index to large, sortidx = ', sortidx
2114      raise IndexError
2115
2116    val = rec[sortidx]
2117    sortlist.append(val)
2118
2119  A = map(None,sortlist,keylist)
2120  A.sort()
2121  keylist = map(lambda x: x[1], A)  # keylist sorted by sortidx
2122
2123  return(keylist)
2124
2125# -----------------------------------------------------------------------------
2126
2127def printline(Widths,char):
2128  """Print textline in fixed field.
2129
2130  USAGE:
2131    printline(Widths,char)
2132  """
2133
2134  s = ''
2135  for n in range(len(Widths)):
2136    s = s+Widths[n]*char
2137    if n > 0:
2138      s = s+char
2139
2140  print s
2141
2142#==============================================================================
2143# Messages
2144#==============================================================================
2145
2146def msg1(funcname, args, kwargs, reason):
2147  """Message 1
2148
2149  USAGE:
2150    msg1(funcname, args, kwargs, reason):
2151  """
2152
2153  import string
2154  #print 'MESSAGE (caching.py): Evaluating function', funcname,
2155
2156  print_header_box('Evaluating function %s' %funcname)
2157 
2158  msg7(args, kwargs)
2159  msg8(reason) 
2160 
2161  print_footer()
2162 
2163  #
2164  # Old message
2165  #
2166  #args_present = 0
2167  #if args:
2168  #  if len(args) == 1:
2169  #    print 'with argument', mkargstr(args[0], textwidth2),
2170  #  else:
2171  #    print 'with arguments', mkargstr(args, textwidth2),
2172  #  args_present = 1     
2173  #   
2174  #if kwargs:
2175  #  if args_present:
2176  #    word = 'and'
2177  #  else:
2178  #    word = 'with'
2179  #     
2180  #  if len(kwargs) == 1:
2181  #    print word + ' keyword argument', mkargstr(kwargs, textwidth2)
2182  #  else:
2183  #    print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
2184  #  args_present = 1           
2185  #else:
2186  #  print    # Newline when no keyword args present
2187  #       
2188  #if not args_present:   
2189  #  print '',  # Default if no args or kwargs present
2190   
2191   
2192
2193# -----------------------------------------------------------------------------
2194
2195def msg2(funcname, args, kwargs, comptime, reason):
2196  """Message 2
2197
2198  USAGE:
2199    msg2(funcname, args, kwargs, comptime, reason)
2200  """
2201
2202  import string
2203
2204  #try:
2205  #  R = Reason_msg[reason]
2206  #except:
2207  #  R = 'Unknown reason' 
2208 
2209  #print_header_box('Caching statistics (storing) - %s' %R)
2210  print_header_box('Caching statistics (storing)') 
2211 
2212  msg6(funcname,args,kwargs)
2213  msg8(reason)
2214
2215  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2216
2217# -----------------------------------------------------------------------------
2218
2219def msg3(savetime, CD, FN, deps, compression):
2220  """Message 3
2221
2222  USAGE:
2223    msg3(savetime, CD, FN, deps, compression)
2224  """
2225
2226  import string
2227  print string.ljust('| Loading time:', textwidth1) + str(round(savetime,2)) + \
2228                     ' seconds (estimated)'
2229  msg5(CD,FN,deps,compression)
2230
2231# -----------------------------------------------------------------------------
2232
2233def msg4(funcname, args, kwargs, deps, comptime, loadtime, CD, FN, compression):
2234  """Message 4
2235
2236  USAGE:
2237    msg4(funcname, args, kwargs, deps, comptime, loadtime, CD, FN, compression)
2238  """
2239
2240  import string
2241
2242  print_header_box('Caching statistics (retrieving)')
2243 
2244  msg6(funcname,args,kwargs)
2245  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2246  print string.ljust('| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
2247  print string.ljust('| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
2248        ' seconds'
2249  msg5(CD,FN,deps,compression)
2250
2251# -----------------------------------------------------------------------------
2252
2253def msg5(CD, FN, deps, compression):
2254  """Message 5
2255
2256  USAGE:
2257    msg5(CD, FN, deps, compression)
2258
2259  DESCRIPTION:
2260   Print dependency stats. Used by msg3 and msg4
2261  """
2262
2263  import os, time, string
2264
2265  print '|'
2266  print string.ljust('| Caching dir: ', textwidth1) + CD
2267
2268  if compression:
2269    suffix = '.z'
2270    bytetext = 'bytes, compressed'
2271  else:
2272    suffix = ''
2273    bytetext = 'bytes'
2274
2275  for file_type in file_types:
2276    file_name = FN + '_' + file_type + suffix
2277    print string.ljust('| ' + file_type + ' file: ', textwidth1) + file_name,
2278    stats = os.stat(CD+file_name)
2279    print '('+ str(stats[6]) + ' ' + bytetext + ')'
2280
2281  print '|'
2282  if len(deps) > 0:
2283    print '| Dependencies:  '
2284    dependencies  = deps.keys()
2285    dlist = []; maxd = 0
2286    tlist = []; maxt = 0
2287    slist = []; maxs = 0
2288    for d in dependencies:
2289      stats = deps[d]
2290      t = time.ctime(stats[1])
2291      s = str(stats[0])
2292      #if s[-1] == 'L':
2293      #  s = s[:-1]  # Strip rightmost 'long integer' L off.
2294      #              # FIXME: Unnecessary in versions later than 1.5.2
2295
2296      if len(d) > maxd: maxd = len(d)
2297      if len(t) > maxt: maxt = len(t)
2298      if len(s) > maxs: maxs = len(s)
2299      dlist.append(d)
2300      tlist.append(t)
2301      slist.append(s)
2302
2303    for n in range(len(dlist)):
2304      d = string.ljust(dlist[n]+':', maxd+1)
2305      t = string.ljust(tlist[n], maxt)
2306      s = string.rjust(slist[n], maxs)
2307
2308      print '| ', d, t, ' ', s, 'bytes'
2309  else:
2310    print '| No dependencies'
2311  print_footer()
2312
2313# -----------------------------------------------------------------------------
2314
2315def msg6(funcname, args, kwargs):
2316  """Message 6
2317
2318  USAGE:
2319    msg6(funcname, args, kwargs)
2320  """
2321
2322  import string
2323  print string.ljust('| Function:', textwidth1) + funcname
2324
2325  msg7(args, kwargs)
2326 
2327# -----------------------------------------------------------------------------   
2328
2329def msg7(args, kwargs):
2330  """Message 7
2331 
2332  USAGE:
2333    msg7(args, kwargs):
2334  """
2335 
2336  import string
2337 
2338  args_present = 0 
2339  if args:
2340    if len(args) == 1:
2341      print string.ljust('| Argument:', textwidth1) + mkargstr(args[0], \
2342                         textwidth2)
2343    else:
2344      print string.ljust('| Arguments:', textwidth1) + \
2345            mkargstr(args, textwidth2)
2346    args_present = 1
2347           
2348  if kwargs:
2349    if len(kwargs) == 1:
2350      print string.ljust('| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
2351                         textwidth2)
2352    else:
2353      print string.ljust('| Keyword Args:', textwidth1) + \
2354            mkargstr(kwargs, textwidth2)
2355    args_present = 1
2356
2357  if not args_present:               
2358    print '| No arguments' # Default if no args or kwargs present
2359
2360# -----------------------------------------------------------------------------
2361
2362def msg8(reason):
2363  """Message 8
2364 
2365  USAGE:
2366    msg8(reason):
2367  """
2368 
2369  import string
2370   
2371  try:
2372    R = Reason_msg[reason]
2373  except:
2374    R = 'Unknown' 
2375 
2376  print string.ljust('| Reason:', textwidth1) + R
2377   
2378# -----------------------------------------------------------------------------
2379
2380def print_header_box(line):
2381  """Print line in a nice box.
2382 
2383  USAGE:
2384    print_header_box(line)
2385
2386  """
2387  global textwidth3
2388
2389  import time
2390
2391  time_stamp = time.ctime(time.time())
2392  line = time_stamp + '. ' + line
2393   
2394  N = len(line) + 1
2395  s = '+' + '-'*N + CR
2396
2397  print s + '| ' + line + CR + s,
2398
2399  textwidth3 = N
2400
2401# -----------------------------------------------------------------------------
2402   
2403def print_footer():
2404  """Print line same width as that of print_header_box.
2405  """
2406 
2407  N = textwidth3
2408  s = '+' + '-'*N + CR   
2409     
2410  print s     
2411     
2412# -----------------------------------------------------------------------------
2413
2414def mkargstr(args, textwidth, argstr = '', level=0):
2415  """ Generate a string containing first textwidth characters of arguments.
2416
2417  USAGE:
2418    mkargstr(args, textwidth, argstr = '', level=0)
2419
2420  DESCRIPTION:
2421    Exactly the same as str(args) possibly followed by truncation,
2422    but faster if args is huge.
2423  """
2424
2425  import types
2426
2427  if level > 10:
2428      # Protect against circular structures
2429      return '...'
2430 
2431  WasTruncated = 0
2432
2433  if not type(args) in [types.TupleType, types.ListType, types.DictType]:
2434    if type(args) == types.StringType:
2435      argstr = argstr + "'"+str(args)+"'"
2436    else:
2437      # Truncate large Numeric arrays before using str()
2438      if type(args) == num.ArrayType:
2439#        if len(args.flat) > textwidth: 
2440#        Changed by Duncan and Nick 21/2/07 .flat has problems with
2441#        non-contigous arrays and ravel is equal to .flat except it
2442#        can work with non-contiguous  arrays
2443        if len(num.ravel(args)) > textwidth:
2444          args = 'Array: ' + str(args.shape)
2445
2446      argstr = argstr + str(args)
2447  else:
2448    if type(args) == types.DictType:
2449      argstr = argstr + "{"
2450      for key in args.keys():
2451        argstr = argstr + mkargstr(key, textwidth, level=level+1) + ": " + \
2452                 mkargstr(args[key], textwidth, level=level+1) + ", "
2453        if len(argstr) > textwidth:
2454          WasTruncated = 1
2455          break
2456      argstr = argstr[:-2]  # Strip off trailing comma     
2457      argstr = argstr + "}"
2458
2459    else:
2460      if type(args) == types.TupleType:
2461        lc = '('
2462        rc = ')'
2463      else:
2464        lc = '['
2465        rc = ']'
2466      argstr = argstr + lc
2467      for arg in args:
2468        argstr = argstr + mkargstr(arg, textwidth, level=level+1) + ', '
2469        if len(argstr) > textwidth:
2470          WasTruncated = 1
2471          break
2472
2473      # Strip off trailing comma and space unless singleton tuple
2474      #
2475      if type(args) == types.TupleType and len(args) == 1:
2476        argstr = argstr[:-1]   
2477      else:
2478        argstr = argstr[:-2]
2479      argstr = argstr + rc
2480
2481  if len(argstr) > textwidth:
2482    WasTruncated = 1
2483
2484  if WasTruncated:
2485    argstr = argstr[:textwidth]+'...'
2486  return(argstr)
2487
2488# -----------------------------------------------------------------------------
2489
2490def test_OK(msg):
2491  """Print OK msg if test is OK.
2492 
2493  USAGE
2494    test_OK(message)
2495  """
2496
2497  import string
2498   
2499  print string.ljust(msg, textwidth4) + ' - OK' 
2500 
2501  #raise StandardError
2502 
2503# -----------------------------------------------------------------------------
2504
2505def test_error(msg):
2506  """Print error if test fails.
2507 
2508  USAGE
2509    test_error(message)
2510  """
2511 
2512  print 'ERROR (caching.test): %s' %msg
2513  print 'Please send this code example and output to '
2514  print 'Ole.Nielsen@anu.edu.au'
2515  print
2516  print
2517 
2518  raise StandardError
2519
2520#-------------------------------------------------------------
2521if __name__ == "__main__":
2522  pass
Note: See TracBrowser for help on using the repository browser.