source: anuga_core/source/anuga/caching/caching.py @ 6230

Last change on this file since 6230 was 6230, checked in by ole, 15 years ago

Fixed issue with caching of different instances of callable objects
as described in changeset:6229
Also added bytecode determination for classes.

File size: 70.4 KB
Line 
1# =============================================================================
2# caching.py - Supervised caching of function results.
3# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4# Australian National University (1999-2003)
5# Geoscience Australia (2003-present)
6#
7#    This program is free software; you can redistribute it and/or modify
8#    it under the terms of the GNU General Public License as published by
9#    the Free Software Foundation; either version 2 of the License, or
10#    (at your option) any later version.
11#
12#    This program is distributed in the hope that it will be useful,
13#    but WITHOUT ANY WARRANTY; without even the implied warranty of
14#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15#    GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16#    for more details.
17#
18#    You should have received a copy of the GNU General Public License
19#    along with this program; if not, write to the Free Software
20#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
21#
22#
23# Contact address: Ole.Nielsen@ga.gov.au
24#
25# Version 1.5.6 February 2002
26# =============================================================================
27 
28"""Module caching.py - Supervised caching of function results.
29
30Public functions:
31
32cache(func,args) -- Cache values returned from func given args.
33cachestat() --      Reports statistics about cache hits and time saved.
34test() --       Conducts a basic test of the caching functionality.
35
36See doc strings of individual functions for detailed documentation.
37"""
38
39# -----------------------------------------------------------------------------
40# Initialisation code
41
42# Determine platform
43#
44from os import getenv
45import types
46
47import os
48if os.name in ['nt', 'dos', 'win32', 'what else?']:
49  unix = 0
50else:
51  unix = 1
52
53import Numeric as num
54
55
56cache_dir = '.python_cache'
57# Make default caching directory name
58#
59if unix:
60  homedir = getenv('INUNDATIONHOME')
61  if homedir is None:
62    homedir = '~'
63  else:
64    homedir = homedir + os.sep + '.cache'
65    # Since homedir will be a group area, individually label the caches
66    user = getenv('LOGNAME')
67    if user is not None:
68      cache_dir += '_' + user
69   
70  CR = '\n'
71else:
72  homedir = 'c:'
73  CR = '\r\n'  #FIXME: Not tested under windows
74 
75cachedir = homedir + os.sep + cache_dir + os.sep
76
77# -----------------------------------------------------------------------------
78# Options directory with default values - to be set by user
79#
80
81options = { 
82  'cachedir': cachedir,  # Default cache directory
83  'maxfiles': 1000000,   # Maximum number of cached files
84  'savestat': 1,         # Log caching info to stats file
85  'verbose': 1,          # Write messages to standard output
86  'bin': 1,              # Use binary format (more efficient)
87  'compression': 1,      # Use zlib compression
88  'bytecode': 1,         # Recompute if bytecode has changed
89  'expire': 0            # Automatically remove files that have been accessed
90                         # least recently
91}
92
93# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
94
95def set_option(key, value):
96  """Function to set values in the options directory.
97
98  USAGE:
99    set_option(key, value)
100
101  ARGUMENTS:
102    key --   Key in options dictionary. (Required)
103    value -- New value for key. (Required)
104
105  DESCRIPTION:
106    Function to set values in the options directory.
107    Raises an exception if key is not in options.
108  """
109
110  if options.has_key(key):
111    options[key] = value
112  else:
113    raise KeyError(key)  # Key not found, raise an exception
114
115# -----------------------------------------------------------------------------
116# Function cache - the main routine
117
118def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
119          verbose=None, compression=None, evaluate=0, test=0, clear=0,
120          return_filename=0):
121  """Supervised caching of function results.
122
123  USAGE:
124    result = cache(func, args, kwargs, dependencies, cachedir, verbose,
125                   compression, evaluate, test, return_filename)
126
127  ARGUMENTS:
128    func --            Function object (Required)
129    args --            Arguments to func (Default: ())
130    kwargs --          Keyword arguments to func (Default: {})   
131    dependencies --    Filenames that func depends on (Default: None)
132    cachedir --        Directory for cache files (Default: options['cachedir'])
133    verbose --         Flag verbose output to stdout
134                       (Default: options['verbose'])
135    compression --     Flag zlib compression (Default: options['compression'])
136    evaluate --        Flag forced evaluation of func (Default: 0)
137    test --            Flag test for cached results (Default: 0)
138    clear --           Flag delete cached results (Default: 0)   
139    return_filename -- Flag return of cache filename (Default: 0)   
140
141  DESCRIPTION:
142    A Python function call of the form
143
144      result = func(arg1,...,argn)
145
146    can be replaced by
147
148      from caching import cache
149      result = cache(func,(arg1,...,argn))
150
151  The latter form returns the same output as the former but reuses cached
152  results if the function has been computed previously in the same context.
153  'result' and the arguments can be simple types, tuples, list, dictionaries or
154  objects, but not unhashable types such as functions or open file objects.
155  The function 'func' may be a member function of an object or a module.
156
157  This type of caching is particularly useful for computationally intensive
158  functions with few frequently used combinations of input arguments. Note that
159  if the inputs or output are very large caching might not save time because
160  disc access may dominate the execution time.
161
162  If the function definition changes after a result has been cached it will be
163  detected by examining the functions bytecode (co_code, co_consts,
164  func_defaults, co_argcount) and it will be recomputed.
165
166  LIMITATIONS:
167    1 Caching uses the apply function and will work with anything that can be
168      pickled, so any limitation in apply or pickle extends to caching.
169    2 A function to be cached should not depend on global variables
170      as wrong results may occur if globals are changed after a result has
171      been cached.
172
173  -----------------------------------------------------------------------------
174  Additional functionality:
175
176  Keyword args
177    Keyword arguments (kwargs) can be added as a dictionary of keyword: value
178    pairs, following the syntax of the built-in function apply().
179    A Python function call of the form
180   
181      result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)   
182
183    is then cached as follows
184
185      from caching import cache
186      result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
187   
188    The default value of kwargs is {} 
189
190  Explicit dependencies:
191    The call
192      cache(func,(arg1,...,argn),dependencies = <list of filenames>)
193    Checks the size, creation time and modification time of each listed file.
194    If any file has changed the function is recomputed and the results stored
195    again.
196
197  Specify caching directory:
198    The call
199      cache(func,(arg1,...,argn), cachedir = <cachedir>)
200    designates <cachedir> where cached data are stored. Use ~ to indicate users
201    home directory - not $HOME. The default is ~/.python_cache on a UNIX
202    platform and c:/.python_cache on a Win platform.
203
204  Silent operation:
205    The call
206      cache(func,(arg1,...,argn),verbose=0)
207    suppresses messages to standard output.
208
209  Compression:
210    The call
211      cache(func,(arg1,...,argn),compression=0)
212    disables compression. (Default: compression=1). If the requested compressed
213    or uncompressed file is not there, it'll try the other version.
214
215  Forced evaluation:
216    The call
217      cache(func,(arg1,...,argn),evaluate=1)
218    forces the function to evaluate even though cached data may exist.
219
220  Testing for presence of cached result:
221    The call
222      cache(func,(arg1,...,argn),test=1)
223    retrieves cached result if it exists, otherwise None. The function will not
224    be evaluated. If both evaluate and test are switched on, evaluate takes
225    precedence.
226    ??NOTE: In case of hash collisions, this may return the wrong result as
227    ??it only checks if *a* cached result is present.
228   
229  Obtain cache filenames:
230    The call   
231      cache(func,(arg1,...,argn),return_filename=1)
232    returns the hashed base filename under which this function and its
233    arguments would be cached
234
235  Clearing cached results:
236    The call
237      cache(func,'clear')
238    clears all cached data for 'func' and
239      cache('clear')
240    clears all cached data.
241 
242    NOTE: The string 'clear' can be passed an *argument* to func using
243      cache(func,('clear',)) or cache(func,tuple(['clear'])).
244
245    New form of clear:
246      cache(func,(arg1,...,argn),clear=1)
247    clears cached data for particular combination func and args
248     
249  """
250
251  # Imports and input checks
252  #
253  import types, time, string
254
255  if not cachedir:
256    cachedir = options['cachedir']
257
258  if verbose == None:  # Do NOT write 'if not verbose:', it could be zero.
259    verbose = options['verbose']
260
261  if compression == None:  # Do NOT write 'if not compression:',
262                           # it could be zero.
263    compression = options['compression']
264
265  # Create cache directory if needed
266  CD = checkdir(cachedir,verbose)
267
268  # Handle the case cache('clear')
269  if type(func) == types.StringType:
270    if string.lower(func) == 'clear':
271      clear_cache(CD,verbose=verbose)
272      return
273
274  # Handle the case cache(func, 'clear')
275  if type(args) == types.StringType:
276    if string.lower(args) == 'clear':
277      clear_cache(CD,func,verbose=verbose)
278      return
279
280  # Force singleton arg into a tuple
281  if type(args) != types.TupleType:
282    args = tuple([args])
283 
284  # Check that kwargs is a dictionary
285  if type(kwargs) != types.DictType:
286    raise TypeError   
287   
288  # Hash arguments (and keyword args) to integer
289  arghash = myhash((args, kwargs))
290 
291  # Get sizes and timestamps for files listed in dependencies.
292  # Force singletons into a tuple.
293  if dependencies and type(dependencies) != types.TupleType \
294                  and type(dependencies) != types.ListType:
295    dependencies = tuple([dependencies])
296  deps = get_depstats(dependencies)
297
298  # Extract function name from func object
299  funcname = get_funcname(func)
300
301  # Create cache filename
302  FN = funcname+'['+`arghash`+']'  # The symbol '(' does not work under unix
303
304  if return_filename:
305    return(FN)
306
307  if clear:
308    for file_type in file_types:
309      file_name = CD+FN+'_'+file_type
310      for fn in [file_name, file_name + '.z']:
311        if os.access(fn, os.F_OK):             
312          if unix:
313            os.remove(fn)
314          else:
315            # FIXME: os.remove doesn't work under windows       
316            os.system('del '+fn)
317          if verbose is True:
318            print 'MESSAGE (caching): File %s deleted' %fn
319        ##else:
320        ##  print '%s was not accessed' %fn
321    return None
322
323
324  #-------------------------------------------------------------------       
325 
326  # Check if previous computation has been cached
327  if evaluate:
328    Retrieved = None  # Force evaluation of func regardless of caching status.
329    reason = 5
330  else:
331    T, FN, Retrieved, reason, comptime, loadtime, compressed = \
332        CacheLookup(CD, FN, func, 
333                    args, kwargs, 
334                    deps, 
335                    verbose, 
336                    compression,
337                    dependencies)
338
339  if not Retrieved:
340    if test:  # Do not attempt to evaluate function
341      T = None
342    else:  # Evaluate function and save to cache
343      if verbose:
344       
345        msg1(funcname, args, kwargs,reason)
346
347      # Remove expired files automatically
348      if options['expire']:
349        DeleteOldFiles(CD,verbose)
350       
351      # Save args before function is evaluated in case
352      # they are modified by function
353      save_args_to_cache(CD,FN,args,kwargs,compression)
354
355      # Execute and time function with supplied arguments
356      t0 = time.time()
357      T = apply(func,args,kwargs)
358      #comptime = round(time.time()-t0)
359      comptime = time.time()-t0
360
361      if verbose:
362        msg2(funcname,args,kwargs,comptime,reason)
363
364      # Save results and estimated loading time to cache
365      loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
366                                       funcname, dependencies, compression)
367      if verbose:
368        msg3(loadtime, CD, FN, deps, compression)
369      compressed = compression
370
371  if options['savestat'] and (not test or Retrieved):
372  ##if options['savestat']:
373    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
374
375  return(T)  # Return results in all cases
376
377# -----------------------------------------------------------------------------
378
379def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
380  """Generate statistics of caching efficiency.
381
382  USAGE:
383    cachestat(sortidx, period, showuser, cachedir)
384
385  ARGUMENTS:
386    sortidx --  Index of field by which lists are (default: 4)
387                Legal values are
388                 0: 'Name'
389                 1: 'Hits'
390                 2: 'CPU'
391                 3: 'Time Saved'
392                 4: 'Gain(%)'
393                 5: 'Size'
394    period --   If set to -1 all available caching history is used.
395                If set 0 only the current month is used (default -1).
396    showuser -- Flag for additional table showing user statistics
397                (default: None).
398    cachedir -- Directory for cache files (default: options['cachedir']).
399
400  DESCRIPTION:
401    Logged caching statistics is converted into summaries of the form
402    --------------------------------------------------------------------------
403    Function Name   Hits   Exec(s)  Cache(s)  Saved(s)   Gain(%)      Size
404    --------------------------------------------------------------------------
405  """
406
407  __cachestat(sortidx, period, showuser, cachedir)
408  return
409
410# -----------------------------------------------------------------------------
411
412# Has mostly been moved to proper unit test.
413# What remains here includes example of the
414# cache statistics form.
415def test(cachedir=None,verbose=0,compression=None):
416  """Test the functionality of caching.
417
418  USAGE:
419    test(verbose)
420
421  ARGUMENTS:
422    verbose --     Flag whether caching will output its statistics (default=0)
423    cachedir --    Directory for cache files (Default: options['cachedir'])
424    compression -- Flag zlib compression (Default: options['compression'])
425  """
426   
427  import string, time
428
429  # Initialise
430  #
431  #import caching
432  #reload(caching)
433
434  if not cachedir:
435    cachedir = options['cachedir']
436
437  if verbose is None:  # Do NOT write 'if not verbose:', it could be zero.
438    verbose = options['verbose']
439 
440  if compression == None:  # Do NOT write 'if not compression:',
441                           # it could be zero.
442    compression = options['compression']
443  else:
444    try:
445      set_option('compression', compression)
446    except:
447      test_error('Set option failed')     
448
449  try:
450    import zlib
451  except:
452    print
453    print '*** Could not find zlib, default to no-compression      ***'
454    print '*** Installing zlib will improve performance of caching ***'
455    print
456    compression = 0       
457    set_option('compression', compression)   
458 
459  print 
460  print_header_box('Testing caching module - please stand by')
461  print   
462
463  # Define a test function to be cached
464  #
465  def f(a,b,c,N,x=0,y='abcdefg'):
466    """f(a,b,c,N)
467       Do something time consuming and produce a complex result.
468    """
469
470    import string
471
472    B = []
473    for n in range(N):
474      s = str(n+2.0/(n + 4.0))+'.a'*10
475      B.append((a,b,c,s,n,x,y))
476    return(B)
477   
478  # Check that default cachedir is OK
479  #     
480  CD = checkdir(cachedir,verbose)   
481   
482   
483  # Make a dependency file
484  #   
485  try:
486    DepFN = CD + 'testfile.tmp'
487    DepFN_wildcard = CD + 'test*.tmp'
488    Depfile = open(DepFN,'w')
489    Depfile.write('We are the knights who say NI!')
490    Depfile.close()
491    test_OK('Wrote file %s' %DepFN)
492  except:
493    test_error('Could not open file %s for writing - check your environment' \
494               % DepFN)
495
496  # Check set_option (and switch stats off
497  #   
498  try:
499    set_option('savestat',0)
500    assert(options['savestat'] == 0)
501    test_OK('Set option')
502  except:
503    test_error('Set option failed')   
504   
505  # Make some test input arguments
506  #
507  N = 5000  #Make N fairly small here
508
509  a = [1,2]
510  b = ('Thou shalt count the number three',4)
511  c = {'Five is right out': 6, (7,8): 9}
512  x = 3
513  y = 'holy hand granate'
514
515  # Test caching
516  #
517  if compression:
518    comprange = 2
519  else:
520    comprange = 1
521
522  for comp in range(comprange):
523 
524    # Evaluate and store
525    #
526    try:
527      T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
528                   verbose=verbose, compression=comp)
529      if comp:                   
530        test_OK('Caching evaluation with compression')
531      else:     
532        test_OK('Caching evaluation without compression')     
533    except:
534      if comp:
535        test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
536      else:
537        test_error('Caching evaluation failed - try caching.test(verbose=1)')
538
539    # Retrieve
540    #                           
541    try:                         
542      T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
543                   compression=comp) 
544
545      if comp:                   
546        test_OK('Caching retrieval with compression')
547      else:     
548        test_OK('Caching retrieval without compression')     
549    except:
550      if comp:
551        test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
552      else:                                     
553        test_error('Caching retrieval failed - try caching.test(verbose=1)')
554
555    # Reference result
556    #   
557    T3 = f(a,b,c,N,x=x,y=y)  # Compute without caching
558   
559    if T1 == T2 and T2 == T3:
560      if comp:
561        test_OK('Basic caching functionality (with compression)')
562      else:
563        test_OK('Basic caching functionality (without compression)')
564    else:
565      test_error('Cached result does not match computed result')
566
567
568  # Test return_filename
569  #   
570  try:
571    FN = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
572                 return_filename=1)   
573    assert(FN[:2] == 'f[')
574    test_OK('Return of cache filename')
575  except:
576    test_error('Return of cache filename failed')
577
578  # Test existence of cachefiles
579 
580  try:
581    (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
582    (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
583    (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
584    test_OK('Presence of cache files')
585    datafile.close()
586    argsfile.close()
587    admfile.close()
588  except:
589    test_error('Expected cache files did not exist') 
590             
591  # Test 'test' function when cache is present
592  #     
593  try:
594    #T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
595    #                   evaluate=1) 
596    T4 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
597    assert(T1 == T4)
598
599    test_OK("Option 'test' when cache file present")
600  except:
601    test_error("Option 'test' when cache file present failed")     
602
603  # Test that 'clear' works
604  #
605  #try:
606  #  cache(f,'clear',verbose=verbose)
607  #  test_OK('Clearing of cache files')
608  #except:
609  #  test_error('Clear does not work')
610  try:
611    cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)   
612    test_OK('Clearing of cache files')
613  except:
614    test_error('Clear does not work') 
615
616 
617
618  # Test 'test' function when cache is absent
619  #     
620  try:
621    T4 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
622    assert(T4 is None)
623    test_OK("Option 'test' when cache absent")
624  except:
625    test_error("Option 'test' when cache absent failed")     
626         
627  # Test dependencies
628  #
629  T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
630               dependencies=DepFN) 
631  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
632               dependencies=DepFN)                     
633                       
634  if T1 == T2:
635    test_OK('Basic dependencies functionality')
636  else:
637    test_error('Dependencies do not work')
638
639  # Test basic wildcard dependency
640  #
641  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
642               dependencies=DepFN_wildcard)                     
643   
644  if T1 == T3:
645    test_OK('Basic dependencies with wildcard functionality')
646  else:
647    test_error('Dependencies with wildcards do not work')
648
649
650  # Test that changed timestamp in dependencies triggers recomputation
651 
652  # Modify dependency file
653  Depfile = open(DepFN,'a')
654  Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
655  Depfile.close()
656 
657  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
658               dependencies=DepFN, test = 1)                     
659 
660  if T3 is None:
661    test_OK('Changed dependencies recognised')
662  else:
663    test_error('Changed dependencies not recognised')   
664 
665  # Test recomputation when dependencies have changed
666  #
667  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
668               dependencies=DepFN)                       
669  if T1 == T3:
670    test_OK('Recomputed value with changed dependencies')
671  else:
672    test_error('Recomputed value with changed dependencies failed')
673
674  # Performance test (with statistics)
675  # Don't really rely on this as it will depend on specific computer.
676  #
677
678  set_option('savestat',1)
679
680  N = 20*N   #Should be large on fast computers...
681  tt = time.time()
682  T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
683  t1 = time.time() - tt
684 
685  tt = time.time()
686  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
687  t2 = time.time() - tt
688 
689  if T1 == T2:
690    if t1 > t2:
691      test_OK('Performance test: relative time saved = %s pct' \
692              %str(round((t1-t2)*100/t1,2)))
693    #else:
694    #  print 'WARNING: Performance a bit low - this could be specific to current platform'
695  else:       
696    test_error('Basic caching failed for new problem')
697           
698  # Test presence of statistics file
699  #
700  try: 
701    DIRLIST = os.listdir(CD)
702    SF = []
703    for FN in DIRLIST:
704      if string.find(FN,statsfile) >= 0:
705        fid = open(CD+FN,'r')
706        fid.close()
707    test_OK('Statistics files present') 
708  except:
709    test_OK('Statistics files cannot be opened')         
710     
711  print_header_box('Show sample output of the caching function:')
712 
713  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
714  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
715  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
716 
717  print_header_box('Show sample output of cachestat():')
718  if unix:
719    cachestat()   
720  else:
721    try:
722      import time
723      t = time.strptime('2030','%Y')
724      cachestat()
725    except: 
726      print 'cachestat() does not work here, because it'
727      print 'relies on time.strptime() which is unavailable in Windows'
728     
729  print
730  test_OK('Caching self test completed')   
731     
732           
733  # Test setoption (not yet implemented)
734  #
735
736 
737#==============================================================================
738# Auxiliary functions
739#==============================================================================
740
741# Import pickler
742# cPickle is used by functions mysave, myload, and compare
743#
744import cPickle  # 10 to 100 times faster than pickle
745pickler = cPickle
746
747# Local immutable constants
748#
749comp_level = 1              # Compression level for zlib.
750                            # comp_level = 1 works well.
751textwidth1 = 16             # Text width of key fields in report forms.
752#textwidth2 = 132            # Maximal width of textual representation of
753textwidth2 = 300            # Maximal width of textual representation of
754                            # arguments.
755textwidth3 = 16             # Initial width of separation lines. Is modified.
756textwidth4 = 50             # Text width in test_OK()
757statsfile  = '.cache_stat'  # Basefilename for cached statistics.
758                            # It will reside in the chosen cache directory.
759
760file_types = ['Result',     # File name extension for cached function results.
761              'Args',       # File name extension for stored function args.
762              'Admin']      # File name extension for administrative info.
763
764Reason_msg = ['OK',         # Verbose reasons for recomputation
765              'No cached result', 
766              'Dependencies have changed', 
767              'Arguments have changed',
768              'Bytecode has changed',
769              'Recomputation was requested by caller',
770              'Cached file was unreadable']             
771             
772# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
773
774def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, 
775                dependencies):
776  """Determine whether cached result exists and return info.
777
778  USAGE:
779    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \ 
780    CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
781                dependencies)
782
783  INPUT ARGUMENTS:
784    CD --            Cache Directory
785    FN --            Suggested cache file name
786    func --          Function object
787    args --          Tuple of arguments
788    kwargs --        Dictionary of keyword arguments   
789    deps --          Dependencies time stamps
790    verbose --       Flag text output
791    compression --   Flag zlib compression
792    dependencies --  Given list of dependencies
793   
794  OUTPUT ARGUMENTS:
795    T --             Cached result if present otherwise None
796    FN --            File name under which new results must be saved
797    Retrieved --     True if a valid cached result was found
798    reason --        0: OK (if Retrieved),
799                     1: No cached result,
800                     2: Dependencies have changed,
801                     3: Arguments have changed
802                     4: Bytecode has changed
803                     5: Recomputation was forced
804                     6: Unreadable file
805    comptime --      Number of seconds it took to computed cachged result
806    loadtime --      Number of seconds it took to load cached result
807    compressed --    Flag (0,1) if cached results were compressed or not
808
809  DESCRIPTION:
810    Determine if cached result exists as follows:
811    Load in saved arguments and bytecode stored under hashed filename.
812    If they are identical to current arguments and bytecode and if dependencies
813    have not changed their time stamp, then return cached result.
814
815    Otherwise return filename under which new results should be cached.
816    Hash collisions are handled recursively by calling CacheLookup again with a
817    modified filename.
818  """
819
820  import time, string, types
821
822  # Assess whether cached result exists - compressed or not.
823  #
824  if verbose:
825    print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
826           %(CD+FN, file_types[0], file_types[1], file_types[2])
827  (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
828  (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
829  (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
830
831  if verbose is True and deps is not None:
832    print 'Caching: Dependencies are', deps.keys()
833
834  if not (argsfile and datafile and admfile) or \
835     not (compressed0 == compressed1 and compressed0 == compressed2):
836    # Cached result does not exist or files were compressed differently
837    #
838    # This will ensure that evaluation will take place unless all files are
839    # present.
840
841    reason = 1
842    return(None,FN,None,reason,None,None,None) #Recompute using same filename
843
844  compressed = compressed0  # Remember if compressed files were actually used
845  datafile.close()
846
847  # Retrieve arguments and adm. info
848  #
849  R, reason = myload(argsfile, compressed)  # The original arguments
850  argsfile.close()
851   
852  if reason > 0:
853      # Recompute using same filename   
854      return(None, FN, None, reason, None, None, None)
855  else:   
856      (argsref, kwargsref) = R
857
858  R, reason = myload(admfile, compressed)
859  admfile.close() 
860
861  if reason > 0:
862    return(None,FN,None,reason,None,None,None) #Recompute using same filename
863
864 
865  depsref  = R[0]  # Dependency statistics
866  comptime = R[1]  # The computation time
867  coderef  = R[2]  # The byte code
868  funcname = R[3]  # The function name
869
870  # Check if dependencies have changed
871  #
872  if dependencies and not compare(depsref, deps):
873    if verbose:
874      print 'MESSAGE (caching.py): Dependencies', dependencies, \
875            'have changed - recomputing'
876    # Don't use cached file - recompute
877    reason = 2
878    return(None, FN, None, reason, None, None, None)
879
880  # Get bytecode from func
881  #
882  bytecode = get_bytecode(func)
883
884  #print compare(argsref,args),   
885  #print compare(kwargsref,kwargs),
886  #print compare(bytecode,coderef)
887
888  # Check if arguments or bytecode have changed
889  if compare(argsref, args) and compare(kwargsref, kwargs) and \
890     (not options['bytecode'] or compare(bytecode, coderef)):
891
892    # Arguments and dependencies match. Get cached results
893    T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
894    if reason > 0:
895        # Recompute using same FN     
896        return(None, FN, None, reason, None, None, None)
897
898    Retrieved = 1
899    reason = 0
900
901    if verbose:
902      msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
903
904      if loadtime >= comptime:
905        print 'WARNING (caching.py): Caching did not yield any gain.'
906        print '                      Consider executing function ',
907        print '('+funcname+') without caching.'
908  else:
909
910    # Non matching arguments or bytecodes signify a hash-collision.
911    # This is resolved by recursive search of cache filenames
912    # until either a matching or an unused filename is found.
913    #
914    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
915        CacheLookup(CD, FN+'x', func, args, kwargs, deps, 
916                    verbose, compression, dependencies)
917
918    # DEBUGGING
919    # if not Retrieved:
920    #   print 'Arguments did not match'
921    # else:
922    #   print 'Match found !'
923   
924    # The real reason is that args or bytecodes have changed.
925    # Not that the recursive seach has found an unused filename
926    if not Retrieved:
927      if not compare(bytecode, coderef):
928        reason = 4 # Bytecode has changed
929      else:   
930        reason = 3 # Arguments have changed
931       
932   
933  return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
934
935# -----------------------------------------------------------------------------
936
937def clear_cache(CD,func=None, verbose=None):
938  """Clear cache for func.
939
940  USAGE:
941     clear(CD, func, verbose)
942
943  ARGUMENTS:
944     CD --       Caching directory (required)
945     func --     Function object (default: None)
946     verbose --  Flag verbose output (default: None)
947
948  DESCRIPTION:
949
950    If func == None, clear everything,
951    otherwise clear only files pertaining to func.
952  """
953
954  import os, re
955   
956  if CD[-1] != os.sep:
957    CD = CD+os.sep
958 
959  if verbose == None:
960    verbose = options['verbose']
961
962  # FIXME: Windows version needs to be tested
963
964  if func:
965    funcname = get_funcname(func)
966    if verbose:
967      print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
968
969    file_names = os.listdir(CD)
970    for file_name in file_names:
971      #RE = re.search('^' + funcname,file_name)  #Inefficient
972      #if RE:
973      if file_name[:len(funcname)] == funcname:
974        if unix:
975          os.remove(CD+file_name)
976        else:
977          os.system('del '+CD+file_name)
978          # FIXME: os.remove doesn't work under windows
979  else:
980    file_names = os.listdir(CD)
981    if len(file_names) > 0:
982      if verbose:
983        print 'MESSAGE (caching.py): Remove the following files:'
984        for file_name in file_names:
985            print file_name
986
987        A = raw_input('Delete (Y/N)[N] ?')
988      else:
989        A = 'Y' 
990       
991      if A == 'Y' or A == 'y':
992        for file_name in file_names:
993          if unix:
994            os.remove(CD+file_name)
995          else:
996            os.system('del '+CD+file_name)
997            # FIXME: os.remove doesn't work under windows
998          #exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
999
1000# -----------------------------------------------------------------------------
1001
1002def DeleteOldFiles(CD,verbose=None):
1003  """Remove expired files
1004
1005  USAGE:
1006    DeleteOldFiles(CD,verbose=None)
1007  """
1008
1009  if verbose == None:
1010    verbose = options['verbose']
1011
1012  maxfiles = options['maxfiles']
1013
1014  # FIXME: Windows version
1015
1016  import os
1017  block = 1000  # How many files to delete per invokation
1018  Files = os.listdir(CD)
1019  numfiles = len(Files)
1020  if not unix: return  # FIXME: Windows case ?
1021
1022  if numfiles > maxfiles:
1023    delfiles = numfiles-maxfiles+block
1024    if verbose:
1025      print 'Deleting '+`delfiles`+' expired files:'
1026      os.system('ls -lur '+CD+'* | head -' + `delfiles`)            # List them
1027    os.system('ls -ur '+CD+'* | head -' + `delfiles` + ' | xargs /bin/rm')
1028                                                                  # Delete them
1029    # FIXME: Replace this with os.listdir and os.remove
1030
1031# -----------------------------------------------------------------------------
1032
1033def save_args_to_cache(CD, FN, args, kwargs, compression):
1034  """Save arguments to cache
1035
1036  USAGE:
1037    save_args_to_cache(CD,FN,args,kwargs,compression)
1038  """
1039
1040  import time, os, sys, types
1041
1042  (argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1043
1044  if argsfile is None:
1045    msg = 'ERROR (caching): Could not open argsfile for writing: %s' %FN
1046    raise IOError, msg
1047
1048  mysave((args,kwargs),argsfile,compression)  # Save args and kwargs to cache
1049  argsfile.close()
1050
1051  # Change access rights if possible
1052  #
1053  #if unix:
1054  #  try:
1055  #    exitcode=os.system('chmod 666 '+argsfile.name)
1056  #  except:
1057  #    pass
1058  #else:
1059  #  pass  # FIXME: Take care of access rights under Windows
1060
1061  return
1062
1063# -----------------------------------------------------------------------------
1064
1065def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1066                          dependencies, compression):
1067  """Save computed results T and admin info to cache
1068
1069  USAGE:
1070    save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1071                          dependencies, compression)
1072  """
1073
1074  import time, os, sys, types
1075
1076  (datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1077  (admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1078
1079  if not datafile:
1080    if verbose:
1081      print 'ERROR (caching): Could not open %s' %datafile.name
1082    raise IOError
1083
1084  if not admfile:
1085    if verbose:
1086      print 'ERROR (caching): Could not open %s' %admfile.name
1087    raise IOError
1088
1089  t0 = time.time()
1090
1091  mysave(T,datafile,compression)  # Save data to cache
1092  datafile.close()
1093  #savetime = round(time.time()-t0,2)
1094  savetime = time.time()-t0 
1095
1096  bytecode = get_bytecode(func)  # Get bytecode from function object
1097  admtup = (deps, comptime, bytecode, funcname)  # Gather admin info
1098
1099  mysave(admtup,admfile,compression)  # Save admin info to cache
1100  admfile.close()
1101
1102  # Change access rights if possible
1103  #
1104  #if unix:
1105  #  try:
1106  #    exitcode=os.system('chmod 666 '+datafile.name)
1107  #    exitcode=os.system('chmod 666 '+admfile.name)
1108  #  except:
1109  #    pass
1110  #else:
1111  #  pass  # FIXME: Take care of access rights under Windows
1112
1113  return(savetime)
1114
1115# -----------------------------------------------------------------------------
1116
1117def load_from_cache(CD,FN,compression):
1118  """Load previously cached data from file FN
1119
1120  USAGE:
1121    load_from_cache(CD,FN,compression)
1122  """
1123
1124  import time
1125
1126  (datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1127  t0 = time.time()
1128  T, reason = myload(datafile,compressed)
1129
1130  loadtime = time.time()-t0
1131  datafile.close() 
1132
1133  return T, loadtime, compressed, reason
1134
1135# -----------------------------------------------------------------------------
1136
1137def myopen(FN,mode,compression=1):
1138  """Open file FN using given mode
1139
1140  USAGE:
1141    myopen(FN,mode,compression=1)
1142
1143  ARGUMENTS:
1144    FN --           File name to be opened
1145    mode --         Open mode (as in open)
1146    compression --  Flag zlib compression
1147
1148  DESCRIPTION:
1149     if compression
1150       Attempt first to open FN + '.z'
1151       If this fails try to open FN
1152     else do the opposite
1153     Return file handle plus info about whether it was compressed or not.
1154  """
1155
1156  import string
1157
1158  # Determine if file exists already (if writing was requested)
1159  # This info is only used to determine if access modes should be set
1160  #
1161  if 'w' in mode or 'a' in mode:
1162    try:
1163      file = open(FN+'.z','r')
1164      file.close()
1165      new_file = 0
1166    except:
1167      try:
1168        file = open(FN,'r') 
1169        file.close()
1170        new_file = 0
1171      except:
1172        new_file = 1
1173  else:
1174    new_file = 0 #Assume it exists if mode was not 'w'
1175 
1176
1177  compressed = 0
1178  if compression:
1179    try:
1180      file = open(FN+'.z',mode)
1181      compressed = 1
1182    except:
1183      try:
1184        file = open(FN,mode)
1185      except:
1186        file = None
1187  else:
1188    try:
1189      file = open(FN,mode)
1190    except:
1191      try:
1192        file = open(FN+'.z',mode)
1193        compressed = 1
1194      except:
1195        file = None
1196
1197  # Now set access rights if it is a new file
1198  #
1199  if file and new_file:
1200    if unix:
1201      exitcode=os.system('chmod 666 '+file.name)
1202    else:
1203      pass  # FIXME: Take care of access rights under Windows
1204
1205  return(file,compressed)
1206
1207# -----------------------------------------------------------------------------
1208
1209def myload(file, compressed):
1210  """Load data from file
1211
1212  USAGE:
1213    myload(file, compressed)
1214  """
1215
1216  reason = 0
1217  try:
1218    if compressed:
1219      import zlib
1220
1221      RsC = file.read()
1222      try:
1223        Rs  = zlib.decompress(RsC)
1224      except:
1225        #  File "./caching.py", line 1032, in load_from_cache
1226        #  T = myload(datafile,compressed)
1227        #  File "./caching.py", line 1124, in myload
1228        #  Rs  = zlib.decompress(RsC)
1229        #  zlib.error: Error -5 while decompressing data
1230        #print 'ERROR (caching): Could not decompress ', file.name
1231        #raise Exception
1232        reason = 6  # Unreadable file
1233        return None, reason 
1234     
1235     
1236      del RsC  # Free up some space
1237      R = pickler.loads(Rs)
1238    else:
1239      try:
1240        R = pickler.load(file)
1241      #except EOFError, e:
1242      except:
1243        #Catch e.g., file with 0 length or corrupted
1244        reason = 6  # Unreadable file
1245        return None, reason
1246     
1247  except MemoryError:
1248    import sys
1249    if options['verbose']:
1250      print 'ERROR (caching): Out of memory while loading %s, aborting' \
1251            %(file.name)
1252
1253    # Raise the error again for now
1254    #
1255    raise MemoryError
1256
1257  return R, reason
1258
1259# -----------------------------------------------------------------------------
1260
1261def mysave(T,file,compression):
1262  """Save data T to file
1263
1264  USAGE:
1265    mysave(T,file,compression)
1266
1267  """
1268
1269  bin = options['bin']
1270
1271  if compression:
1272    try:
1273      import zlib
1274    except:
1275      print
1276      print '*** Could not find zlib ***'
1277      print '*** Try to run caching with compression off ***'
1278      print "*** caching.set_option('compression', 0) ***"
1279      raise Exception
1280     
1281
1282    try:
1283      Ts  = pickler.dumps(T, bin)
1284    except MemoryError:
1285      msg = '****WARNING (caching.py): Could not pickle data for compression.'
1286      msg += ' Try using compression = False'
1287      raise MemoryError, msg
1288    else: 
1289      # Compressed pickling     
1290      TsC = zlib.compress(Ts, comp_level)
1291      file.write(TsC)
1292  else:
1293      # Uncompressed pickling
1294      pickler.dump(T, file, bin)
1295
1296      # FIXME: This may not work on Windoze network drives.
1297      # The error msg is IOError: [Errno 22] Invalid argument
1298      # Testing with small files was OK, though.
1299      # I think this is an OS problem.
1300
1301      # Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1302     
1303# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1304#
1305#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1306#
1307#The following extract is taken from the site http://www.python.org:
1308#
1309#---------------------------------------------------------------------------------------------
1310#exception IOError
1311#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1312#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1313#---------------------------------------------------------------------------------------------
1314#
1315#The error code(s) that accompany exceptions are described at:
1316#http://www.python.org/dev/doc/devel//lib/module-errno.html
1317#
1318#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1319#
1320#exceptions.IOError invalid argument Errno 22
1321       
1322      #try:
1323      #  pickler.dump(T,file,bin)
1324      #except IOError, e:
1325      #  print e
1326      #  msg = 'Could not store to %s, bin=%s' %(file, bin)
1327      #  raise msg
1328     
1329
1330# -----------------------------------------------------------------------------
1331
1332   
1333def myhash(T, ids=None):
1334  """Compute hashed integer from a range of inputs.
1335  If T is not hashable being e.g. a tuple T, myhash will recursively
1336  hash the values individually
1337
1338  USAGE:
1339    myhash(T)
1340
1341  ARGUMENTS:
1342    T -- Anything
1343  """
1344
1345  from types import TupleType, ListType, DictType, InstanceType 
1346   
1347  if type(T) in [TupleType, ListType, DictType, InstanceType]: 
1348      # Keep track of unique id's to protect against infinite recursion
1349      if ids is None: ids = []
1350
1351      # Check if T has already been encountered
1352      i = id(T) 
1353 
1354      if i in ids:
1355          return 0 # T has been hashed already     
1356      else:
1357          ids.append(i)
1358   
1359
1360   
1361  # Start hashing 
1362 
1363  # On some architectures None, False and True gets different hash values
1364  if T is None:
1365      return(-1)
1366  if T is False:
1367      return(0)
1368  if T is True:
1369      return(1)
1370
1371  # Get hash values for hashable entries
1372  if type(T) in [TupleType, ListType]:
1373      hvals = []
1374      for t in T:
1375          h = myhash(t, ids)
1376          hvals.append(h)
1377      val = hash(tuple(hvals))
1378  elif type(T) == DictType:
1379      # Make dictionary ordering unique 
1380      I = T.items()
1381      I.sort()   
1382      val = myhash(I, ids)
1383  elif type(T) == num.ArrayType:
1384      T = num.array(T) # Ensure array is contiguous
1385
1386      # Use mean value for efficiency
1387      val = hash(num.average(T.flat))
1388  elif type(T) == InstanceType:
1389      # Use the attribute values
1390      val = myhash(T.__dict__, ids)
1391  else:
1392      try:
1393          val = hash(T)
1394      except:
1395          val = 1
1396
1397  return(val)
1398
1399
1400
1401def compare(A, B, ids=None):
1402    """Safe comparison of general objects
1403
1404    USAGE:
1405      compare(A,B)
1406
1407    DESCRIPTION:
1408      Return 1 if A and B they are identical, 0 otherwise
1409    """
1410
1411    from types import TupleType, ListType, DictType, InstanceType
1412   
1413    # Keep track of unique id's to protect against infinite recursion
1414    if ids is None: ids = {}
1415
1416    # Check if T has already been encountered
1417    iA = id(A) 
1418    iB = id(B)     
1419   
1420    if ids.has_key((iA, iB)):
1421        # A and B have been compared already
1422        #print 'Found', (iA, iB), A, B
1423        return ids[(iA, iB)]
1424    else:
1425        ids[(iA, iB)] = True
1426   
1427   
1428    # Check if arguments are of same type
1429    if type(A) != type(B):
1430        return False
1431       
1432 
1433    # Compare recursively based on argument type
1434    if type(A) in [TupleType, ListType]:
1435        N = len(A)
1436        if len(B) != N: 
1437            identical = False
1438        else:
1439            identical = True
1440            for i in range(N):
1441                if not compare(A[i], B[i], ids): 
1442                    identical = False; break
1443                   
1444    elif type(A) == DictType:
1445        if len(A) != len(B):
1446            identical = False
1447        else:                       
1448            # Make dictionary ordering unique
1449            a = A.items(); a.sort()   
1450            b = B.items(); b.sort()
1451           
1452            identical = compare(a, b, ids)
1453           
1454    elif type(A) == num.ArrayType:
1455        # Use element by element comparison
1456        identical = num.alltrue(A==B)
1457
1458    elif type(A) == InstanceType:
1459        # Take care of special case where elements are instances           
1460        # Base comparison on attributes     
1461        identical = compare(A.__dict__, 
1462                            B.__dict__, 
1463                            ids)
1464    else:       
1465        # Fall back to general code
1466        try:
1467            identical = (A == B)
1468        except:
1469            import pickle
1470            # Use pickle to compare data
1471            # The native pickler must be used
1472            # since the faster cPickle does not
1473            # guarantee a unique translation           
1474            try:
1475                identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
1476            except:
1477                identical = False
1478
1479    # Record result of comparison and return           
1480    ids[(iA, iB)] = identical
1481   
1482    return(identical)
1483
1484   
1485# -----------------------------------------------------------------------------
1486
1487def nospace(s):
1488  """Replace spaces in string s with underscores
1489
1490  USAGE:
1491    nospace(s)
1492
1493  ARGUMENTS:
1494    s -- string
1495  """
1496
1497  import string
1498
1499  newstr = ''
1500  for i in range(len(s)):
1501    if s[i] == ' ':
1502      newstr = newstr+'_'
1503    else:
1504      newstr = newstr+s[i]
1505
1506  return(newstr)
1507
1508# -----------------------------------------------------------------------------
1509
1510def get_funcname(func):
1511  """Retrieve name of function object func (depending on its type)
1512
1513  USAGE:
1514    get_funcname(func)
1515  """
1516
1517  import types, string
1518
1519  if type(func) == types.FunctionType:
1520    funcname = func.func_name
1521  elif type(func) == types.BuiltinFunctionType:
1522    funcname = func.__name__
1523  else:
1524    tab = string.maketrans("<>'","   ")
1525    tmp = string.translate(`func`,tab)
1526    tmp = string.split(tmp)
1527    funcname = string.join(tmp)
1528
1529    # Truncate memory address as in
1530    # class __main__.Dummy at 0x00A915D0
1531    index = funcname.find('at 0x')
1532    if index >= 0:
1533      funcname = funcname[:index+5] # Keep info that there is an address
1534
1535  funcname = nospace(funcname)
1536  return(funcname)
1537
1538# -----------------------------------------------------------------------------
1539
1540def get_bytecode(func):
1541  """ Get bytecode from function object.
1542
1543  USAGE:
1544    get_bytecode(func)
1545  """
1546
1547  import types
1548
1549  if type(func) == types.FunctionType:
1550    return get_func_code_details(func)
1551  elif type(func) == types.MethodType:
1552    return get_func_code_details(func.im_func)
1553  elif type(func) == types.InstanceType:   
1554    if callable(func):
1555      # Get bytecode from __call__ method
1556      bytecode = get_func_code_details(func.__call__.im_func)
1557     
1558      # Add hash value of object to detect attribute changes
1559      return bytecode + (myhash(func),) 
1560    else:
1561      msg = 'Instance %s was passed into caching in the role of a function ' % str(func)
1562      msg = ' but it was not callable.'
1563      raise Exception, msg
1564  elif type(func) in [types.BuiltinFunctionType, types.BuiltinMethodType]:     
1565    # Built-in functions are assumed not to change 
1566    return None, 0, 0, 0
1567  elif type(func) == types.ClassType:
1568      # Get bytecode from __init__ method
1569      bytecode = get_func_code_details(func.__init__.im_func)   
1570      return bytecode     
1571  else:
1572    msg = 'Unknown function type: %s' % type(func)
1573    raise Exception, msg
1574
1575
1576 
1577 
1578def get_func_code_details(func):
1579  """Extract co_code, co_consts, co_argcount, func_defaults
1580  """
1581 
1582  bytecode = func.func_code.co_code
1583  consts = func.func_code.co_consts
1584  argcount = func.func_code.co_argcount   
1585  defaults = func.func_defaults       
1586 
1587  return bytecode, consts, argcount, defaults 
1588
1589# -----------------------------------------------------------------------------
1590
1591def get_depstats(dependencies):
1592  """ Build dictionary of dependency files and their size, mod. time and ctime.
1593
1594  USAGE:
1595    get_depstats(dependencies):
1596  """
1597
1598  import types
1599
1600  #print 'Caching DEBUG: Dependencies are', dependencies
1601  d = {}
1602  if dependencies:
1603
1604    #Expand any wildcards
1605    import glob
1606    expanded_dependencies = []
1607    for FN in dependencies:
1608      expanded_FN = glob.glob(FN)
1609
1610      if expanded_FN == []:
1611        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1612        raise Exception, errmsg     
1613
1614      expanded_dependencies += expanded_FN
1615
1616   
1617    for FN in expanded_dependencies:
1618      if not type(FN) == types.StringType:
1619        errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1620        errmsg += '                   Dependency given: %s' %FN
1621        raise Exception, errmsg     
1622      if not os.access(FN,os.F_OK):
1623        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1624        raise Exception, errmsg
1625      (size,atime,mtime,ctime) = filestat(FN)
1626
1627      # We don't use atime because that would cause recomputation every time.
1628      # We don't use ctime because that is irrelevant and confusing for users.
1629      d.update({FN : (size,mtime)})
1630
1631  return(d)
1632
1633# -----------------------------------------------------------------------------
1634
1635def filestat(FN):
1636  """A safe wrapper using os.stat to get basic file statistics
1637     The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1638
1639  USAGE:
1640    filestat(FN)
1641
1642  DESCRIPTION:
1643     Must compile Python with
1644     CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1645              configure
1646     as given in section 8.1.1 Large File Support in the Libray Reference
1647  """
1648
1649  import os, time
1650
1651  try:
1652    stats = os.stat(FN)
1653    size  = stats[6]
1654    atime = stats[7]
1655    mtime = stats[8]
1656    ctime = stats[9]
1657  except:
1658
1659    # Hack to get the results anyway (works only on Unix at the moment)
1660    #
1661    print 'Hack to get os.stat when files are too large'
1662
1663    if unix:
1664      tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1665      # Unique filename, FIXME: Use random number
1666
1667      # Get size and access time (atime)
1668      #
1669      exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1670      (size,atime) = get_lsline(tmp)
1671
1672      # Get size and modification time (mtime)
1673      #
1674      exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1675      (size,mtime) = get_lsline(tmp)
1676
1677      # Get size and ctime
1678      #
1679      exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1680      (size,ctime) = get_lsline(tmp)
1681
1682      try:
1683        exitcode=os.system('rm '+tmp)
1684        # FIXME: Gives error if file doesn't exist
1685      except:
1686        pass
1687    else:
1688      pass
1689      raise Exception  # FIXME: Windows case
1690
1691  return(long(size),atime,mtime,ctime)
1692
1693# -----------------------------------------------------------------------------
1694
1695def get_lsline(FN):
1696  """get size and time for filename
1697
1698  USAGE:
1699    get_lsline(file_name)
1700
1701  DESCRIPTION:
1702    Read in one line 'ls -la' item from file (generated by filestat) and
1703    convert time to seconds since epoch. Return file size and time.
1704  """
1705
1706  import string, time
1707
1708  f = open(FN,'r')
1709  info = f.read()
1710  info = string.split(info)
1711
1712  size = info[4]
1713  week = info[5]
1714  mon  = info[6]
1715  day  = info[7]
1716  hour = info[8]
1717  year = info[9]
1718
1719  str = week+' '+mon+' '+day+' '+hour+' '+year
1720  timetup = time.strptime(str)
1721  t = time.mktime(timetup)
1722  return(size, t)
1723
1724# -----------------------------------------------------------------------------
1725
1726def checkdir(CD,verbose=None, warn=False):
1727  """Check or create caching directory
1728
1729  USAGE:
1730    checkdir(CD,verbose):
1731
1732  ARGUMENTS:
1733    CD -- Directory
1734    verbose -- Flag verbose output (default: None)
1735
1736  DESCRIPTION:
1737    If CD does not exist it will be created if possible
1738  """
1739
1740  import os
1741  import os.path
1742
1743  if CD[-1] != os.sep: 
1744    CD = CD + os.sep  # Add separator for directories
1745
1746  CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1747  if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1748    try:
1749      exitcode=os.mkdir(CD)
1750
1751      # Change access rights if possible
1752      #
1753      if unix:
1754        exitcode=os.system('chmod 777 '+CD)
1755      else:
1756        pass  # FIXME: What about acces rights under Windows?
1757      if verbose: print 'MESSAGE: Directory', CD, 'created.'
1758    except:
1759      if warn is True:
1760        print 'WARNING: Directory', CD, 'could not be created.'
1761      if unix:
1762        CD = '/tmp/'
1763      else:
1764        CD = 'C:' 
1765      if warn is True:
1766        print 'Using directory %s instead' %CD
1767
1768  return(CD)
1769
1770checkdir(cachedir, warn=True)
1771
1772#==============================================================================
1773# Statistics
1774#==============================================================================
1775
1776def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
1777                 compression):
1778  """Add stats entry
1779
1780  USAGE:
1781    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1782
1783  DESCRIPTION:
1784    Make one entry in the stats file about one cache hit recording time saved
1785    and other statistics. The data are used by the function cachestat.
1786  """
1787
1788  import os, time
1789
1790  try:
1791    TimeTuple = time.localtime(time.time())
1792    extension = time.strftime('%b%Y',TimeTuple)
1793    SFN = CD+statsfile+'.'+extension
1794    #statfile = open(SFN,'a')
1795    (statfile, dummy) = myopen(SFN,'a',compression=0)
1796
1797    # Change access rights if possible
1798    #
1799    #if unix:
1800    #  try:
1801    #    exitcode=os.system('chmod 666 '+SFN)
1802    #  except:
1803    #    pass
1804  except:
1805    print 'Warning: Stat file could not be opened'
1806
1807  try:
1808    if os.environ.has_key('USER'):
1809      user = os.environ['USER']
1810    else:
1811      user = 'Nobody'
1812
1813    date = time.asctime(TimeTuple)
1814
1815    if Retrieved:
1816      hit = '1'
1817    else:
1818      hit = '0'
1819
1820    # Get size of result file
1821    #   
1822    if compression:
1823      stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1824    else:
1825      stats = os.stat(CD+FN+'_'+file_types[0])
1826 
1827    if stats: 
1828      size = stats[6]
1829    else:
1830      size = -1  # Error condition, but don't crash. This is just statistics 
1831
1832    # Build entry
1833   
1834    entry = date             + ',' +\
1835            user             + ',' +\
1836            FN               + ',' +\
1837            str(int(size))   + ',' +\
1838            str(compression) + ',' +\
1839            hit              + ',' +\
1840            str(reason)      + ',' +\
1841            str(round(comptime,4)) + ',' +\
1842            str(round(loadtime,4)) +\
1843            CR
1844           
1845    statfile.write(entry)
1846    statfile.close()
1847  except:
1848    print 'Warning: Writing of stat file failed'
1849
1850# -----------------------------------------------------------------------------
1851
1852# FIXME: should take cachedir as an optional arg
1853#
1854def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1855  """  List caching statistics.
1856
1857  USAGE:
1858    __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1859
1860      Generate statistics of caching efficiency.
1861      The parameter sortidx determines by what field lists are sorted.
1862      If the optional keyword period is set to -1,
1863      all available caching history is used.
1864      If it is 0 only the current month is used.
1865      Future versions will include more than one month....
1866      OMN 20/8/2000
1867  """
1868
1869  import os
1870  import os.path
1871  from string import split, rstrip, find
1872  from time import strptime, localtime, strftime, mktime, ctime
1873
1874  # sortidx = 4    # Index into Fields[1:]. What to sort by.
1875
1876  Fields = ['Name', 'Hits', 'Exec(s)', \
1877            'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1878  Widths = [25,7,9,9,9,9,13]
1879  #Types = ['s','d','d','d','d','.2f','d']
1880  Types = ['s','d','.2f','.2f','.2f','.2f','d'] 
1881
1882  Dictnames = ['Function', 'User']
1883
1884  if not cachedir:
1885    cachedir = checkdir(options['cachedir'])
1886
1887  SD = os.path.expanduser(cachedir)  # Expand ~ or ~user in pathname
1888
1889  if period == -1:  # Take all available stats
1890    SFILENAME = statsfile
1891  else:  # Only stats from current month 
1892       # MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1893    TimeTuple = localtime(time())
1894    extension = strftime('%b%Y',TimeTuple)
1895    SFILENAME = statsfile+'.'+extension
1896
1897  DIRLIST = os.listdir(SD)
1898  SF = []
1899  for FN in DIRLIST:
1900    if find(FN,SFILENAME) >= 0:
1901      SF.append(FN)
1902
1903  blocksize = 15000000
1904  total_read = 0
1905  total_hits = 0
1906  total_discarded = 0
1907  firstday = mktime(strptime('2030','%Y'))
1908             # FIXME: strptime don't exist in WINDOWS ?
1909  lastday = 0
1910
1911  FuncDict = {}
1912  UserDict = {}
1913  for FN in SF:
1914    input = open(SD+FN,'r')
1915    print 'Reading file ', SD+FN
1916
1917    while 1:
1918      A = input.readlines(blocksize)
1919      if len(A) == 0: break
1920      total_read = total_read + len(A)
1921      for record in A:
1922        record = tuple(split(rstrip(record),','))
1923        #print record, len(record)
1924
1925        if len(record) == 9:
1926          timestamp = record[0]
1927       
1928          try:
1929            t = mktime(strptime(timestamp))
1930          except:
1931            total_discarded = total_discarded + 1         
1932            continue   
1933             
1934          if t > lastday:
1935            lastday = t
1936          if t < firstday:
1937            firstday = t
1938
1939          user     = record[1]
1940          func     = record[2]
1941
1942          # Strip hash-stamp off
1943          #
1944          i = find(func,'[')
1945          func = func[:i]
1946
1947          size        = float(record[3])
1948
1949          # Compression kepword can be Boolean
1950          if record[4] in ['True', '1']:
1951            compression = 1
1952          elif record[4] in ['False', '0']: 
1953            compression = 0
1954          else:
1955            print 'Unknown value of compression', record[4]
1956            print record
1957            total_discarded = total_discarded + 1           
1958            continue
1959
1960          #compression = int(record[4]) # Can be Boolean
1961          hit         = int(record[5])
1962          reason      = int(record[6])   # Not used here   
1963          cputime     = float(record[7])
1964          loadtime    = float(record[8])
1965
1966          if hit:
1967            total_hits = total_hits + 1
1968            saving = cputime-loadtime
1969
1970            if cputime != 0:
1971              rel_saving = round(100.0*saving/cputime,2)
1972            else:
1973              #rel_saving = round(1.0*saving,2)
1974              rel_saving = 100.0 - round(1.0*saving,2)  # A bit of a hack
1975
1976            info = [1,cputime,loadtime,saving,rel_saving,size]
1977
1978            UpdateDict(UserDict,user,info)
1979            UpdateDict(FuncDict,func,info)
1980          else:
1981            pass #Stats on recomputations and their reasons could go in here
1982             
1983        else:
1984          #print 'Record discarded'
1985          #print record
1986          total_discarded = total_discarded + 1
1987
1988    input.close()
1989
1990  # Compute averages of all sums and write list
1991  #
1992
1993  if total_read == 0:
1994    printline(Widths,'=')
1995    print 'CACHING STATISTICS: No valid records read'
1996    printline(Widths,'=')
1997    return
1998
1999  print
2000  printline(Widths,'=')
2001  print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
2002  printline(Widths,'=')
2003  #print '  Period:', ctime(firstday), 'to', ctime(lastday)
2004  print '  Total number of valid records', total_read
2005  print '  Total number of discarded records', total_discarded
2006  print '  Total number of hits', total_hits
2007  print
2008
2009  print '  Fields', Fields[2:], 'are averaged over number of hits'
2010  print '  Time is measured in seconds and size in bytes'
2011  print '  Tables are sorted by', Fields[1:][sortidx]
2012
2013  # printline(Widths,'-')
2014
2015  if showuser:
2016    Dictionaries = [FuncDict, UserDict]
2017  else:
2018    Dictionaries = [FuncDict]
2019
2020  i = 0
2021  for Dict in Dictionaries:
2022    for key in Dict.keys():
2023      rec = Dict[key]
2024      for n in range(len(rec)):
2025        if n > 0:
2026          rec[n] = round(1.0*rec[n]/rec[0],2)
2027      Dict[key] = rec
2028
2029    # Sort and output
2030    #
2031    keylist = SortDict(Dict,sortidx)
2032
2033    # Write Header
2034    #
2035    print
2036    #print Dictnames[i], 'statistics:'; i=i+1
2037    printline(Widths,'-')
2038    n = 0
2039    for s in Fields:
2040      if s == Fields[0]:  # Left justify
2041        s = Dictnames[i] + ' ' + s; i=i+1
2042        exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
2043      else:
2044        exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
2045    print
2046    printline(Widths,'-')
2047
2048    # Output Values
2049    #
2050    for key in keylist:
2051      rec = Dict[key]
2052      n = 0
2053      if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
2054      exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
2055      for val in rec:
2056        exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
2057      print
2058    print
2059
2060#==============================================================================
2061# Auxiliary stats functions
2062#==============================================================================
2063
2064def UpdateDict(Dict,key,info):
2065  """Update dictionary by adding new values to existing.
2066
2067  USAGE:
2068    UpdateDict(Dict,key,info)
2069  """
2070
2071  if Dict.has_key(key):
2072    dinfo = Dict[key]
2073    for n in range(len(dinfo)):
2074      dinfo[n] = info[n] + dinfo[n]
2075  else:
2076    dinfo = info[:]  # Make a copy of info list
2077
2078  Dict[key] = dinfo
2079  return Dict
2080
2081# -----------------------------------------------------------------------------
2082
2083def SortDict(Dict,sortidx=0):
2084  """Sort dictionary
2085
2086  USAGE:
2087    SortDict(Dict,sortidx):
2088
2089  DESCRIPTION:
2090    Sort dictionary of lists according field number 'sortidx'
2091  """
2092
2093  import types
2094
2095  sortlist  = []
2096  keylist = Dict.keys()
2097  for key in keylist:
2098    rec = Dict[key]
2099    if not type(rec) in [types.ListType, types.TupleType]:
2100      rec = [rec]
2101
2102    if sortidx > len(rec)-1:
2103      if options['verbose']:
2104        print 'ERROR: Sorting index to large, sortidx = ', sortidx
2105      raise IndexError
2106
2107    val = rec[sortidx]
2108    sortlist.append(val)
2109
2110  A = map(None,sortlist,keylist)
2111  A.sort()
2112  keylist = map(lambda x: x[1], A)  # keylist sorted by sortidx
2113
2114  return(keylist)
2115
2116# -----------------------------------------------------------------------------
2117
2118def printline(Widths,char):
2119  """Print textline in fixed field.
2120
2121  USAGE:
2122    printline(Widths,char)
2123  """
2124
2125  s = ''
2126  for n in range(len(Widths)):
2127    s = s+Widths[n]*char
2128    if n > 0:
2129      s = s+char
2130
2131  print s
2132
2133#==============================================================================
2134# Messages
2135#==============================================================================
2136
2137def msg1(funcname,args,kwargs,reason):
2138  """Message 1
2139
2140  USAGE:
2141    msg1(funcname,args,kwargs,reason):
2142  """
2143
2144  import string
2145  #print 'MESSAGE (caching.py): Evaluating function', funcname,
2146
2147  print_header_box('Evaluating function %s' %funcname)
2148 
2149  msg7(args, kwargs)
2150  msg8(reason) 
2151 
2152  print_footer()
2153 
2154  #
2155  # Old message
2156  #
2157  #args_present = 0
2158  #if args:
2159  #  if len(args) == 1:
2160  #    print 'with argument', mkargstr(args[0], textwidth2),
2161  #  else:
2162  #    print 'with arguments', mkargstr(args, textwidth2),
2163  #  args_present = 1     
2164  #   
2165  #if kwargs:
2166  #  if args_present:
2167  #    word = 'and'
2168  #  else:
2169  #    word = 'with'
2170  #     
2171  #  if len(kwargs) == 1:
2172  #    print word + ' keyword argument', mkargstr(kwargs, textwidth2)
2173  #  else:
2174  #    print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
2175  #  args_present = 1           
2176  #else:
2177  #  print    # Newline when no keyword args present
2178  #       
2179  #if not args_present:   
2180  #  print '',  # Default if no args or kwargs present
2181   
2182   
2183
2184# -----------------------------------------------------------------------------
2185
2186def msg2(funcname,args,kwargs,comptime,reason):
2187  """Message 2
2188
2189  USAGE:
2190    msg2(funcname,args,kwargs,comptime,reason)
2191  """
2192
2193  import string
2194
2195  #try:
2196  #  R = Reason_msg[reason]
2197  #except:
2198  #  R = 'Unknown reason' 
2199 
2200  #print_header_box('Caching statistics (storing) - %s' %R)
2201  print_header_box('Caching statistics (storing)') 
2202 
2203  msg6(funcname,args,kwargs)
2204  msg8(reason)
2205
2206  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2207
2208# -----------------------------------------------------------------------------
2209
2210def msg3(savetime, CD, FN, deps,compression):
2211  """Message 3
2212
2213  USAGE:
2214    msg3(savetime, CD, FN, deps,compression)
2215  """
2216
2217  import string
2218  print string.ljust('| Loading time:', textwidth1) + str(round(savetime,2)) + \
2219                     ' seconds (estimated)'
2220  msg5(CD,FN,deps,compression)
2221
2222# -----------------------------------------------------------------------------
2223
2224def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
2225  """Message 4
2226
2227  USAGE:
2228    msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
2229  """
2230
2231  import string
2232
2233  print_header_box('Caching statistics (retrieving)')
2234 
2235  msg6(funcname,args,kwargs)
2236  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2237  print string.ljust('| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
2238  print string.ljust('| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
2239        ' seconds'
2240  msg5(CD,FN,deps,compression)
2241
2242# -----------------------------------------------------------------------------
2243
2244def msg5(CD,FN,deps,compression):
2245  """Message 5
2246
2247  USAGE:
2248    msg5(CD,FN,deps,compression)
2249
2250  DESCRIPTION:
2251   Print dependency stats. Used by msg3 and msg4
2252  """
2253
2254  import os, time, string
2255
2256  print '|'
2257  print string.ljust('| Caching dir: ', textwidth1) + CD
2258
2259  if compression:
2260    suffix = '.z'
2261    bytetext = 'bytes, compressed'
2262  else:
2263    suffix = ''
2264    bytetext = 'bytes'
2265
2266  for file_type in file_types:
2267    file_name = FN + '_' + file_type + suffix
2268    print string.ljust('| ' + file_type + ' file: ', textwidth1) + file_name,
2269    stats = os.stat(CD+file_name)
2270    print '('+ str(stats[6]) + ' ' + bytetext + ')'
2271
2272  print '|'
2273  if len(deps) > 0:
2274    print '| Dependencies:  '
2275    dependencies  = deps.keys()
2276    dlist = []; maxd = 0
2277    tlist = []; maxt = 0
2278    slist = []; maxs = 0
2279    for d in dependencies:
2280      stats = deps[d]
2281      t = time.ctime(stats[1])
2282      s = str(stats[0])
2283      #if s[-1] == 'L':
2284      #  s = s[:-1]  # Strip rightmost 'long integer' L off.
2285      #              # FIXME: Unnecessary in versions later than 1.5.2
2286
2287      if len(d) > maxd: maxd = len(d)
2288      if len(t) > maxt: maxt = len(t)
2289      if len(s) > maxs: maxs = len(s)
2290      dlist.append(d)
2291      tlist.append(t)
2292      slist.append(s)
2293
2294    for n in range(len(dlist)):
2295      d = string.ljust(dlist[n]+':', maxd+1)
2296      t = string.ljust(tlist[n], maxt)
2297      s = string.rjust(slist[n], maxs)
2298
2299      print '| ', d, t, ' ', s, 'bytes'
2300  else:
2301    print '| No dependencies'
2302  print_footer()
2303
2304# -----------------------------------------------------------------------------
2305
2306def msg6(funcname,args,kwargs):
2307  """Message 6
2308
2309  USAGE:
2310    msg6(funcname,args,kwargs)
2311  """
2312
2313  import string
2314  print string.ljust('| Function:', textwidth1) + funcname
2315
2316  msg7(args, kwargs)
2317 
2318# -----------------------------------------------------------------------------   
2319
2320def msg7(args, kwargs):
2321  """Message 7
2322 
2323  USAGE:
2324    msg7(args, kwargs):
2325  """
2326 
2327  import string
2328 
2329  args_present = 0 
2330  if args:
2331    if len(args) == 1:
2332      print string.ljust('| Argument:', textwidth1) + mkargstr(args[0], \
2333                         textwidth2)
2334    else:
2335      print string.ljust('| Arguments:', textwidth1) + \
2336            mkargstr(args, textwidth2)
2337    args_present = 1
2338           
2339  if kwargs:
2340    if len(kwargs) == 1:
2341      print string.ljust('| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
2342                         textwidth2)
2343    else:
2344      print string.ljust('| Keyword Args:', textwidth1) + \
2345            mkargstr(kwargs, textwidth2)
2346    args_present = 1
2347
2348  if not args_present:               
2349    print '| No arguments' # Default if no args or kwargs present
2350
2351# -----------------------------------------------------------------------------
2352
2353def msg8(reason):
2354  """Message 8
2355 
2356  USAGE:
2357    msg8(reason):
2358  """
2359 
2360  import string
2361   
2362  try:
2363    R = Reason_msg[reason]
2364  except:
2365    R = 'Unknown' 
2366 
2367  print string.ljust('| Reason:', textwidth1) + R
2368   
2369# -----------------------------------------------------------------------------
2370
2371def print_header_box(line):
2372  """Print line in a nice box.
2373 
2374  USAGE:
2375    print_header_box(line)
2376
2377  """
2378  global textwidth3
2379
2380  import time
2381
2382  time_stamp = time.ctime(time.time())
2383  line = time_stamp + '. ' + line
2384   
2385  N = len(line) + 1
2386  s = '+' + '-'*N + CR
2387
2388  print s + '| ' + line + CR + s,
2389
2390  textwidth3 = N
2391
2392# -----------------------------------------------------------------------------
2393   
2394def print_footer():
2395  """Print line same width as that of print_header_box.
2396  """
2397 
2398  N = textwidth3
2399  s = '+' + '-'*N + CR   
2400     
2401  print s     
2402     
2403# -----------------------------------------------------------------------------
2404
2405def mkargstr(args, textwidth, argstr = '', level=0):
2406  """ Generate a string containing first textwidth characters of arguments.
2407
2408  USAGE:
2409    mkargstr(args, textwidth, argstr = '', level=0)
2410
2411  DESCRIPTION:
2412    Exactly the same as str(args) possibly followed by truncation,
2413    but faster if args is huge.
2414  """
2415
2416  import types
2417
2418  if level > 10:
2419      # Protect against circular structures
2420      return '...'
2421 
2422  WasTruncated = 0
2423
2424  if not type(args) in [types.TupleType, types.ListType, types.DictType]:
2425    if type(args) == types.StringType:
2426      argstr = argstr + "'"+str(args)+"'"
2427    else:
2428      # Truncate large Numeric arrays before using str()
2429      if type(args) == num.ArrayType:
2430#        if len(args.flat) > textwidth: 
2431#        Changed by Duncan and Nick 21/2/07 .flat has problems with
2432#        non-contigous arrays and ravel is equal to .flat except it
2433#        can work with non-contiguous  arrays
2434        if len(num.ravel(args)) > textwidth:
2435          args = 'Array: ' + str(args.shape)
2436
2437      argstr = argstr + str(args)
2438  else:
2439    if type(args) == types.DictType:
2440      argstr = argstr + "{"
2441      for key in args.keys():
2442        argstr = argstr + mkargstr(key, textwidth, level=level+1) + ": " + \
2443                 mkargstr(args[key], textwidth, level=level+1) + ", "
2444        if len(argstr) > textwidth:
2445          WasTruncated = 1
2446          break
2447      argstr = argstr[:-2]  # Strip off trailing comma     
2448      argstr = argstr + "}"
2449
2450    else:
2451      if type(args) == types.TupleType:
2452        lc = '('
2453        rc = ')'
2454      else:
2455        lc = '['
2456        rc = ']'
2457      argstr = argstr + lc
2458      for arg in args:
2459        argstr = argstr + mkargstr(arg, textwidth, level=level+1) + ', '
2460        if len(argstr) > textwidth:
2461          WasTruncated = 1
2462          break
2463
2464      # Strip off trailing comma and space unless singleton tuple
2465      #
2466      if type(args) == types.TupleType and len(args) == 1:
2467        argstr = argstr[:-1]   
2468      else:
2469        argstr = argstr[:-2]
2470      argstr = argstr + rc
2471
2472  if len(argstr) > textwidth:
2473    WasTruncated = 1
2474
2475  if WasTruncated:
2476    argstr = argstr[:textwidth]+'...'
2477  return(argstr)
2478
2479# -----------------------------------------------------------------------------
2480
2481def test_OK(msg):
2482  """Print OK msg if test is OK.
2483 
2484  USAGE
2485    test_OK(message)
2486  """
2487
2488  import string
2489   
2490  print string.ljust(msg, textwidth4) + ' - OK' 
2491 
2492  #raise StandardError
2493 
2494# -----------------------------------------------------------------------------
2495
2496def test_error(msg):
2497  """Print error if test fails.
2498 
2499  USAGE
2500    test_error(message)
2501  """
2502 
2503  print 'ERROR (caching.test): %s' %msg
2504  print 'Please send this code example and output to '
2505  print 'Ole.Nielsen@anu.edu.au'
2506  print
2507  print
2508 
2509  #import sys
2510  #sys.exit()
2511  raise StandardError
2512
2513#-------------------------------------------------------------
2514if __name__ == "__main__":
2515  pass
Note: See TracBrowser for help on using the repository browser.