source: anuga_core/source/anuga/caching/caching.py @ 5856

Last change on this file since 5856 was 5856, checked in by ole, 16 years ago

A bit more fiddling with caching and circular structures not
involving instances.

All relevant tests pass
File size: 72.7 KB
Line 
1# =============================================================================
2# caching.py - Supervised caching of function results.
3# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4# Australian National University (1999-2003)
5# Geoscience Australia (2003-present)
6#
7#    This program is free software; you can redistribute it and/or modify
8#    it under the terms of the GNU General Public License as published by
9#    the Free Software Foundation; either version 2 of the License, or
10#    (at your option) any later version.
11#
12#    This program is distributed in the hope that it will be useful,
13#    but WITHOUT ANY WARRANTY; without even the implied warranty of
14#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15#    GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16#    for more details.
17#
18#    You should have received a copy of the GNU General Public License
19#    along with this program; if not, write to the Free Software
20#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
21#
22#
23# Contact address: Ole.Nielsen@ga.gov.au
24#
25# Version 1.5.6 February 2002
26# =============================================================================
27 
28"""Module caching.py - Supervised caching of function results.
29
30Public functions:
31
32cache(func,args) -- Cache values returned from func given args.
33cachestat() --      Reports statistics about cache hits and time saved.
34test() --       Conducts a basic test of the caching functionality.
35
36See doc strings of individual functions for detailed documentation.
37"""
38
39# -----------------------------------------------------------------------------
40# Initialisation code
41
42# Determine platform
43#
44from os import getenv
45import types
46
47import os
48if os.name in ['nt', 'dos', 'win32', 'what else?']:
49  unix = 0
50else:
51  unix = 1
52
53cache_dir = '.python_cache'
54# Make default caching directory name
55#
56if unix:
57  homedir = getenv('INUNDATIONHOME')
58  if homedir is None:
59    homedir = '~'
60  else:
61    homedir = homedir + os.sep + '.cache'
62    # Since homedir will be a group area, individually label the caches
63    user = getenv('LOGNAME')
64    if user is not None:
65      cache_dir += '_' + user
66   
67  CR = '\n'
68else:
69  homedir = 'c:'
70  CR = '\r\n'  #FIXME: Not tested under windows
71 
72cachedir = homedir + os.sep + cache_dir + os.sep
73
74# -----------------------------------------------------------------------------
75# Options directory with default values - to be set by user
76#
77
78options = { 
79  'cachedir': cachedir,  # Default cache directory
80  'maxfiles': 1000000,   # Maximum number of cached files
81  'savestat': 1,         # Log caching info to stats file
82  'verbose': 1,          # Write messages to standard output
83  'bin': 1,              # Use binary format (more efficient)
84  'compression': 1,      # Use zlib compression
85  'bytecode': 0,         # Recompute if bytecode has changed
86  'expire': 0            # Automatically remove files that have been accessed
87                         # least recently
88}
89
90# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
91
92def set_option(key, value):
93  """Function to set values in the options directory.
94
95  USAGE:
96    set_option(key, value)
97
98  ARGUMENTS:
99    key --   Key in options dictionary. (Required)
100    value -- New value for key. (Required)
101
102  DESCRIPTION:
103    Function to set values in the options directory.
104    Raises an exception if key is not in options.
105  """
106
107  if options.has_key(key):
108    options[key] = value
109  else:
110    raise KeyError(key)  # Key not found, raise an exception
111
112# -----------------------------------------------------------------------------
113# Function cache - the main routine
114
115def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
116          verbose=None, compression=None, evaluate=0, test=0, clear=0,
117          return_filename=0):
118  """Supervised caching of function results.
119
120  USAGE:
121    result = cache(func, args, kwargs, dependencies, cachedir, verbose,
122                   compression, evaluate, test, return_filename)
123
124  ARGUMENTS:
125    func --            Function object (Required)
126    args --            Arguments to func (Default: ())
127    kwargs --          Keyword arguments to func (Default: {})   
128    dependencies --    Filenames that func depends on (Default: None)
129    cachedir --        Directory for cache files (Default: options['cachedir'])
130    verbose --         Flag verbose output to stdout
131                       (Default: options['verbose'])
132    compression --     Flag zlib compression (Default: options['compression'])
133    evaluate --        Flag forced evaluation of func (Default: 0)
134    test --            Flag test for cached results (Default: 0)
135    clear --           Flag delete cached results (Default: 0)   
136    return_filename -- Flag return of cache filename (Default: 0)   
137
138  DESCRIPTION:
139    A Python function call of the form
140
141      result = func(arg1,...,argn)
142
143    can be replaced by
144
145      from caching import cache
146      result = cache(func,(arg1,...,argn))
147
148  The latter form returns the same output as the former but reuses cached
149  results if the function has been computed previously in the same context.
150  'result' and the arguments can be simple types, tuples, list, dictionaries or
151  objects, but not unhashable types such as functions or open file objects.
152  The function 'func' may be a member function of an object or a module.
153
154  This type of caching is particularly useful for computationally intensive
155  functions with few frequently used combinations of input arguments. Note that
156  if the inputs or output are very large caching might not save time because
157  disc access may dominate the execution time.
158
159  If the function definition changes after a result has been cached it will be
160  detected by examining the functions bytecode (co_code, co_consts,
161  func_defualts, co_argcount) and it will be recomputed.
162
163  LIMITATIONS:
164    1 Caching uses the apply function and will work with anything that can be
165      pickled, so any limitation in apply or pickle extends to caching.
166    2 A function to be cached should not depend on global variables
167      as wrong results may occur if globals are changed after a result has
168      been cached.
169
170  -----------------------------------------------------------------------------
171  Additional functionality:
172
173  Keyword args
174    Keyword arguments (kwargs) can be added as a dictionary of keyword: value
175    pairs, following the syntax of the built-in function apply().
176    A Python function call of the form
177   
178      result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)   
179
180    is then cached as follows
181
182      from caching import cache
183      result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
184   
185    The default value of kwargs is {} 
186
187  Explicit dependencies:
188    The call
189      cache(func,(arg1,...,argn),dependencies = <list of filenames>)
190    Checks the size, creation time and modification time of each listed file.
191    If any file has changed the function is recomputed and the results stored
192    again.
193
194  Specify caching directory:
195    The call
196      cache(func,(arg1,...,argn), cachedir = <cachedir>)
197    designates <cachedir> where cached data are stored. Use ~ to indicate users
198    home directory - not $HOME. The default is ~/.python_cache on a UNIX
199    platform and c:/.python_cache on a Win platform.
200
201  Silent operation:
202    The call
203      cache(func,(arg1,...,argn),verbose=0)
204    suppresses messages to standard output.
205
206  Compression:
207    The call
208      cache(func,(arg1,...,argn),compression=0)
209    disables compression. (Default: compression=1). If the requested compressed
210    or uncompressed file is not there, it'll try the other version.
211
212  Forced evaluation:
213    The call
214      cache(func,(arg1,...,argn),evaluate=1)
215    forces the function to evaluate even though cached data may exist.
216
217  Testing for presence of cached result:
218    The call
219      cache(func,(arg1,...,argn),test=1)
220    retrieves cached result if it exists, otherwise None. The function will not
221    be evaluated. If both evaluate and test are switched on, evaluate takes
222    precedence.
223   
224  Obtain cache filenames:
225    The call   
226      cache(func,(arg1,...,argn),return_filename=1)
227    returns the hashed base filename under which this function and its
228    arguments would be cached
229
230  Clearing cached results:
231    The call
232      cache(func,'clear')
233    clears all cached data for 'func' and
234      cache('clear')
235    clears all cached data.
236 
237    NOTE: The string 'clear' can be passed an *argument* to func using
238      cache(func,('clear',)) or cache(func,tuple(['clear'])).
239
240    New form of clear:
241      cache(func,(arg1,...,argn),clear=1)
242    clears cached data for particular combination func and args
243     
244  """
245
246  # Imports and input checks
247  #
248  import types, time, string
249
250  if not cachedir:
251    cachedir = options['cachedir']
252
253  if verbose == None:  # Do NOT write 'if not verbose:', it could be zero.
254    verbose = options['verbose']
255
256  if compression == None:  # Do NOT write 'if not compression:',
257                           # it could be zero.
258    compression = options['compression']
259
260  # Create cache directory if needed
261  CD = checkdir(cachedir,verbose)
262
263  # Handle the case cache('clear')
264  if type(func) == types.StringType:
265    if string.lower(func) == 'clear':
266      clear_cache(CD,verbose=verbose)
267      return
268
269  # Handle the case cache(func, 'clear')
270  if type(args) == types.StringType:
271    if string.lower(args) == 'clear':
272      clear_cache(CD,func,verbose=verbose)
273      return
274
275  # Force singleton arg into a tuple
276  if type(args) != types.TupleType:
277    args = tuple([args])
278 
279  # Check that kwargs is a dictionary
280  if type(kwargs) != types.DictType:
281    raise TypeError   
282   
283  # Hash arguments (and keyword args) to integer
284  arghash = myhash((args, kwargs))
285 
286  # Get sizes and timestamps for files listed in dependencies.
287  # Force singletons into a tuple.
288  if dependencies and type(dependencies) != types.TupleType \
289                  and type(dependencies) != types.ListType:
290    dependencies = tuple([dependencies])
291  deps = get_depstats(dependencies)
292
293  # Extract function name from func object
294  funcname = get_funcname(func)
295
296  # Create cache filename
297  FN = funcname+'['+`arghash`+']'  # The symbol '(' does not work under unix
298
299  if return_filename:
300    return(FN)
301
302  if clear:
303    for file_type in file_types:
304      file_name = CD+FN+'_'+file_type
305      for fn in [file_name, file_name + '.z']:
306        if os.access(fn, os.F_OK):             
307          if unix:
308            os.remove(fn)
309          else:
310            # FIXME: os.remove doesn't work under windows       
311            os.system('del '+fn)
312          if verbose is True:
313            print 'MESSAGE (caching): File %s deleted' %fn
314        ##else:
315        ##  print '%s was not accessed' %fn
316    return None
317
318
319  #-------------------------------------------------------------------       
320 
321  # Check if previous computation has been cached
322  if evaluate:
323    Retrieved = None  # Force evaluation of func regardless of caching status.
324    reason = 5
325  else:
326    T, FN, Retrieved, reason, comptime, loadtime, compressed = \
327        CacheLookup(CD, FN, func, 
328                    args, kwargs, 
329                    deps, 
330                    verbose, 
331                    compression,
332                    dependencies)
333
334  if not Retrieved:
335    if test:  # Do not attempt to evaluate function
336      T = None
337    else:  # Evaluate function and save to cache
338      if verbose:
339       
340        msg1(funcname, args, kwargs,reason)
341
342      # Remove expired files automatically
343      if options['expire']:
344        DeleteOldFiles(CD,verbose)
345       
346      # Save args before function is evaluated in case
347      # they are modified by function
348      save_args_to_cache(CD,FN,args,kwargs,compression)
349
350      # Execute and time function with supplied arguments
351      t0 = time.time()
352      T = apply(func,args,kwargs)
353      #comptime = round(time.time()-t0)
354      comptime = time.time()-t0
355
356      if verbose:
357        msg2(funcname,args,kwargs,comptime,reason)
358
359      # Save results and estimated loading time to cache
360      loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
361                                       funcname, dependencies, compression)
362      if verbose:
363        msg3(loadtime, CD, FN, deps, compression)
364      compressed = compression
365
366  if options['savestat'] and (not test or Retrieved):
367  ##if options['savestat']:
368    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
369
370  return(T)  # Return results in all cases
371
372# -----------------------------------------------------------------------------
373
374def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
375  """Generate statistics of caching efficiency.
376
377  USAGE:
378    cachestat(sortidx, period, showuser, cachedir)
379
380  ARGUMENTS:
381    sortidx --  Index of field by which lists are (default: 4)
382                Legal values are
383                 0: 'Name'
384                 1: 'Hits'
385                 2: 'CPU'
386                 3: 'Time Saved'
387                 4: 'Gain(%)'
388                 5: 'Size'
389    period --   If set to -1 all available caching history is used.
390                If set 0 only the current month is used (default -1).
391    showuser -- Flag for additional table showing user statistics
392                (default: None).
393    cachedir -- Directory for cache files (default: options['cachedir']).
394
395  DESCRIPTION:
396    Logged caching statistics is converted into summaries of the form
397    --------------------------------------------------------------------------
398    Function Name   Hits   Exec(s)  Cache(s)  Saved(s)   Gain(%)      Size
399    --------------------------------------------------------------------------
400  """
401
402  __cachestat(sortidx, period, showuser, cachedir)
403  return
404
405# -----------------------------------------------------------------------------
406
407#Has mostly been moved to proper unit test
408def test(cachedir=None,verbose=0,compression=None):
409  """Test the functionality of caching.
410
411  USAGE:
412    test(verbose)
413
414  ARGUMENTS:
415    verbose --     Flag whether caching will output its statistics (default=0)
416    cachedir --    Directory for cache files (Default: options['cachedir'])
417    compression -- Flag zlib compression (Default: options['compression'])
418  """
419   
420  import string, time
421
422  # Initialise
423  #
424  import caching
425  reload(caching)
426
427  if not cachedir:
428    cachedir = options['cachedir']
429
430  if verbose is None:  # Do NOT write 'if not verbose:', it could be zero.
431    verbose = options['verbose']
432 
433  if compression == None:  # Do NOT write 'if not compression:',
434                           # it could be zero.
435    compression = options['compression']
436  else:
437    try:
438      set_option('compression', compression)
439    except:
440      test_error('Set option failed')     
441
442  try:
443    import zlib
444  except:
445    print
446    print '*** Could not find zlib, default to no-compression      ***'
447    print '*** Installing zlib will improve performance of caching ***'
448    print
449    compression = 0       
450    set_option('compression', compression)   
451 
452  print 
453  print_header_box('Testing caching module - please stand by')
454  print   
455
456  # Define a test function to be cached
457  #
458  def f(a,b,c,N,x=0,y='abcdefg'):
459    """f(a,b,c,N)
460       Do something time consuming and produce a complex result.
461    """
462
463    import string
464
465    B = []
466    for n in range(N):
467      s = str(n+2.0/(n + 4.0))+'.a'*10
468      B.append((a,b,c,s,n,x,y))
469    return(B)
470   
471  # Check that default cachedir is OK
472  #     
473  CD = checkdir(cachedir,verbose)   
474   
475   
476  # Make a dependency file
477  #   
478  try:
479    DepFN = CD + 'testfile.tmp'
480    DepFN_wildcard = CD + 'test*.tmp'
481    Depfile = open(DepFN,'w')
482    Depfile.write('We are the knights who say NI!')
483    Depfile.close()
484    test_OK('Wrote file %s' %DepFN)
485  except:
486    test_error('Could not open file %s for writing - check your environment' \
487               % DepFN)
488
489  # Check set_option (and switch stats off
490  #   
491  try:
492    set_option('savestat',0)
493    assert(options['savestat'] == 0)
494    test_OK('Set option')
495  except:
496    test_error('Set option failed')   
497   
498  # Make some test input arguments
499  #
500  N = 5000  #Make N fairly small here
501
502  a = [1,2]
503  b = ('Thou shalt count the number three',4)
504  c = {'Five is right out': 6, (7,8): 9}
505  x = 3
506  y = 'holy hand granate'
507
508  # Test caching
509  #
510  if compression:
511    comprange = 2
512  else:
513    comprange = 1
514
515  for comp in range(comprange):
516 
517    # Evaluate and store
518    #
519    try:
520      T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
521                         verbose=verbose, compression=comp)
522      if comp:                   
523        test_OK('Caching evaluation with compression')
524      else:     
525        test_OK('Caching evaluation without compression')     
526    except:
527      if comp:
528        test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
529      else:
530        test_error('Caching evaluation failed - try caching.test(verbose=1)')
531
532    # Retrieve
533    #                           
534    try:                         
535      T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
536                         compression=comp) 
537
538      if comp:                   
539        test_OK('Caching retrieval with compression')
540      else:     
541        test_OK('Caching retrieval without compression')     
542    except:
543      if comp:
544        test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
545      else:                                     
546        test_error('Caching retrieval failed - try caching.test(verbose=1)')
547
548    # Reference result
549    #   
550    T3 = f(a,b,c,N,x=x,y=y)  # Compute without caching
551   
552    if T1 == T2 and T2 == T3:
553      if comp:
554        test_OK('Basic caching functionality (with compression)')
555      else:
556        test_OK('Basic caching functionality (without compression)')
557    else:
558      test_error('Cached result does not match computed result')
559
560
561  # Test return_filename
562  #   
563  try:
564    FN = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
565                       return_filename=1)   
566    assert(FN[:2] == 'f[')
567    test_OK('Return of cache filename')
568  except:
569    test_error('Return of cache filename failed')
570
571  # Test existence of cachefiles
572  # 
573  try:
574    (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
575    (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
576    (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
577    test_OK('Presence of cache files')
578    datafile.close()
579    argsfile.close()
580    admfile.close()
581  except:
582    test_error('Expected cache files did not exist') 
583             
584  # Test 'test' function when cache is present
585  #     
586  try:
587    #T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
588    #                   evaluate=1) 
589    T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
590    assert(T1 == T4)
591
592    test_OK("Option 'test' when cache file present")
593  except:
594    test_error("Option 'test' when cache file present failed")     
595
596  # Test that 'clear' works
597  #
598  #try:
599  #  caching.cache(f,'clear',verbose=verbose)
600  #  test_OK('Clearing of cache files')
601  #except:
602  #  test_error('Clear does not work')
603  try:
604    caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)   
605    test_OK('Clearing of cache files')
606  except:
607    test_error('Clear does not work') 
608
609 
610
611  # Test 'test' function when cache is absent
612  #     
613  try:
614    T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
615    assert(T4 is None)
616    test_OK("Option 'test' when cache absent")
617  except:
618    test_error("Option 'test' when cache absent failed")     
619         
620  # Test dependencies
621  #
622  T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
623                       dependencies=DepFN) 
624  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
625                       dependencies=DepFN)                     
626                       
627  if T1 == T2:
628    test_OK('Basic dependencies functionality')
629  else:
630    test_error('Dependencies do not work')
631
632  # Test basic wildcard dependency
633  #
634  T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
635                       dependencies=DepFN_wildcard)                     
636   
637  if T1 == T3:
638    test_OK('Basic dependencies with wildcard functionality')
639  else:
640    test_error('Dependencies with wildcards do not work')
641
642
643  # Test that changed timestamp in dependencies triggers recomputation
644 
645  # Modify dependency file
646  Depfile = open(DepFN,'a')
647  Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
648  Depfile.close()
649 
650  T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
651                       dependencies=DepFN, test = 1)                     
652 
653  if T3 is None:
654    test_OK('Changed dependencies recognised')
655  else:
656    test_error('Changed dependencies not recognised')   
657 
658  # Test recomputation when dependencies have changed
659  #
660  T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
661                       dependencies=DepFN)                       
662  if T1 == T3:
663    test_OK('Recomputed value with changed dependencies')
664  else:
665    test_error('Recomputed value with changed dependencies failed')
666
667  # Performance test (with statistics)
668  # Don't really rely on this as it will depend on specific computer.
669  #
670
671  set_option('savestat',1)
672
673  N = 20*N   #Should be large on fast computers...
674  tt = time.time()
675  T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
676  t1 = time.time() - tt
677 
678  tt = time.time()
679  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
680  t2 = time.time() - tt
681 
682  if T1 == T2:
683    if t1 > t2:
684      test_OK('Performance test: relative time saved = %s pct' \
685              %str(round((t1-t2)*100/t1,2)))
686    #else:
687    #  print 'WARNING: Performance a bit low - this could be specific to current platform'
688  else:       
689    test_error('Basic caching failed for new problem')
690           
691  # Test presence of statistics file
692  #
693  try: 
694    DIRLIST = os.listdir(CD)
695    SF = []
696    for FN in DIRLIST:
697      if string.find(FN,statsfile) >= 0:
698        fid = open(CD+FN,'r')
699        fid.close()
700    test_OK('Statistics files present') 
701  except:
702    test_OK('Statistics files cannot be opened')         
703     
704  print_header_box('Show sample output of the caching function:')
705 
706  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
707  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
708  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
709 
710  print_header_box('Show sample output of cachestat():')
711  if unix:
712    cachestat()   
713  else:
714    try:
715      import time
716      t = time.strptime('2030','%Y')
717      cachestat()
718    except: 
719      print 'caching.cachestat() does not work here, because it'
720      print 'relies on time.strptime() which is unavailable in Windows'
721     
722  print
723  test_OK('Caching self test completed')   
724     
725           
726  # Test setoption (not yet implemented)
727  #
728
729 
730#==============================================================================
731# Auxiliary functions
732#==============================================================================
733
734# Import pickler
735# cPickle is used by functions mysave, myload, and compare
736#
737import cPickle  # 10 to 100 times faster than pickle
738pickler = cPickle
739
740# Local immutable constants
741#
742comp_level = 1              # Compression level for zlib.
743                            # comp_level = 1 works well.
744textwidth1 = 16             # Text width of key fields in report forms.
745#textwidth2 = 132            # Maximal width of textual representation of
746textwidth2 = 300            # Maximal width of textual representation of
747                            # arguments.
748textwidth3 = 16             # Initial width of separation lines. Is modified.
749textwidth4 = 50             # Text width in test_OK()
750statsfile  = '.cache_stat'  # Basefilename for cached statistics.
751                            # It will reside in the chosen cache directory.
752
753file_types = ['Result',     # File name extension for cached function results.
754              'Args',       # File name extension for stored function args.
755              'Admin']      # File name extension for administrative info.
756
757Reason_msg = ['OK',         # Verbose reasons for recomputation
758              'No cached result', 
759              'Dependencies have changed', 
760              'Arguments have changed',
761              'Bytecode has changed',
762              'Recomputation was requested by caller',
763              'Cached file was unreadable']             
764             
765# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
766
767def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, 
768                dependencies):
769  """Determine whether cached result exists and return info.
770
771  USAGE:
772    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \ 
773    CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
774                dependencies)
775
776  INPUT ARGUMENTS:
777    CD --            Cache Directory
778    FN --            Suggested cache file name
779    func --          Function object
780    args --          Tuple of arguments
781    kwargs --        Dictionary of keyword arguments   
782    deps --          Dependencies time stamps
783    verbose --       Flag text output
784    compression --   Flag zlib compression
785    dependencies --  Given list of dependencies
786   
787  OUTPUT ARGUMENTS:
788    T --             Cached result if present otherwise None
789    FN --            File name under which new results must be saved
790    Retrieved --     True if a valid cached result was found
791    reason --        0: OK (if Retrieved),
792                     1: No cached result,
793                     2: Dependencies have changed,
794                     3: Arguments have changed
795                     4: Bytecode has changed
796                     5: Recomputation was forced
797                     6: Unreadable file
798    comptime --      Number of seconds it took to computed cachged result
799    loadtime --      Number of seconds it took to load cached result
800    compressed --    Flag (0,1) if cached results were compressed or not
801
802  DESCRIPTION:
803    Determine if cached result exists as follows:
804    Load in saved arguments and bytecode stored under hashed filename.
805    If they are identical to current arguments and bytecode and if dependencies
806    have not changed their time stamp, then return cached result.
807
808    Otherwise return filename under which new results should be cached.
809    Hash collisions are handled recursively by calling CacheLookup again with a
810    modified filename.
811  """
812
813  import time, string, types
814
815  # Assess whether cached result exists - compressed or not.
816  #
817  if verbose:
818    print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
819           %(CD+FN, file_types[0], file_types[1], file_types[2])
820  (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
821  (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
822  (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
823
824  if verbose is True and deps is not None:
825    print 'Caching: Dependencies are', deps.keys()
826
827  if not (argsfile and datafile and admfile) or \
828     not (compressed0 == compressed1 and compressed0 == compressed2):
829    # Cached result does not exist or files were compressed differently
830    #
831    # This will ensure that evaluation will take place unless all files are
832    # present.
833
834    reason = 1
835    return(None,FN,None,reason,None,None,None) #Recompute using same filename
836
837  compressed = compressed0  # Remember if compressed files were actually used
838  datafile.close()
839
840  # Retrieve arguments and adm. info
841  #
842  R, reason = myload(argsfile, compressed)  # The original arguments
843  argsfile.close()
844   
845  if reason > 0:
846      # Recompute using same filename   
847      return(None, FN, None, reason, None, None, None)
848  else:   
849      (argsref, kwargsref) = R
850
851  R, reason = myload(admfile, compressed)
852  admfile.close() 
853
854  if reason > 0:
855    return(None,FN,None,reason,None,None,None) #Recompute using same filename
856
857 
858  depsref  = R[0]  # Dependency statistics
859  comptime = R[1]  # The computation time
860  coderef  = R[2]  # The byte code
861  funcname = R[3]  # The function name
862
863  # Check if dependencies have changed
864  #
865  if dependencies and not compare(depsref, deps):
866    if verbose:
867      print 'MESSAGE (caching.py): Dependencies', dependencies, \
868            'have changed - recomputing'
869    # Don't use cached file - recompute
870    reason = 2
871    return(None, FN, None, reason, None, None, None)
872
873  # Get bytecode from func
874  #
875  bytecode = get_bytecode(func)
876
877  #print compare(argsref,args),   
878  #print compare(kwargsref,kwargs),
879  #print compare(bytecode,coderef)
880
881  # Check if arguments or bytecode have changed
882  if compare(argsref, args) and compare(kwargsref, kwargs) and \
883     (not options['bytecode'] or compare(bytecode, coderef)):
884
885    # Arguments and dependencies match. Get cached results
886    T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
887    if reason > 0:
888        # Recompute using same FN     
889        return(None, FN, None, reason, None, None, None)
890
891    Retrieved = 1
892    reason = 0
893
894    if verbose:
895      msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
896
897      if loadtime >= comptime:
898        print 'WARNING (caching.py): Caching did not yield any gain.'
899        print '                      Consider executing function ',
900        print '('+funcname+') without caching.'
901  else:
902
903    # Non matching arguments or bytecodes signify a hash-collision.
904    # This is resolved by recursive search of cache filenames
905    # until either a matching or an unused filename is found.
906    #
907    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
908        CacheLookup(CD, FN+'x', func, args, kwargs, deps, 
909                    verbose, compression, dependencies)
910
911    # DEBUGGING
912    # if not Retrieved:
913    #   print 'Arguments did not match'
914    # else:
915    #   print 'Match found !'
916   
917    # The real reason is that args or bytecodes have changed.
918    # Not that the recursive seach has found an unused filename
919    if not Retrieved:
920      if not compare(bytecode, coderef):
921        reason = 4 # Bytecode has changed
922      else:   
923        reason = 3 # Arguments have changed
924       
925   
926  return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
927
928# -----------------------------------------------------------------------------
929
930def clear_cache(CD,func=None, verbose=None):
931  """Clear cache for func.
932
933  USAGE:
934     clear(CD, func, verbose)
935
936  ARGUMENTS:
937     CD --       Caching directory (required)
938     func --     Function object (default: None)
939     verbose --  Flag verbose output (default: None)
940
941  DESCRIPTION:
942
943    If func == None, clear everything,
944    otherwise clear only files pertaining to func.
945  """
946
947  import os, re
948   
949  if CD[-1] != os.sep:
950    CD = CD+os.sep
951 
952  if verbose == None:
953    verbose = options['verbose']
954
955  # FIXME: Windows version needs to be tested
956
957  if func:
958    funcname = get_funcname(func)
959    if verbose:
960      print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
961
962    file_names = os.listdir(CD)
963    for file_name in file_names:
964      #RE = re.search('^' + funcname,file_name)  #Inefficient
965      #if RE:
966      if file_name[:len(funcname)] == funcname:
967        if unix:
968          os.remove(CD+file_name)
969        else:
970          os.system('del '+CD+file_name)
971          # FIXME: os.remove doesn't work under windows
972  else:
973    file_names = os.listdir(CD)
974    if len(file_names) > 0:
975      if verbose:
976        print 'MESSAGE (caching.py): Remove the following files:'
977        for file_name in file_names:
978            print file_name
979
980        A = raw_input('Delete (Y/N)[N] ?')
981      else:
982        A = 'Y' 
983       
984      if A == 'Y' or A == 'y':
985        for file_name in file_names:
986          if unix:
987            os.remove(CD+file_name)
988          else:
989            os.system('del '+CD+file_name)
990            # FIXME: os.remove doesn't work under windows
991          #exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
992
993# -----------------------------------------------------------------------------
994
995def DeleteOldFiles(CD,verbose=None):
996  """Remove expired files
997
998  USAGE:
999    DeleteOldFiles(CD,verbose=None)
1000  """
1001
1002  if verbose == None:
1003    verbose = options['verbose']
1004
1005  maxfiles = options['maxfiles']
1006
1007  # FIXME: Windows version
1008
1009  import os
1010  block = 1000  # How many files to delete per invokation
1011  Files = os.listdir(CD)
1012  numfiles = len(Files)
1013  if not unix: return  # FIXME: Windows case ?
1014
1015  if numfiles > maxfiles:
1016    delfiles = numfiles-maxfiles+block
1017    if verbose:
1018      print 'Deleting '+`delfiles`+' expired files:'
1019      os.system('ls -lur '+CD+'* | head -' + `delfiles`)            # List them
1020    os.system('ls -ur '+CD+'* | head -' + `delfiles` + ' | xargs /bin/rm')
1021                                                                  # Delete them
1022    # FIXME: Replace this with os.listdir and os.remove
1023
1024# -----------------------------------------------------------------------------
1025
1026def save_args_to_cache(CD, FN, args, kwargs, compression):
1027  """Save arguments to cache
1028
1029  USAGE:
1030    save_args_to_cache(CD,FN,args,kwargs,compression)
1031  """
1032
1033  import time, os, sys, types
1034
1035  (argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1036
1037  if argsfile is None:
1038    msg = 'ERROR (caching): Could not open argsfile for writing: %s' %FN
1039    raise IOError, msg
1040
1041  mysave((args,kwargs),argsfile,compression)  # Save args and kwargs to cache
1042  argsfile.close()
1043
1044  # Change access rights if possible
1045  #
1046  #if unix:
1047  #  try:
1048  #    exitcode=os.system('chmod 666 '+argsfile.name)
1049  #  except:
1050  #    pass
1051  #else:
1052  #  pass  # FIXME: Take care of access rights under Windows
1053
1054  return
1055
1056# -----------------------------------------------------------------------------
1057
1058def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1059                          dependencies, compression):
1060  """Save computed results T and admin info to cache
1061
1062  USAGE:
1063    save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1064                          dependencies, compression)
1065  """
1066
1067  import time, os, sys, types
1068
1069  (datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1070  (admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1071
1072  if not datafile:
1073    if verbose:
1074      print 'ERROR (caching): Could not open %s' %datafile.name
1075    raise IOError
1076
1077  if not admfile:
1078    if verbose:
1079      print 'ERROR (caching): Could not open %s' %admfile.name
1080    raise IOError
1081
1082  t0 = time.time()
1083
1084  mysave(T,datafile,compression)  # Save data to cache
1085  datafile.close()
1086  #savetime = round(time.time()-t0,2)
1087  savetime = time.time()-t0 
1088
1089  bytecode = get_bytecode(func)  # Get bytecode from function object
1090  admtup = (deps, comptime, bytecode, funcname)  # Gather admin info
1091
1092  mysave(admtup,admfile,compression)  # Save admin info to cache
1093  admfile.close()
1094
1095  # Change access rights if possible
1096  #
1097  #if unix:
1098  #  try:
1099  #    exitcode=os.system('chmod 666 '+datafile.name)
1100  #    exitcode=os.system('chmod 666 '+admfile.name)
1101  #  except:
1102  #    pass
1103  #else:
1104  #  pass  # FIXME: Take care of access rights under Windows
1105
1106  return(savetime)
1107
1108# -----------------------------------------------------------------------------
1109
1110def load_from_cache(CD,FN,compression):
1111  """Load previously cached data from file FN
1112
1113  USAGE:
1114    load_from_cache(CD,FN,compression)
1115  """
1116
1117  import time
1118
1119  (datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1120  t0 = time.time()
1121  T, reason = myload(datafile,compressed)
1122
1123  loadtime = time.time()-t0
1124  datafile.close() 
1125
1126  return T, loadtime, compressed, reason
1127
1128# -----------------------------------------------------------------------------
1129
1130def myopen(FN,mode,compression=1):
1131  """Open file FN using given mode
1132
1133  USAGE:
1134    myopen(FN,mode,compression=1)
1135
1136  ARGUMENTS:
1137    FN --           File name to be opened
1138    mode --         Open mode (as in open)
1139    compression --  Flag zlib compression
1140
1141  DESCRIPTION:
1142     if compression
1143       Attempt first to open FN + '.z'
1144       If this fails try to open FN
1145     else do the opposite
1146     Return file handle plus info about whether it was compressed or not.
1147  """
1148
1149  import string
1150
1151  # Determine if file exists already (if writing was requested)
1152  # This info is only used to determine if access modes should be set
1153  #
1154  if 'w' in mode or 'a' in mode:
1155    try:
1156      file = open(FN+'.z','r')
1157      file.close()
1158      new_file = 0
1159    except:
1160      try:
1161        file = open(FN,'r') 
1162        file.close()
1163        new_file = 0
1164      except:
1165        new_file = 1
1166  else:
1167    new_file = 0 #Assume it exists if mode was not 'w'
1168 
1169
1170  compressed = 0
1171  if compression:
1172    try:
1173      file = open(FN+'.z',mode)
1174      compressed = 1
1175    except:
1176      try:
1177        file = open(FN,mode)
1178      except:
1179        file = None
1180  else:
1181    try:
1182      file = open(FN,mode)
1183    except:
1184      try:
1185        file = open(FN+'.z',mode)
1186        compressed = 1
1187      except:
1188        file = None
1189
1190  # Now set access rights if it is a new file
1191  #
1192  if file and new_file:
1193    if unix:
1194      exitcode=os.system('chmod 666 '+file.name)
1195    else:
1196      pass  # FIXME: Take care of access rights under Windows
1197
1198  return(file,compressed)
1199
1200# -----------------------------------------------------------------------------
1201
1202def myload(file, compressed):
1203  """Load data from file
1204
1205  USAGE:
1206    myload(file, compressed)
1207  """
1208
1209  reason = 0
1210  try:
1211    if compressed:
1212      import zlib
1213
1214      RsC = file.read()
1215      try:
1216        Rs  = zlib.decompress(RsC)
1217      except:
1218        #  File "./caching.py", line 1032, in load_from_cache
1219        #  T = myload(datafile,compressed)
1220        #  File "./caching.py", line 1124, in myload
1221        #  Rs  = zlib.decompress(RsC)
1222        #  zlib.error: Error -5 while decompressing data
1223        #print 'ERROR (caching): Could not decompress ', file.name
1224        #raise Exception
1225        reason = 6  # Unreadable file
1226        return None, reason 
1227     
1228     
1229      del RsC  # Free up some space
1230      R = pickler.loads(Rs)
1231    else:
1232      try:
1233        R = pickler.load(file)
1234      #except EOFError, e:
1235      except:
1236        #Catch e.g., file with 0 length or corrupted
1237        reason = 6  # Unreadable file
1238        return None, reason
1239     
1240  except MemoryError:
1241    import sys
1242    if options['verbose']:
1243      print 'ERROR (caching): Out of memory while loading %s, aborting' \
1244            %(file.name)
1245
1246    # Raise the error again for now
1247    #
1248    raise MemoryError
1249
1250  return R, reason
1251
1252# -----------------------------------------------------------------------------
1253
1254def mysave(T,file,compression):
1255  """Save data T to file
1256
1257  USAGE:
1258    mysave(T,file,compression)
1259
1260  """
1261
1262  bin = options['bin']
1263
1264  if compression:
1265    try:
1266      import zlib
1267    except:
1268      print
1269      print '*** Could not find zlib ***'
1270      print '*** Try to run caching with compression off ***'
1271      print "*** caching.set_option('compression', 0) ***"
1272      raise Exception
1273     
1274
1275    try:
1276      Ts  = pickler.dumps(T, bin)
1277    except MemoryError:
1278      msg = '****WARNING (caching.py): Could not pickle data for compression.'
1279      msg += ' Try using compression = False'
1280      raise MemoryError, msg
1281    else: 
1282      # Compressed pickling     
1283      TsC = zlib.compress(Ts, comp_level)
1284      file.write(TsC)
1285  else:
1286      # Uncompressed pickling
1287      pickler.dump(T, file, bin)
1288
1289      # FIXME: This may not work on Windoze network drives.
1290      # The error msg is IOError: [Errno 22] Invalid argument
1291      # Testing with small files was OK, though.
1292      # I think this is an OS problem.
1293
1294      # Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1295     
1296# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1297#
1298#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1299#
1300#The following extract is taken from the site http://www.python.org:
1301#
1302#---------------------------------------------------------------------------------------------
1303#exception IOError
1304#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1305#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1306#---------------------------------------------------------------------------------------------
1307#
1308#The error code(s) that accompany exceptions are described at:
1309#http://www.python.org/dev/doc/devel//lib/module-errno.html
1310#
1311#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1312#
1313#exceptions.IOError invalid argument Errno 22
1314       
1315      #try:
1316      #  pickler.dump(T,file,bin)
1317      #except IOError, e:
1318      #  print e
1319      #  msg = 'Could not store to %s, bin=%s' %(file, bin)
1320      #  raise msg
1321     
1322
1323# -----------------------------------------------------------------------------
1324
1325   
1326def myhash(T, ids=None):
1327  """Compute hashed integer from a range of inputs.
1328  If T is not hashable being e.g. a tuple T, myhash will recursively
1329  hash the values individually
1330
1331  USAGE:
1332    myhash(T)
1333
1334  ARGUMENTS:
1335    T -- Anything
1336  """
1337
1338  from types import TupleType, ListType, DictType, InstanceType 
1339  from Numeric import ArrayType, average
1340
1341   
1342  if type(T) in [TupleType, ListType, DictType, InstanceType]: 
1343      # Keep track of unique id's to protect against infinite recursion
1344      if ids is None: ids = []
1345
1346      # Check if T has already been encountered
1347      i = id(T) 
1348 
1349      if i in ids:
1350          return 0 # T has been hashed already     
1351      else:
1352          ids.append(i)
1353   
1354
1355   
1356  # Start hashing 
1357 
1358  # On some architectures None, False and True gets different hash values
1359  if T is None:
1360      return(-1)
1361  if T is False:
1362      return(0)
1363  if T is True:
1364      return(1)
1365
1366  # Get hash values for hashable entries
1367  if type(T) in [TupleType, ListType]:
1368      hvals = []
1369      for t in T:
1370          h = myhash(t, ids)
1371          hvals.append(h)
1372      val = hash(tuple(hvals))
1373  elif type(T) == DictType:
1374      # Make dictionary ordering unique 
1375      I = T.items()
1376      I.sort()   
1377      val = myhash(I, ids)
1378  elif type(T) == ArrayType:
1379      # Use mean value for efficiency 
1380      val = hash(average(T.flat))
1381  elif type(T) == InstanceType:
1382      val = myhash(T.__dict__, ids)
1383  else:
1384      try:
1385          val = hash(T)
1386      except:
1387          val = 1
1388
1389  return(val)
1390
1391# -----------------------------------------------------------------------------
1392
1393
1394def compare(A, B, ids=None):
1395    """Safe comparison of general objects
1396
1397    USAGE:
1398      compare(A,B)
1399
1400    DESCRIPTION:
1401      Return 1 if A and B they are identical, 0 otherwise
1402    """
1403
1404    try:
1405        identical = (A == B)
1406    except:
1407        # E.g. if A and B are circular or otherwise can't be compared.
1408        identical = False
1409       
1410       
1411    if identical is False:   
1412        # Use pickle to compare data
1413        # The native pickler must be used
1414        # since the faster cPickle does not
1415        # guarantee a unique translation
1416       
1417        import pickle as pickler # Use non-C version here
1418        #import cPickle as pickler
1419        identical = (pickler.dumps(A,0) == pickler.dumps(B,0))
1420
1421    return(identical)
1422
1423
1424def old_compare(A, B, ids=None):
1425    """Safe comparison of general objects
1426
1427    USAGE:
1428      compare(A,B)
1429
1430    DESCRIPTION:
1431      Return 1 if A and B they are identical, 0 otherwise
1432    """
1433
1434    # FIXME (Ole): This is probably obsolete now
1435   
1436    from types import TupleType, ListType, DictType, InstanceType
1437   
1438   
1439    # Keep track of unique id's to protect against infinite recursion
1440    if ids is None: ids = {}
1441
1442
1443    # Check if T has already been encountered
1444    iA = id(A) 
1445    iB = id(B)     
1446   
1447    if ids.has_key((iA, iB)):
1448        # A and B have been compared already
1449        #print 'Found', (iA, iB), A, B
1450        return ids[(iA, iB)]
1451    else:
1452        ids[(iA, iB)] = True
1453   
1454   
1455    #print 'Comparing', A, B, (iA, iB)
1456    #print ids
1457    #raw_input()
1458   
1459    if type(A) in [TupleType, ListType] and type(B) in [TupleType, ListType]:
1460        N = len(A)
1461        if len(B) != N: 
1462            identical = False
1463        else:
1464            identical = True
1465            for i in range(N):
1466                if not compare(A[i], B[i], ids): 
1467                    identical = False; break
1468               
1469    elif type(A) == DictType and type(B) == DictType:
1470        if len(A) != len(B):
1471            identical = False
1472        else:   
1473            identical = True
1474            for key in A.keys():
1475                if not B.has_key(key):
1476                    identical = False; break
1477         
1478                if not compare(A[key], B[key], ids): 
1479                    identical = False; break     
1480   
1481    elif type(A) == type(B) == types.InstanceType:   
1482        # Take care of special case where elements are instances           
1483        # Base comparison on attributes
1484               
1485        a = A.__dict__                 
1486        b = B.__dict__                             
1487               
1488        identical = compare(a, b, ids)               
1489
1490   
1491       
1492    else:       
1493        # Fall back to general code
1494        try:
1495            identical = (A == B)
1496        except:
1497            import pickle # Use non-C version here
1498            try:
1499                identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
1500            except:
1501                identical = 0
1502
1503    # Record result of comparison and return           
1504    ids[(iA, iB)] = identical
1505   
1506    return(identical)
1507
1508# -----------------------------------------------------------------------------
1509
1510def nospace(s):
1511  """Replace spaces in string s with underscores
1512
1513  USAGE:
1514    nospace(s)
1515
1516  ARGUMENTS:
1517    s -- string
1518  """
1519
1520  import string
1521
1522  newstr = ''
1523  for i in range(len(s)):
1524    if s[i] == ' ':
1525      newstr = newstr+'_'
1526    else:
1527      newstr = newstr+s[i]
1528
1529  return(newstr)
1530
1531# -----------------------------------------------------------------------------
1532
1533def get_funcname(func):
1534  """Retrieve name of function object func (depending on its type)
1535
1536  USAGE:
1537    get_funcname(func)
1538  """
1539
1540  import types, string
1541
1542  if type(func) == types.FunctionType:
1543    funcname = func.func_name
1544  elif type(func) == types.BuiltinFunctionType:
1545    funcname = func.__name__
1546  else:
1547    tab = string.maketrans("<>'","   ")
1548    tmp = string.translate(`func`,tab)
1549    tmp = string.split(tmp)
1550    funcname = string.join(tmp)
1551
1552    # Truncate memory address as in
1553    # class __main__.Dummy at 0x00A915D0
1554    index = funcname.find('at 0x')
1555    if index >= 0:
1556      funcname = funcname[:index+5] # Keep info that there is an address
1557
1558  funcname = nospace(funcname)
1559  return(funcname)
1560
1561# -----------------------------------------------------------------------------
1562
1563def get_bytecode(func):
1564  """ Get bytecode from function object.
1565
1566  USAGE:
1567    get_bytecode(func)
1568  """
1569
1570  import types
1571
1572  if type(func) == types.FunctionType:
1573    bytecode = func.func_code.co_code
1574    consts = func.func_code.co_consts
1575    argcount = func.func_code.co_argcount   
1576    defaults = func.func_defaults     
1577  elif type(func) == types.MethodType:
1578    bytecode = func.im_func.func_code.co_code
1579    consts =  func.im_func.func_code.co_consts
1580    argcount =  func.im_func.func_code.co_argcount   
1581    defaults = func.im_func.func_defaults         
1582  else:
1583    #raise Exception  #Test only
1584    bytecode = None   #Built-in functions are assumed not to change
1585    consts = 0
1586    argcount = 0
1587    defaults = 0
1588
1589  return (bytecode, consts, argcount, defaults)
1590
1591# -----------------------------------------------------------------------------
1592
1593def get_depstats(dependencies):
1594  """ Build dictionary of dependency files and their size, mod. time and ctime.
1595
1596  USAGE:
1597    get_depstats(dependencies):
1598  """
1599
1600  import types
1601
1602  #print 'Caching DEBUG: Dependencies are', dependencies
1603  d = {}
1604  if dependencies:
1605
1606    #Expand any wildcards
1607    import glob
1608    expanded_dependencies = []
1609    for FN in dependencies:
1610      expanded_FN = glob.glob(FN)
1611
1612      if expanded_FN == []:
1613        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1614        raise Exception, errmsg     
1615
1616      expanded_dependencies += expanded_FN
1617
1618   
1619    for FN in expanded_dependencies:
1620      if not type(FN) == types.StringType:
1621        errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1622        errmsg += '                   Dependency given: %s' %FN
1623        raise Exception, errmsg     
1624      if not os.access(FN,os.F_OK):
1625        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1626        raise Exception, errmsg
1627      (size,atime,mtime,ctime) = filestat(FN)
1628
1629      # We don't use atime because that would cause recomputation every time.
1630      # We don't use ctime because that is irrelevant and confusing for users.
1631      d.update({FN : (size,mtime)})
1632
1633  return(d)
1634
1635# -----------------------------------------------------------------------------
1636
1637def filestat(FN):
1638  """A safe wrapper using os.stat to get basic file statistics
1639     The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1640
1641  USAGE:
1642    filestat(FN)
1643
1644  DESCRIPTION:
1645     Must compile Python with
1646     CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1647              configure
1648     as given in section 8.1.1 Large File Support in the Libray Reference
1649  """
1650
1651  import os, time
1652
1653  try:
1654    stats = os.stat(FN)
1655    size  = stats[6]
1656    atime = stats[7]
1657    mtime = stats[8]
1658    ctime = stats[9]
1659  except:
1660
1661    # Hack to get the results anyway (works only on Unix at the moment)
1662    #
1663    print 'Hack to get os.stat when files are too large'
1664
1665    if unix:
1666      tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1667      # Unique filename, FIXME: Use random number
1668
1669      # Get size and access time (atime)
1670      #
1671      exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1672      (size,atime) = get_lsline(tmp)
1673
1674      # Get size and modification time (mtime)
1675      #
1676      exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1677      (size,mtime) = get_lsline(tmp)
1678
1679      # Get size and ctime
1680      #
1681      exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1682      (size,ctime) = get_lsline(tmp)
1683
1684      try:
1685        exitcode=os.system('rm '+tmp)
1686        # FIXME: Gives error if file doesn't exist
1687      except:
1688        pass
1689    else:
1690      pass
1691      raise Exception  # FIXME: Windows case
1692
1693  return(long(size),atime,mtime,ctime)
1694
1695# -----------------------------------------------------------------------------
1696
1697def get_lsline(FN):
1698  """get size and time for filename
1699
1700  USAGE:
1701    get_lsline(file_name)
1702
1703  DESCRIPTION:
1704    Read in one line 'ls -la' item from file (generated by filestat) and
1705    convert time to seconds since epoch. Return file size and time.
1706  """
1707
1708  import string, time
1709
1710  f = open(FN,'r')
1711  info = f.read()
1712  info = string.split(info)
1713
1714  size = info[4]
1715  week = info[5]
1716  mon  = info[6]
1717  day  = info[7]
1718  hour = info[8]
1719  year = info[9]
1720
1721  str = week+' '+mon+' '+day+' '+hour+' '+year
1722  timetup = time.strptime(str)
1723  t = time.mktime(timetup)
1724  return(size, t)
1725
1726# -----------------------------------------------------------------------------
1727
1728def checkdir(CD,verbose=None):
1729  """Check or create caching directory
1730
1731  USAGE:
1732    checkdir(CD,verbose):
1733
1734  ARGUMENTS:
1735    CD -- Directory
1736    verbose -- Flag verbose output (default: None)
1737
1738  DESCRIPTION:
1739    If CD does not exist it will be created if possible
1740  """
1741
1742  import os
1743  import os.path
1744
1745  if CD[-1] != os.sep: 
1746    CD = CD + os.sep  # Add separator for directories
1747
1748  CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1749  if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1750    try:
1751      exitcode=os.mkdir(CD)
1752
1753      # Change access rights if possible
1754      #
1755      if unix:
1756        exitcode=os.system('chmod 777 '+CD)
1757      else:
1758        pass  # FIXME: What about acces rights under Windows?
1759      if verbose: print 'MESSAGE: Directory', CD, 'created.'
1760    except:
1761      print 'WARNING: Directory', CD, 'could not be created.'
1762      if unix:
1763        CD = '/tmp/'
1764      else:
1765        CD = 'C:' 
1766      print 'Using directory %s instead' %CD
1767
1768  return(CD)
1769
1770#==============================================================================
1771# Statistics
1772#==============================================================================
1773
1774def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
1775                 compression):
1776  """Add stats entry
1777
1778  USAGE:
1779    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1780
1781  DESCRIPTION:
1782    Make one entry in the stats file about one cache hit recording time saved
1783    and other statistics. The data are used by the function cachestat.
1784  """
1785
1786  import os, time
1787
1788  try:
1789    TimeTuple = time.localtime(time.time())
1790    extension = time.strftime('%b%Y',TimeTuple)
1791    SFN = CD+statsfile+'.'+extension
1792    #statfile = open(SFN,'a')
1793    (statfile, dummy) = myopen(SFN,'a',compression=0)
1794
1795    # Change access rights if possible
1796    #
1797    #if unix:
1798    #  try:
1799    #    exitcode=os.system('chmod 666 '+SFN)
1800    #  except:
1801    #    pass
1802  except:
1803    print 'Warning: Stat file could not be opened'
1804
1805  try:
1806    if os.environ.has_key('USER'):
1807      user = os.environ['USER']
1808    else:
1809      user = 'Nobody'
1810
1811    date = time.asctime(TimeTuple)
1812
1813    if Retrieved:
1814      hit = '1'
1815    else:
1816      hit = '0'
1817
1818    # Get size of result file
1819    #   
1820    if compression:
1821      stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1822    else:
1823      stats = os.stat(CD+FN+'_'+file_types[0])
1824 
1825    if stats: 
1826      size = stats[6]
1827    else:
1828      size = -1  # Error condition, but don't crash. This is just statistics 
1829
1830    # Build entry
1831    # 
1832    entry = date             + ',' +\
1833            user             + ',' +\
1834            FN               + ',' +\
1835            str(int(size))   + ',' +\
1836            str(compression) + ',' +\
1837            hit              + ',' +\
1838            str(reason)      + ',' +\
1839            str(round(comptime,4)) + ',' +\
1840            str(round(loadtime,4)) +\
1841            CR
1842           
1843    statfile.write(entry)
1844    statfile.close()
1845  except:
1846    print 'Warning: Writing of stat file failed'
1847
1848# -----------------------------------------------------------------------------
1849
1850# FIXME: should take cachedir as an optional arg
1851#
1852def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1853  """  List caching statistics.
1854
1855  USAGE:
1856    __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1857
1858      Generate statistics of caching efficiency.
1859      The parameter sortidx determines by what field lists are sorted.
1860      If the optional keyword period is set to -1,
1861      all available caching history is used.
1862      If it is 0 only the current month is used.
1863      Future versions will include more than one month....
1864      OMN 20/8/2000
1865  """
1866
1867  import os
1868  import os.path
1869  from string import split, rstrip, find
1870  from time import strptime, localtime, strftime, mktime, ctime
1871
1872  # sortidx = 4    # Index into Fields[1:]. What to sort by.
1873
1874  Fields = ['Name', 'Hits', 'Exec(s)', \
1875            'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1876  Widths = [25,7,9,9,9,9,13]
1877  #Types = ['s','d','d','d','d','.2f','d']
1878  Types = ['s','d','.2f','.2f','.2f','.2f','d'] 
1879
1880  Dictnames = ['Function', 'User']
1881
1882  if not cachedir:
1883    cachedir = checkdir(options['cachedir'])
1884
1885  SD = os.path.expanduser(cachedir)  # Expand ~ or ~user in pathname
1886
1887  if period == -1:  # Take all available stats
1888    SFILENAME = statsfile
1889  else:  # Only stats from current month 
1890       # MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1891    TimeTuple = localtime(time())
1892    extension = strftime('%b%Y',TimeTuple)
1893    SFILENAME = statsfile+'.'+extension
1894
1895  DIRLIST = os.listdir(SD)
1896  SF = []
1897  for FN in DIRLIST:
1898    if find(FN,SFILENAME) >= 0:
1899      SF.append(FN)
1900
1901  blocksize = 15000000
1902  total_read = 0
1903  total_hits = 0
1904  total_discarded = 0
1905  firstday = mktime(strptime('2030','%Y'))
1906             # FIXME: strptime don't exist in WINDOWS ?
1907  lastday = 0
1908
1909  FuncDict = {}
1910  UserDict = {}
1911  for FN in SF:
1912    input = open(SD+FN,'r')
1913    print 'Reading file ', SD+FN
1914
1915    while 1:
1916      A = input.readlines(blocksize)
1917      if len(A) == 0: break
1918      total_read = total_read + len(A)
1919      for record in A:
1920        record = tuple(split(rstrip(record),','))
1921        #print record, len(record)
1922
1923        if len(record) == 9:
1924          timestamp = record[0]
1925       
1926          try:
1927            t = mktime(strptime(timestamp))
1928          except:
1929            total_discarded = total_discarded + 1         
1930            continue   
1931             
1932          if t > lastday:
1933            lastday = t
1934          if t < firstday:
1935            firstday = t
1936
1937          user     = record[1]
1938          func     = record[2]
1939
1940          # Strip hash-stamp off
1941          #
1942          i = find(func,'[')
1943          func = func[:i]
1944
1945          size        = float(record[3])
1946
1947          # Compression kepword can be Boolean
1948          if record[4] in ['True', '1']:
1949            compression = 1
1950          elif record[4] in ['False', '0']: 
1951            compression = 0
1952          else:
1953            print 'Unknown value of compression', record[4]
1954            print record
1955            total_discarded = total_discarded + 1           
1956            continue
1957
1958          #compression = int(record[4]) # Can be Boolean
1959          hit         = int(record[5])
1960          reason      = int(record[6])   # Not used here   
1961          cputime     = float(record[7])
1962          loadtime    = float(record[8])
1963
1964          if hit:
1965            total_hits = total_hits + 1
1966            saving = cputime-loadtime
1967
1968            if cputime != 0:
1969              rel_saving = round(100.0*saving/cputime,2)
1970            else:
1971              #rel_saving = round(1.0*saving,2)
1972              rel_saving = 100.0 - round(1.0*saving,2)  # A bit of a hack
1973
1974            info = [1,cputime,loadtime,saving,rel_saving,size]
1975
1976            UpdateDict(UserDict,user,info)
1977            UpdateDict(FuncDict,func,info)
1978          else:
1979            pass #Stats on recomputations and their reasons could go in here
1980             
1981        else:
1982          #print 'Record discarded'
1983          #print record
1984          total_discarded = total_discarded + 1
1985
1986    input.close()
1987
1988  # Compute averages of all sums and write list
1989  #
1990
1991  if total_read == 0:
1992    printline(Widths,'=')
1993    print 'CACHING STATISTICS: No valid records read'
1994    printline(Widths,'=')
1995    return
1996
1997  print
1998  printline(Widths,'=')
1999  print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
2000  printline(Widths,'=')
2001  #print '  Period:', ctime(firstday), 'to', ctime(lastday)
2002  print '  Total number of valid records', total_read
2003  print '  Total number of discarded records', total_discarded
2004  print '  Total number of hits', total_hits
2005  print
2006
2007  print '  Fields', Fields[2:], 'are averaged over number of hits'
2008  print '  Time is measured in seconds and size in bytes'
2009  print '  Tables are sorted by', Fields[1:][sortidx]
2010
2011  # printline(Widths,'-')
2012
2013  if showuser:
2014    Dictionaries = [FuncDict, UserDict]
2015  else:
2016    Dictionaries = [FuncDict]
2017
2018  i = 0
2019  for Dict in Dictionaries:
2020    for key in Dict.keys():
2021      rec = Dict[key]
2022      for n in range(len(rec)):
2023        if n > 0:
2024          rec[n] = round(1.0*rec[n]/rec[0],2)
2025      Dict[key] = rec
2026
2027    # Sort and output
2028    #
2029    keylist = SortDict(Dict,sortidx)
2030
2031    # Write Header
2032    #
2033    print
2034    #print Dictnames[i], 'statistics:'; i=i+1
2035    printline(Widths,'-')
2036    n = 0
2037    for s in Fields:
2038      if s == Fields[0]:  # Left justify
2039        s = Dictnames[i] + ' ' + s; i=i+1
2040        exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
2041      else:
2042        exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
2043    print
2044    printline(Widths,'-')
2045
2046    # Output Values
2047    #
2048    for key in keylist:
2049      rec = Dict[key]
2050      n = 0
2051      if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
2052      exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
2053      for val in rec:
2054        exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
2055      print
2056    print
2057
2058#==============================================================================
2059# Auxiliary stats functions
2060#==============================================================================
2061
2062def UpdateDict(Dict,key,info):
2063  """Update dictionary by adding new values to existing.
2064
2065  USAGE:
2066    UpdateDict(Dict,key,info)
2067  """
2068
2069  if Dict.has_key(key):
2070    dinfo = Dict[key]
2071    for n in range(len(dinfo)):
2072      dinfo[n] = info[n] + dinfo[n]
2073  else:
2074    dinfo = info[:]  # Make a copy of info list
2075
2076  Dict[key] = dinfo
2077  return Dict
2078
2079# -----------------------------------------------------------------------------
2080
2081def SortDict(Dict,sortidx=0):
2082  """Sort dictionary
2083
2084  USAGE:
2085    SortDict(Dict,sortidx):
2086
2087  DESCRIPTION:
2088    Sort dictionary of lists according field number 'sortidx'
2089  """
2090
2091  import types
2092
2093  sortlist  = []
2094  keylist = Dict.keys()
2095  for key in keylist:
2096    rec = Dict[key]
2097    if not type(rec) in [types.ListType, types.TupleType]:
2098      rec = [rec]
2099
2100    if sortidx > len(rec)-1:
2101      if options['verbose']:
2102        print 'ERROR: Sorting index to large, sortidx = ', sortidx
2103      raise IndexError
2104
2105    val = rec[sortidx]
2106    sortlist.append(val)
2107
2108  A = map(None,sortlist,keylist)
2109  A.sort()
2110  keylist = map(lambda x: x[1], A)  # keylist sorted by sortidx
2111
2112  return(keylist)
2113
2114# -----------------------------------------------------------------------------
2115
2116def printline(Widths,char):
2117  """Print textline in fixed field.
2118
2119  USAGE:
2120    printline(Widths,char)
2121  """
2122
2123  s = ''
2124  for n in range(len(Widths)):
2125    s = s+Widths[n]*char
2126    if n > 0:
2127      s = s+char
2128
2129  print s
2130
2131#==============================================================================
2132# Messages
2133#==============================================================================
2134
2135def msg1(funcname,args,kwargs,reason):
2136  """Message 1
2137
2138  USAGE:
2139    msg1(funcname,args,kwargs,reason):
2140  """
2141
2142  import string
2143  #print 'MESSAGE (caching.py): Evaluating function', funcname,
2144
2145  print_header_box('Evaluating function %s' %funcname)
2146 
2147  msg7(args, kwargs)
2148  msg8(reason) 
2149 
2150  print_footer()
2151 
2152  #
2153  # Old message
2154  #
2155  #args_present = 0
2156  #if args:
2157  #  if len(args) == 1:
2158  #    print 'with argument', mkargstr(args[0], textwidth2),
2159  #  else:
2160  #    print 'with arguments', mkargstr(args, textwidth2),
2161  #  args_present = 1     
2162  #   
2163  #if kwargs:
2164  #  if args_present:
2165  #    word = 'and'
2166  #  else:
2167  #    word = 'with'
2168  #     
2169  #  if len(kwargs) == 1:
2170  #    print word + ' keyword argument', mkargstr(kwargs, textwidth2)
2171  #  else:
2172  #    print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
2173  #  args_present = 1           
2174  #else:
2175  #  print    # Newline when no keyword args present
2176  #       
2177  #if not args_present:   
2178  #  print '',  # Default if no args or kwargs present
2179   
2180   
2181
2182# -----------------------------------------------------------------------------
2183
2184def msg2(funcname,args,kwargs,comptime,reason):
2185  """Message 2
2186
2187  USAGE:
2188    msg2(funcname,args,kwargs,comptime,reason)
2189  """
2190
2191  import string
2192
2193  #try:
2194  #  R = Reason_msg[reason]
2195  #except:
2196  #  R = 'Unknown reason' 
2197 
2198  #print_header_box('Caching statistics (storing) - %s' %R)
2199  print_header_box('Caching statistics (storing)') 
2200 
2201  msg6(funcname,args,kwargs)
2202  msg8(reason)
2203
2204  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2205
2206# -----------------------------------------------------------------------------
2207
2208def msg3(savetime, CD, FN, deps,compression):
2209  """Message 3
2210
2211  USAGE:
2212    msg3(savetime, CD, FN, deps,compression)
2213  """
2214
2215  import string
2216  print string.ljust('| Loading time:', textwidth1) + str(round(savetime,2)) + \
2217                     ' seconds (estimated)'
2218  msg5(CD,FN,deps,compression)
2219
2220# -----------------------------------------------------------------------------
2221
2222def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
2223  """Message 4
2224
2225  USAGE:
2226    msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
2227  """
2228
2229  import string
2230
2231  print_header_box('Caching statistics (retrieving)')
2232 
2233  msg6(funcname,args,kwargs)
2234  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2235  print string.ljust('| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
2236  print string.ljust('| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
2237        ' seconds'
2238  msg5(CD,FN,deps,compression)
2239
2240# -----------------------------------------------------------------------------
2241
2242def msg5(CD,FN,deps,compression):
2243  """Message 5
2244
2245  USAGE:
2246    msg5(CD,FN,deps,compression)
2247
2248  DESCRIPTION:
2249   Print dependency stats. Used by msg3 and msg4
2250  """
2251
2252  import os, time, string
2253
2254  print '|'
2255  print string.ljust('| Caching dir: ', textwidth1) + CD
2256
2257  if compression:
2258    suffix = '.z'
2259    bytetext = 'bytes, compressed'
2260  else:
2261    suffix = ''
2262    bytetext = 'bytes'
2263
2264  for file_type in file_types:
2265    file_name = FN + '_' + file_type + suffix
2266    print string.ljust('| ' + file_type + ' file: ', textwidth1) + file_name,
2267    stats = os.stat(CD+file_name)
2268    print '('+ str(stats[6]) + ' ' + bytetext + ')'
2269
2270  print '|'
2271  if len(deps) > 0:
2272    print '| Dependencies:  '
2273    dependencies  = deps.keys()
2274    dlist = []; maxd = 0
2275    tlist = []; maxt = 0
2276    slist = []; maxs = 0
2277    for d in dependencies:
2278      stats = deps[d]
2279      t = time.ctime(stats[1])
2280      s = str(stats[0])
2281      #if s[-1] == 'L':
2282      #  s = s[:-1]  # Strip rightmost 'long integer' L off.
2283      #              # FIXME: Unnecessary in versions later than 1.5.2
2284
2285      if len(d) > maxd: maxd = len(d)
2286      if len(t) > maxt: maxt = len(t)
2287      if len(s) > maxs: maxs = len(s)
2288      dlist.append(d)
2289      tlist.append(t)
2290      slist.append(s)
2291
2292    for n in range(len(dlist)):
2293      d = string.ljust(dlist[n]+':', maxd+1)
2294      t = string.ljust(tlist[n], maxt)
2295      s = string.rjust(slist[n], maxs)
2296
2297      print '| ', d, t, ' ', s, 'bytes'
2298  else:
2299    print '| No dependencies'
2300  print_footer()
2301
2302# -----------------------------------------------------------------------------
2303
2304def msg6(funcname,args,kwargs):
2305  """Message 6
2306
2307  USAGE:
2308    msg6(funcname,args,kwargs)
2309  """
2310
2311  import string
2312  print string.ljust('| Function:', textwidth1) + funcname
2313
2314  msg7(args, kwargs)
2315 
2316# -----------------------------------------------------------------------------   
2317
2318def msg7(args, kwargs):
2319  """Message 7
2320 
2321  USAGE:
2322    msg7(args, kwargs):
2323  """
2324 
2325  import string
2326 
2327  args_present = 0 
2328  if args:
2329    if len(args) == 1:
2330      print string.ljust('| Argument:', textwidth1) + mkargstr(args[0], \
2331                         textwidth2)
2332    else:
2333      print string.ljust('| Arguments:', textwidth1) + \
2334            mkargstr(args, textwidth2)
2335    args_present = 1
2336           
2337  if kwargs:
2338    if len(kwargs) == 1:
2339      print string.ljust('| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
2340                         textwidth2)
2341    else:
2342      print string.ljust('| Keyword Args:', textwidth1) + \
2343            mkargstr(kwargs, textwidth2)
2344    args_present = 1
2345
2346  if not args_present:               
2347    print '| No arguments' # Default if no args or kwargs present
2348
2349# -----------------------------------------------------------------------------
2350
2351def msg8(reason):
2352  """Message 8
2353 
2354  USAGE:
2355    msg8(reason):
2356  """
2357 
2358  import string
2359   
2360  try:
2361    R = Reason_msg[reason]
2362  except:
2363    R = 'Unknown' 
2364 
2365  print string.ljust('| Reason:', textwidth1) + R
2366   
2367# -----------------------------------------------------------------------------
2368
2369def print_header_box(line):
2370  """Print line in a nice box.
2371 
2372  USAGE:
2373    print_header_box(line)
2374
2375  """
2376  global textwidth3
2377
2378  import time
2379
2380  time_stamp = time.ctime(time.time())
2381  line = time_stamp + '. ' + line
2382   
2383  N = len(line) + 1
2384  s = '+' + '-'*N + CR
2385
2386  print s + '| ' + line + CR + s,
2387
2388  textwidth3 = N
2389
2390# -----------------------------------------------------------------------------
2391   
2392def print_footer():
2393  """Print line same width as that of print_header_box.
2394  """
2395 
2396  N = textwidth3
2397  s = '+' + '-'*N + CR   
2398     
2399  print s     
2400     
2401# -----------------------------------------------------------------------------
2402
2403def mkargstr(args, textwidth, argstr = '', level=0):
2404  """ Generate a string containing first textwidth characters of arguments.
2405
2406  USAGE:
2407    mkargstr(args, textwidth, argstr = '', level=0)
2408
2409  DESCRIPTION:
2410    Exactly the same as str(args) possibly followed by truncation,
2411    but faster if args is huge.
2412  """
2413
2414  import types
2415
2416  if level > 10:
2417      # Protect against circular structures
2418      return '...'
2419 
2420  WasTruncated = 0
2421
2422  if not type(args) in [types.TupleType, types.ListType, types.DictType]:
2423    if type(args) == types.StringType:
2424      argstr = argstr + "'"+str(args)+"'"
2425    else:
2426      # Truncate large Numeric arrays before using str()
2427      import Numeric
2428      if type(args) == Numeric.ArrayType:
2429#        if len(args.flat) > textwidth: 
2430#        Changed by Duncan and Nick 21/2/07 .flat has problems with
2431#        non-contigous arrays and ravel is equal to .flat except it
2432#        can work with non-contiguous  arrays
2433        if len(Numeric.ravel(args)) > textwidth:
2434          args = 'Array: ' + str(args.shape)
2435
2436      argstr = argstr + str(args)
2437  else:
2438    if type(args) == types.DictType:
2439      argstr = argstr + "{"
2440      for key in args.keys():
2441        argstr = argstr + mkargstr(key, textwidth, level=level+1) + ": " + \
2442                 mkargstr(args[key], textwidth, level=level+1) + ", "
2443        if len(argstr) > textwidth:
2444          WasTruncated = 1
2445          break
2446      argstr = argstr[:-2]  # Strip off trailing comma     
2447      argstr = argstr + "}"
2448
2449    else:
2450      if type(args) == types.TupleType:
2451        lc = '('
2452        rc = ')'
2453      else:
2454        lc = '['
2455        rc = ']'
2456      argstr = argstr + lc
2457      for arg in args:
2458        argstr = argstr + mkargstr(arg, textwidth, level=level+1) + ', '
2459        if len(argstr) > textwidth:
2460          WasTruncated = 1
2461          break
2462
2463      # Strip off trailing comma and space unless singleton tuple
2464      #
2465      if type(args) == types.TupleType and len(args) == 1:
2466        argstr = argstr[:-1]   
2467      else:
2468        argstr = argstr[:-2]
2469      argstr = argstr + rc
2470
2471  if len(argstr) > textwidth:
2472    WasTruncated = 1
2473
2474  if WasTruncated:
2475    argstr = argstr[:textwidth]+'...'
2476  return(argstr)
2477
2478# -----------------------------------------------------------------------------
2479
2480def test_OK(msg):
2481  """Print OK msg if test is OK.
2482 
2483  USAGE
2484    test_OK(message)
2485  """
2486
2487  import string
2488   
2489  print string.ljust(msg, textwidth4) + ' - OK' 
2490 
2491  #raise StandardError
2492 
2493# -----------------------------------------------------------------------------
2494
2495def test_error(msg):
2496  """Print error if test fails.
2497 
2498  USAGE
2499    test_error(message)
2500  """
2501 
2502  print 'ERROR (caching.test): %s' %msg
2503  print 'Please send this code example and output to '
2504  print 'Ole.Nielsen@anu.edu.au'
2505  print
2506  print
2507 
2508  #import sys
2509  #sys.exit()
2510  raise StandardError
2511
2512#-------------------------------------------------------------
2513if __name__ == "__main__":
2514  pass
Note: See TracBrowser for help on using the repository browser.