source: anuga_core/source/anuga/caching/caching.py @ 5855

Last change on this file since 5855 was 5855, checked in by ole, 15 years ago

Much more work on caching and fitting.
Caching of set_quantity as per Okushiri example now works again.

Moreover, caching can now handle instances and ever circular references.

This should address ticket:244 and ticket:302
File size: 73.0 KB
Line 
1# =============================================================================
2# caching.py - Supervised caching of function results.
3# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4# Australian National University (1999-2003)
5# Geoscience Australia (2003-present)
6#
7#    This program is free software; you can redistribute it and/or modify
8#    it under the terms of the GNU General Public License as published by
9#    the Free Software Foundation; either version 2 of the License, or
10#    (at your option) any later version.
11#
12#    This program is distributed in the hope that it will be useful,
13#    but WITHOUT ANY WARRANTY; without even the implied warranty of
14#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15#    GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16#    for more details.
17#
18#    You should have received a copy of the GNU General Public License
19#    along with this program; if not, write to the Free Software
20#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
21#
22#
23# Contact address: Ole.Nielsen@ga.gov.au
24#
25# Version 1.5.6 February 2002
26# =============================================================================
27 
28"""Module caching.py - Supervised caching of function results.
29
30Public functions:
31
32cache(func,args) -- Cache values returned from func given args.
33cachestat() --      Reports statistics about cache hits and time saved.
34test() --       Conducts a basic test of the caching functionality.
35
36See doc strings of individual functions for detailed documentation.
37"""
38
39# -----------------------------------------------------------------------------
40# Initialisation code
41
42# Determine platform
43#
44from os import getenv
45import types
46
47import os
48if os.name in ['nt', 'dos', 'win32', 'what else?']:
49  unix = 0
50else:
51  unix = 1
52
53cache_dir = '.python_cache'
54# Make default caching directory name
55#
56if unix:
57  homedir = getenv('INUNDATIONHOME')
58  if homedir is None:
59    homedir = '~'
60  else:
61    homedir = homedir + os.sep + '.cache'
62    # Since homedir will be a group area, individually label the caches
63    user = getenv('LOGNAME')
64    if user is not None:
65      cache_dir += '_' + user
66   
67  CR = '\n'
68else:
69  homedir = 'c:'
70  CR = '\r\n'  #FIXME: Not tested under windows
71 
72cachedir = homedir + os.sep + cache_dir + os.sep
73
74# -----------------------------------------------------------------------------
75# Options directory with default values - to be set by user
76#
77
78options = { 
79  'cachedir': cachedir,  # Default cache directory
80  'maxfiles': 1000000,   # Maximum number of cached files
81  'savestat': 1,         # Log caching info to stats file
82  'verbose': 1,          # Write messages to standard output
83  'bin': 1,              # Use binary format (more efficient)
84  'compression': 1,      # Use zlib compression
85  'bytecode': 0,         # Recompute if bytecode has changed
86  'expire': 0            # Automatically remove files that have been accessed
87                         # least recently
88}
89
90# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
91
92def set_option(key, value):
93  """Function to set values in the options directory.
94
95  USAGE:
96    set_option(key, value)
97
98  ARGUMENTS:
99    key --   Key in options dictionary. (Required)
100    value -- New value for key. (Required)
101
102  DESCRIPTION:
103    Function to set values in the options directory.
104    Raises an exception if key is not in options.
105  """
106
107  if options.has_key(key):
108    options[key] = value
109  else:
110    raise KeyError(key)  # Key not found, raise an exception
111
112# -----------------------------------------------------------------------------
113# Function cache - the main routine
114
115def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
116          verbose=None, compression=None, evaluate=0, test=0, clear=0,
117          return_filename=0):
118  """Supervised caching of function results.
119
120  USAGE:
121    result = cache(func, args, kwargs, dependencies, cachedir, verbose,
122                   compression, evaluate, test, return_filename)
123
124  ARGUMENTS:
125    func --            Function object (Required)
126    args --            Arguments to func (Default: ())
127    kwargs --          Keyword arguments to func (Default: {})   
128    dependencies --    Filenames that func depends on (Default: None)
129    cachedir --        Directory for cache files (Default: options['cachedir'])
130    verbose --         Flag verbose output to stdout
131                       (Default: options['verbose'])
132    compression --     Flag zlib compression (Default: options['compression'])
133    evaluate --        Flag forced evaluation of func (Default: 0)
134    test --            Flag test for cached results (Default: 0)
135    clear --           Flag delete cached results (Default: 0)   
136    return_filename -- Flag return of cache filename (Default: 0)   
137
138  DESCRIPTION:
139    A Python function call of the form
140
141      result = func(arg1,...,argn)
142
143    can be replaced by
144
145      from caching import cache
146      result = cache(func,(arg1,...,argn))
147
148  The latter form returns the same output as the former but reuses cached
149  results if the function has been computed previously in the same context.
150  'result' and the arguments can be simple types, tuples, list, dictionaries or
151  objects, but not unhashable types such as functions or open file objects.
152  The function 'func' may be a member function of an object or a module.
153
154  This type of caching is particularly useful for computationally intensive
155  functions with few frequently used combinations of input arguments. Note that
156  if the inputs or output are very large caching might not save time because
157  disc access may dominate the execution time.
158
159  If the function definition changes after a result has been cached it will be
160  detected by examining the functions bytecode (co_code, co_consts,
161  func_defualts, co_argcount) and it will be recomputed.
162
163  LIMITATIONS:
164    1 Caching uses the apply function and will work with anything that can be
165      pickled, so any limitation in apply or pickle extends to caching.
166    2 A function to be cached should not depend on global variables
167      as wrong results may occur if globals are changed after a result has
168      been cached.
169
170  -----------------------------------------------------------------------------
171  Additional functionality:
172
173  Keyword args
174    Keyword arguments (kwargs) can be added as a dictionary of keyword: value
175    pairs, following the syntax of the built-in function apply().
176    A Python function call of the form
177   
178      result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)   
179
180    is then cached as follows
181
182      from caching import cache
183      result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
184   
185    The default value of kwargs is {} 
186
187  Explicit dependencies:
188    The call
189      cache(func,(arg1,...,argn),dependencies = <list of filenames>)
190    Checks the size, creation time and modification time of each listed file.
191    If any file has changed the function is recomputed and the results stored
192    again.
193
194  Specify caching directory:
195    The call
196      cache(func,(arg1,...,argn), cachedir = <cachedir>)
197    designates <cachedir> where cached data are stored. Use ~ to indicate users
198    home directory - not $HOME. The default is ~/.python_cache on a UNIX
199    platform and c:/.python_cache on a Win platform.
200
201  Silent operation:
202    The call
203      cache(func,(arg1,...,argn),verbose=0)
204    suppresses messages to standard output.
205
206  Compression:
207    The call
208      cache(func,(arg1,...,argn),compression=0)
209    disables compression. (Default: compression=1). If the requested compressed
210    or uncompressed file is not there, it'll try the other version.
211
212  Forced evaluation:
213    The call
214      cache(func,(arg1,...,argn),evaluate=1)
215    forces the function to evaluate even though cached data may exist.
216
217  Testing for presence of cached result:
218    The call
219      cache(func,(arg1,...,argn),test=1)
220    retrieves cached result if it exists, otherwise None. The function will not
221    be evaluated. If both evaluate and test are switched on, evaluate takes
222    precedence.
223   
224  Obtain cache filenames:
225    The call   
226      cache(func,(arg1,...,argn),return_filename=1)
227    returns the hashed base filename under which this function and its
228    arguments would be cached
229
230  Clearing cached results:
231    The call
232      cache(func,'clear')
233    clears all cached data for 'func' and
234      cache('clear')
235    clears all cached data.
236 
237    NOTE: The string 'clear' can be passed an *argument* to func using
238      cache(func,('clear',)) or cache(func,tuple(['clear'])).
239
240    New form of clear:
241      cache(func,(arg1,...,argn),clear=1)
242    clears cached data for particular combination func and args
243     
244  """
245
246  # Imports and input checks
247  #
248  import types, time, string
249
250  if not cachedir:
251    cachedir = options['cachedir']
252
253  if verbose == None:  # Do NOT write 'if not verbose:', it could be zero.
254    verbose = options['verbose']
255
256  if compression == None:  # Do NOT write 'if not compression:',
257                           # it could be zero.
258    compression = options['compression']
259
260  # Create cache directory if needed
261  CD = checkdir(cachedir,verbose)
262
263  # Handle the case cache('clear')
264  if type(func) == types.StringType:
265    if string.lower(func) == 'clear':
266      clear_cache(CD,verbose=verbose)
267      return
268
269  # Handle the case cache(func, 'clear')
270  if type(args) == types.StringType:
271    if string.lower(args) == 'clear':
272      clear_cache(CD,func,verbose=verbose)
273      return
274
275  # Force singleton arg into a tuple
276  if type(args) != types.TupleType:
277    args = tuple([args])
278 
279  # Check that kwargs is a dictionary
280  if type(kwargs) != types.DictType:
281    raise TypeError   
282   
283  # Hash arguments (and keyword args) to integer
284  arghash = myhash((args, kwargs))
285 
286  # Get sizes and timestamps for files listed in dependencies.
287  # Force singletons into a tuple.
288  if dependencies and type(dependencies) != types.TupleType \
289                  and type(dependencies) != types.ListType:
290    dependencies = tuple([dependencies])
291  deps = get_depstats(dependencies)
292
293  # Extract function name from func object
294  funcname = get_funcname(func)
295
296  # Create cache filename
297  FN = funcname+'['+`arghash`+']'  # The symbol '(' does not work under unix
298
299  if return_filename:
300    return(FN)
301
302  if clear:
303    for file_type in file_types:
304      file_name = CD+FN+'_'+file_type
305      for fn in [file_name, file_name + '.z']:
306        if os.access(fn, os.F_OK):             
307          if unix:
308            os.remove(fn)
309          else:
310            # FIXME: os.remove doesn't work under windows       
311            os.system('del '+fn)
312          if verbose is True:
313            print 'MESSAGE (caching): File %s deleted' %fn
314        ##else:
315        ##  print '%s was not accessed' %fn
316    return None
317
318
319  #-------------------------------------------------------------------       
320 
321  # Check if previous computation has been cached
322  if evaluate:
323    Retrieved = None  # Force evaluation of func regardless of caching status.
324    reason = 5
325  else:
326    T, FN, Retrieved, reason, comptime, loadtime, compressed = \
327        CacheLookup(CD, FN, func, 
328                    args, kwargs, 
329                    deps, 
330                    verbose, 
331                    compression,
332                    dependencies)
333
334  if not Retrieved:
335    if test:  # Do not attempt to evaluate function
336      T = None
337    else:  # Evaluate function and save to cache
338      if verbose:
339       
340        msg1(funcname, args, kwargs,reason)
341
342      # Remove expired files automatically
343      if options['expire']:
344        DeleteOldFiles(CD,verbose)
345       
346      # Save args before function is evaluated in case
347      # they are modified by function
348      save_args_to_cache(CD,FN,args,kwargs,compression)
349
350      # Execute and time function with supplied arguments
351      t0 = time.time()
352      T = apply(func,args,kwargs)
353      #comptime = round(time.time()-t0)
354      comptime = time.time()-t0
355
356      if verbose:
357        msg2(funcname,args,kwargs,comptime,reason)
358
359      # Save results and estimated loading time to cache
360      loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
361                                       funcname, dependencies, compression)
362      if verbose:
363        msg3(loadtime, CD, FN, deps, compression)
364      compressed = compression
365
366  if options['savestat'] and (not test or Retrieved):
367  ##if options['savestat']:
368    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
369
370  return(T)  # Return results in all cases
371
372# -----------------------------------------------------------------------------
373
374def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
375  """Generate statistics of caching efficiency.
376
377  USAGE:
378    cachestat(sortidx, period, showuser, cachedir)
379
380  ARGUMENTS:
381    sortidx --  Index of field by which lists are (default: 4)
382                Legal values are
383                 0: 'Name'
384                 1: 'Hits'
385                 2: 'CPU'
386                 3: 'Time Saved'
387                 4: 'Gain(%)'
388                 5: 'Size'
389    period --   If set to -1 all available caching history is used.
390                If set 0 only the current month is used (default -1).
391    showuser -- Flag for additional table showing user statistics
392                (default: None).
393    cachedir -- Directory for cache files (default: options['cachedir']).
394
395  DESCRIPTION:
396    Logged caching statistics is converted into summaries of the form
397    --------------------------------------------------------------------------
398    Function Name   Hits   Exec(s)  Cache(s)  Saved(s)   Gain(%)      Size
399    --------------------------------------------------------------------------
400  """
401
402  __cachestat(sortidx, period, showuser, cachedir)
403  return
404
405# -----------------------------------------------------------------------------
406
407#Has mostly been moved to proper unit test
408def test(cachedir=None,verbose=0,compression=None):
409  """Test the functionality of caching.
410
411  USAGE:
412    test(verbose)
413
414  ARGUMENTS:
415    verbose --     Flag whether caching will output its statistics (default=0)
416    cachedir --    Directory for cache files (Default: options['cachedir'])
417    compression -- Flag zlib compression (Default: options['compression'])
418  """
419   
420  import string, time
421
422  # Initialise
423  #
424  import caching
425  reload(caching)
426
427  if not cachedir:
428    cachedir = options['cachedir']
429
430  if verbose is None:  # Do NOT write 'if not verbose:', it could be zero.
431    verbose = options['verbose']
432 
433  if compression == None:  # Do NOT write 'if not compression:',
434                           # it could be zero.
435    compression = options['compression']
436  else:
437    try:
438      set_option('compression', compression)
439    except:
440      test_error('Set option failed')     
441
442  try:
443    import zlib
444  except:
445    print
446    print '*** Could not find zlib, default to no-compression      ***'
447    print '*** Installing zlib will improve performance of caching ***'
448    print
449    compression = 0       
450    set_option('compression', compression)   
451 
452  print 
453  print_header_box('Testing caching module - please stand by')
454  print   
455
456  # Define a test function to be cached
457  #
458  def f(a,b,c,N,x=0,y='abcdefg'):
459    """f(a,b,c,N)
460       Do something time consuming and produce a complex result.
461    """
462
463    import string
464
465    B = []
466    for n in range(N):
467      s = str(n+2.0/(n + 4.0))+'.a'*10
468      B.append((a,b,c,s,n,x,y))
469    return(B)
470   
471  # Check that default cachedir is OK
472  #     
473  CD = checkdir(cachedir,verbose)   
474   
475   
476  # Make a dependency file
477  #   
478  try:
479    DepFN = CD + 'testfile.tmp'
480    DepFN_wildcard = CD + 'test*.tmp'
481    Depfile = open(DepFN,'w')
482    Depfile.write('We are the knights who say NI!')
483    Depfile.close()
484    test_OK('Wrote file %s' %DepFN)
485  except:
486    test_error('Could not open file %s for writing - check your environment' \
487               % DepFN)
488
489  # Check set_option (and switch stats off
490  #   
491  try:
492    set_option('savestat',0)
493    assert(options['savestat'] == 0)
494    test_OK('Set option')
495  except:
496    test_error('Set option failed')   
497   
498  # Make some test input arguments
499  #
500  N = 5000  #Make N fairly small here
501
502  a = [1,2]
503  b = ('Thou shalt count the number three',4)
504  c = {'Five is right out': 6, (7,8): 9}
505  x = 3
506  y = 'holy hand granate'
507
508  # Test caching
509  #
510  if compression:
511    comprange = 2
512  else:
513    comprange = 1
514
515  for comp in range(comprange):
516 
517    # Evaluate and store
518    #
519    try:
520      T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
521                         verbose=verbose, compression=comp)
522      if comp:                   
523        test_OK('Caching evaluation with compression')
524      else:     
525        test_OK('Caching evaluation without compression')     
526    except:
527      if comp:
528        test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
529      else:
530        test_error('Caching evaluation failed - try caching.test(verbose=1)')
531
532    # Retrieve
533    #                           
534    try:                         
535      T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
536                         compression=comp) 
537
538      if comp:                   
539        test_OK('Caching retrieval with compression')
540      else:     
541        test_OK('Caching retrieval without compression')     
542    except:
543      if comp:
544        test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
545      else:                                     
546        test_error('Caching retrieval failed - try caching.test(verbose=1)')
547
548    # Reference result
549    #   
550    T3 = f(a,b,c,N,x=x,y=y)  # Compute without caching
551   
552    if T1 == T2 and T2 == T3:
553      if comp:
554        test_OK('Basic caching functionality (with compression)')
555      else:
556        test_OK('Basic caching functionality (without compression)')
557    else:
558      test_error('Cached result does not match computed result')
559
560
561  # Test return_filename
562  #   
563  try:
564    FN = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
565                       return_filename=1)   
566    assert(FN[:2] == 'f[')
567    test_OK('Return of cache filename')
568  except:
569    test_error('Return of cache filename failed')
570
571  # Test existence of cachefiles
572  # 
573  try:
574    (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
575    (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
576    (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
577    test_OK('Presence of cache files')
578    datafile.close()
579    argsfile.close()
580    admfile.close()
581  except:
582    test_error('Expected cache files did not exist') 
583             
584  # Test 'test' function when cache is present
585  #     
586  try:
587    #T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
588    #                   evaluate=1) 
589    T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
590    assert(T1 == T4)
591
592    test_OK("Option 'test' when cache file present")
593  except:
594    test_error("Option 'test' when cache file present failed")     
595
596  # Test that 'clear' works
597  #
598  #try:
599  #  caching.cache(f,'clear',verbose=verbose)
600  #  test_OK('Clearing of cache files')
601  #except:
602  #  test_error('Clear does not work')
603  try:
604    caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)   
605    test_OK('Clearing of cache files')
606  except:
607    test_error('Clear does not work') 
608
609 
610
611  # Test 'test' function when cache is absent
612  #     
613  try:
614    T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
615    assert(T4 is None)
616    test_OK("Option 'test' when cache absent")
617  except:
618    test_error("Option 'test' when cache absent failed")     
619         
620  # Test dependencies
621  #
622  T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
623                       dependencies=DepFN) 
624  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
625                       dependencies=DepFN)                     
626                       
627  if T1 == T2:
628    test_OK('Basic dependencies functionality')
629  else:
630    test_error('Dependencies do not work')
631
632  # Test basic wildcard dependency
633  #
634  T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
635                       dependencies=DepFN_wildcard)                     
636   
637  if T1 == T3:
638    test_OK('Basic dependencies with wildcard functionality')
639  else:
640    test_error('Dependencies with wildcards do not work')
641
642
643  # Test that changed timestamp in dependencies triggers recomputation
644 
645  # Modify dependency file
646  Depfile = open(DepFN,'a')
647  Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
648  Depfile.close()
649 
650  T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
651                       dependencies=DepFN, test = 1)                     
652 
653  if T3 is None:
654    test_OK('Changed dependencies recognised')
655  else:
656    test_error('Changed dependencies not recognised')   
657 
658  # Test recomputation when dependencies have changed
659  #
660  T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
661                       dependencies=DepFN)                       
662  if T1 == T3:
663    test_OK('Recomputed value with changed dependencies')
664  else:
665    test_error('Recomputed value with changed dependencies failed')
666
667  # Performance test (with statistics)
668  # Don't really rely on this as it will depend on specific computer.
669  #
670
671  set_option('savestat',1)
672
673  N = 20*N   #Should be large on fast computers...
674  tt = time.time()
675  T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
676  t1 = time.time() - tt
677 
678  tt = time.time()
679  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
680  t2 = time.time() - tt
681 
682  if T1 == T2:
683    if t1 > t2:
684      test_OK('Performance test: relative time saved = %s pct' \
685              %str(round((t1-t2)*100/t1,2)))
686    #else:
687    #  print 'WARNING: Performance a bit low - this could be specific to current platform'
688  else:       
689    test_error('Basic caching failed for new problem')
690           
691  # Test presence of statistics file
692  #
693  try: 
694    DIRLIST = os.listdir(CD)
695    SF = []
696    for FN in DIRLIST:
697      if string.find(FN,statsfile) >= 0:
698        fid = open(CD+FN,'r')
699        fid.close()
700    test_OK('Statistics files present') 
701  except:
702    test_OK('Statistics files cannot be opened')         
703     
704  print_header_box('Show sample output of the caching function:')
705 
706  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
707  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
708  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
709 
710  print_header_box('Show sample output of cachestat():')
711  if unix:
712    cachestat()   
713  else:
714    try:
715      import time
716      t = time.strptime('2030','%Y')
717      cachestat()
718    except: 
719      print 'caching.cachestat() does not work here, because it'
720      print 'relies on time.strptime() which is unavailable in Windows'
721     
722  print
723  test_OK('Caching self test completed')   
724     
725           
726  # Test setoption (not yet implemented)
727  #
728
729 
730#==============================================================================
731# Auxiliary functions
732#==============================================================================
733
734# Import pickler
735# cPickle is used by functions mysave, myload, and compare
736#
737import cPickle  # 10 to 100 times faster than pickle
738pickler = cPickle
739
740# Local immutable constants
741#
742comp_level = 1              # Compression level for zlib.
743                            # comp_level = 1 works well.
744textwidth1 = 16             # Text width of key fields in report forms.
745#textwidth2 = 132            # Maximal width of textual representation of
746textwidth2 = 300            # Maximal width of textual representation of
747                            # arguments.
748textwidth3 = 16             # Initial width of separation lines. Is modified.
749textwidth4 = 50             # Text width in test_OK()
750statsfile  = '.cache_stat'  # Basefilename for cached statistics.
751                            # It will reside in the chosen cache directory.
752
753file_types = ['Result',     # File name extension for cached function results.
754              'Args',       # File name extension for stored function args.
755              'Admin']      # File name extension for administrative info.
756
757Reason_msg = ['OK',         # Verbose reasons for recomputation
758              'No cached result', 
759              'Dependencies have changed', 
760              'Arguments have changed',
761              'Bytecode has changed',
762              'Recomputation was requested by caller',
763              'Cached file was unreadable']             
764             
765# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
766
767def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, 
768                dependencies):
769  """Determine whether cached result exists and return info.
770
771  USAGE:
772    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \ 
773    CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
774                dependencies)
775
776  INPUT ARGUMENTS:
777    CD --            Cache Directory
778    FN --            Suggested cache file name
779    func --          Function object
780    args --          Tuple of arguments
781    kwargs --        Dictionary of keyword arguments   
782    deps --          Dependencies time stamps
783    verbose --       Flag text output
784    compression --   Flag zlib compression
785    dependencies --  Given list of dependencies
786   
787  OUTPUT ARGUMENTS:
788    T --             Cached result if present otherwise None
789    FN --            File name under which new results must be saved
790    Retrieved --     True if a valid cached result was found
791    reason --        0: OK (if Retrieved),
792                     1: No cached result,
793                     2: Dependencies have changed,
794                     3: Arguments have changed
795                     4: Bytecode has changed
796                     5: Recomputation was forced
797                     6: Unreadable file
798    comptime --      Number of seconds it took to computed cachged result
799    loadtime --      Number of seconds it took to load cached result
800    compressed --    Flag (0,1) if cached results were compressed or not
801
802  DESCRIPTION:
803    Determine if cached result exists as follows:
804    Load in saved arguments and bytecode stored under hashed filename.
805    If they are identical to current arguments and bytecode and if dependencies
806    have not changed their time stamp, then return cached result.
807
808    Otherwise return filename under which new results should be cached.
809    Hash collisions are handled recursively by calling CacheLookup again with a
810    modified filename.
811  """
812
813  import time, string, types
814
815  # Assess whether cached result exists - compressed or not.
816  #
817  if verbose:
818    print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
819           %(CD+FN, file_types[0], file_types[1], file_types[2])
820  (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
821  (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
822  (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
823
824  if verbose is True and deps is not None:
825    print 'Caching: Dependencies are', deps.keys()
826
827  if not (argsfile and datafile and admfile) or \
828     not (compressed0 == compressed1 and compressed0 == compressed2):
829    # Cached result does not exist or files were compressed differently
830    #
831    # This will ensure that evaluation will take place unless all files are
832    # present.
833
834    reason = 1
835    return(None,FN,None,reason,None,None,None) #Recompute using same filename
836
837  compressed = compressed0  # Remember if compressed files were actually used
838  datafile.close()
839
840  # Retrieve arguments and adm. info
841  #
842  R, reason = myload(argsfile, compressed)  # The original arguments
843  argsfile.close()
844   
845  if reason > 0:
846      # Recompute using same filename   
847      return(None, FN, None, reason, None, None, None)
848  else:   
849      (argsref, kwargsref) = R
850
851  R, reason = myload(admfile, compressed)
852  admfile.close() 
853
854  if reason > 0:
855    return(None,FN,None,reason,None,None,None) #Recompute using same filename
856
857 
858  depsref  = R[0]  # Dependency statistics
859  comptime = R[1]  # The computation time
860  coderef  = R[2]  # The byte code
861  funcname = R[3]  # The function name
862
863  # Check if dependencies have changed
864  #
865  if dependencies and not compare(depsref, deps):
866    if verbose:
867      print 'MESSAGE (caching.py): Dependencies', dependencies, \
868            'have changed - recomputing'
869    # Don't use cached file - recompute
870    reason = 2
871    return(None, FN, None, reason, None, None, None)
872
873  # Get bytecode from func
874  #
875  bytecode = get_bytecode(func)
876
877  #print 'Diags'
878  #print argsref
879  #print args
880  #print compare(argsref,args),   
881 
882  #print compare(kwargsref,kwargs),
883  #print compare(bytecode,coderef)
884
885  # Check if arguments or bytecode have changed
886  if compare(argsref, args) and compare(kwargsref, kwargs) and \
887     (not options['bytecode'] or compare(bytecode, coderef)):
888
889    # Arguments and dependencies match. Get cached results
890    T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
891    if reason > 0:
892        # Recompute using same FN     
893        return(None, FN, None, reason, None, None, None)
894
895    Retrieved = 1
896    reason = 0
897
898    if verbose:
899      msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
900
901      if loadtime >= comptime:
902        print 'WARNING (caching.py): Caching did not yield any gain.'
903        print '                      Consider executing function ',
904        print '('+funcname+') without caching.'
905  else:
906
907    # Non matching arguments or bytecodes signify a hash-collision.
908    # This is resolved by recursive search of cache filenames
909    # until either a matching or an unused filename is found.
910    #
911    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
912        CacheLookup(CD, FN+'x', func, args, kwargs, deps, 
913                    verbose, compression, dependencies)
914
915    # DEBUGGING
916    # if not Retrieved:
917    #   print 'Arguments did not match'
918    # else:
919    #   print 'Match found !'
920   
921    # The real reason is that args or bytecodes have changed.
922    # Not that the recursive seach has found an unused filename
923    if not Retrieved:
924      if not compare(bytecode, coderef):
925        reason = 4 # Bytecode has changed
926      else:   
927        reason = 3 # Arguments have changed
928       
929   
930  return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
931
932# -----------------------------------------------------------------------------
933
934def clear_cache(CD,func=None, verbose=None):
935  """Clear cache for func.
936
937  USAGE:
938     clear(CD, func, verbose)
939
940  ARGUMENTS:
941     CD --       Caching directory (required)
942     func --     Function object (default: None)
943     verbose --  Flag verbose output (default: None)
944
945  DESCRIPTION:
946
947    If func == None, clear everything,
948    otherwise clear only files pertaining to func.
949  """
950
951  import os, re
952   
953  if CD[-1] != os.sep:
954    CD = CD+os.sep
955 
956  if verbose == None:
957    verbose = options['verbose']
958
959  # FIXME: Windows version needs to be tested
960
961  if func:
962    funcname = get_funcname(func)
963    if verbose:
964      print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
965
966    file_names = os.listdir(CD)
967    for file_name in file_names:
968      #RE = re.search('^' + funcname,file_name)  #Inefficient
969      #if RE:
970      if file_name[:len(funcname)] == funcname:
971        if unix:
972          os.remove(CD+file_name)
973        else:
974          os.system('del '+CD+file_name)
975          # FIXME: os.remove doesn't work under windows
976  else:
977    file_names = os.listdir(CD)
978    if len(file_names) > 0:
979      if verbose:
980        print 'MESSAGE (caching.py): Remove the following files:'
981        for file_name in file_names:
982            print file_name
983
984        A = raw_input('Delete (Y/N)[N] ?')
985      else:
986        A = 'Y' 
987       
988      if A == 'Y' or A == 'y':
989        for file_name in file_names:
990          if unix:
991            os.remove(CD+file_name)
992          else:
993            os.system('del '+CD+file_name)
994            # FIXME: os.remove doesn't work under windows
995          #exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
996
997# -----------------------------------------------------------------------------
998
999def DeleteOldFiles(CD,verbose=None):
1000  """Remove expired files
1001
1002  USAGE:
1003    DeleteOldFiles(CD,verbose=None)
1004  """
1005
1006  if verbose == None:
1007    verbose = options['verbose']
1008
1009  maxfiles = options['maxfiles']
1010
1011  # FIXME: Windows version
1012
1013  import os
1014  block = 1000  # How many files to delete per invokation
1015  Files = os.listdir(CD)
1016  numfiles = len(Files)
1017  if not unix: return  # FIXME: Windows case ?
1018
1019  if numfiles > maxfiles:
1020    delfiles = numfiles-maxfiles+block
1021    if verbose:
1022      print 'Deleting '+`delfiles`+' expired files:'
1023      os.system('ls -lur '+CD+'* | head -' + `delfiles`)            # List them
1024    os.system('ls -ur '+CD+'* | head -' + `delfiles` + ' | xargs /bin/rm')
1025                                                                  # Delete them
1026    # FIXME: Replace this with os.listdir and os.remove
1027
1028# -----------------------------------------------------------------------------
1029
1030def save_args_to_cache(CD, FN, args, kwargs, compression):
1031  """Save arguments to cache
1032
1033  USAGE:
1034    save_args_to_cache(CD,FN,args,kwargs,compression)
1035  """
1036
1037  import time, os, sys, types
1038
1039  (argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1040
1041  if argsfile is None:
1042    msg = 'ERROR (caching): Could not open argsfile for writing: %s' %FN
1043    raise IOError, msg
1044
1045  mysave((args,kwargs),argsfile,compression)  # Save args and kwargs to cache
1046  argsfile.close()
1047
1048  # Change access rights if possible
1049  #
1050  #if unix:
1051  #  try:
1052  #    exitcode=os.system('chmod 666 '+argsfile.name)
1053  #  except:
1054  #    pass
1055  #else:
1056  #  pass  # FIXME: Take care of access rights under Windows
1057
1058  return
1059
1060# -----------------------------------------------------------------------------
1061
1062def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1063                          dependencies, compression):
1064  """Save computed results T and admin info to cache
1065
1066  USAGE:
1067    save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1068                          dependencies, compression)
1069  """
1070
1071  import time, os, sys, types
1072
1073  (datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1074  (admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1075
1076  if not datafile:
1077    if verbose:
1078      print 'ERROR (caching): Could not open %s' %datafile.name
1079    raise IOError
1080
1081  if not admfile:
1082    if verbose:
1083      print 'ERROR (caching): Could not open %s' %admfile.name
1084    raise IOError
1085
1086  t0 = time.time()
1087
1088  mysave(T,datafile,compression)  # Save data to cache
1089  datafile.close()
1090  #savetime = round(time.time()-t0,2)
1091  savetime = time.time()-t0 
1092
1093  bytecode = get_bytecode(func)  # Get bytecode from function object
1094  admtup = (deps, comptime, bytecode, funcname)  # Gather admin info
1095
1096  mysave(admtup,admfile,compression)  # Save admin info to cache
1097  admfile.close()
1098
1099  # Change access rights if possible
1100  #
1101  #if unix:
1102  #  try:
1103  #    exitcode=os.system('chmod 666 '+datafile.name)
1104  #    exitcode=os.system('chmod 666 '+admfile.name)
1105  #  except:
1106  #    pass
1107  #else:
1108  #  pass  # FIXME: Take care of access rights under Windows
1109
1110  return(savetime)
1111
1112# -----------------------------------------------------------------------------
1113
1114def load_from_cache(CD,FN,compression):
1115  """Load previously cached data from file FN
1116
1117  USAGE:
1118    load_from_cache(CD,FN,compression)
1119  """
1120
1121  import time
1122
1123  (datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1124  t0 = time.time()
1125  T, reason = myload(datafile,compressed)
1126
1127  loadtime = time.time()-t0
1128  datafile.close() 
1129
1130  return T, loadtime, compressed, reason
1131
1132# -----------------------------------------------------------------------------
1133
1134def myopen(FN,mode,compression=1):
1135  """Open file FN using given mode
1136
1137  USAGE:
1138    myopen(FN,mode,compression=1)
1139
1140  ARGUMENTS:
1141    FN --           File name to be opened
1142    mode --         Open mode (as in open)
1143    compression --  Flag zlib compression
1144
1145  DESCRIPTION:
1146     if compression
1147       Attempt first to open FN + '.z'
1148       If this fails try to open FN
1149     else do the opposite
1150     Return file handle plus info about whether it was compressed or not.
1151  """
1152
1153  import string
1154
1155  # Determine if file exists already (if writing was requested)
1156  # This info is only used to determine if access modes should be set
1157  #
1158  if 'w' in mode or 'a' in mode:
1159    try:
1160      file = open(FN+'.z','r')
1161      file.close()
1162      new_file = 0
1163    except:
1164      try:
1165        file = open(FN,'r') 
1166        file.close()
1167        new_file = 0
1168      except:
1169        new_file = 1
1170  else:
1171    new_file = 0 #Assume it exists if mode was not 'w'
1172 
1173
1174  compressed = 0
1175  if compression:
1176    try:
1177      file = open(FN+'.z',mode)
1178      compressed = 1
1179    except:
1180      try:
1181        file = open(FN,mode)
1182      except:
1183        file = None
1184  else:
1185    try:
1186      file = open(FN,mode)
1187    except:
1188      try:
1189        file = open(FN+'.z',mode)
1190        compressed = 1
1191      except:
1192        file = None
1193
1194  # Now set access rights if it is a new file
1195  #
1196  if file and new_file:
1197    if unix:
1198      exitcode=os.system('chmod 666 '+file.name)
1199    else:
1200      pass  # FIXME: Take care of access rights under Windows
1201
1202  return(file,compressed)
1203
1204# -----------------------------------------------------------------------------
1205
1206def myload(file, compressed):
1207  """Load data from file
1208
1209  USAGE:
1210    myload(file, compressed)
1211  """
1212
1213  reason = 0
1214  try:
1215    if compressed:
1216      import zlib
1217
1218      RsC = file.read()
1219      try:
1220        Rs  = zlib.decompress(RsC)
1221      except:
1222        #  File "./caching.py", line 1032, in load_from_cache
1223        #  T = myload(datafile,compressed)
1224        #  File "./caching.py", line 1124, in myload
1225        #  Rs  = zlib.decompress(RsC)
1226        #  zlib.error: Error -5 while decompressing data
1227        #print 'ERROR (caching): Could not decompress ', file.name
1228        #raise Exception
1229        reason = 6  # Unreadable file
1230        return None, reason 
1231     
1232     
1233      del RsC  # Free up some space
1234      R = pickler.loads(Rs)
1235    else:
1236      try:
1237        R = pickler.load(file)
1238      #except EOFError, e:
1239      except:
1240        #Catch e.g., file with 0 length or corrupted
1241        reason = 6  # Unreadable file
1242        return None, reason
1243     
1244  except MemoryError:
1245    import sys
1246    if options['verbose']:
1247      print 'ERROR (caching): Out of memory while loading %s, aborting' \
1248            %(file.name)
1249
1250    # Raise the error again for now
1251    #
1252    raise MemoryError
1253
1254  return R, reason
1255
1256# -----------------------------------------------------------------------------
1257
1258def mysave(T,file,compression):
1259  """Save data T to file
1260
1261  USAGE:
1262    mysave(T,file,compression)
1263
1264  """
1265
1266  bin = options['bin']
1267
1268  if compression:
1269    try:
1270      import zlib
1271    except:
1272      print
1273      print '*** Could not find zlib ***'
1274      print '*** Try to run caching with compression off ***'
1275      print "*** caching.set_option('compression', 0) ***"
1276      raise Exception
1277     
1278
1279    try:
1280      Ts  = pickler.dumps(T, bin)
1281    except MemoryError:
1282      msg = '****WARNING (caching.py): Could not pickle data for compression.'
1283      msg += ' Try using compression = False'
1284      raise MemoryError, msg
1285    else: 
1286      # Compressed pickling     
1287      TsC = zlib.compress(Ts, comp_level)
1288      file.write(TsC)
1289  else:
1290      # Uncompressed pickling
1291      pickler.dump(T, file, bin)
1292
1293      # FIXME: This may not work on Windoze network drives.
1294      # The error msg is IOError: [Errno 22] Invalid argument
1295      # Testing with small files was OK, though.
1296      # I think this is an OS problem.
1297
1298      # Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1299     
1300# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1301#
1302#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1303#
1304#The following extract is taken from the site http://www.python.org:
1305#
1306#---------------------------------------------------------------------------------------------
1307#exception IOError
1308#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1309#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1310#---------------------------------------------------------------------------------------------
1311#
1312#The error code(s) that accompany exceptions are described at:
1313#http://www.python.org/dev/doc/devel//lib/module-errno.html
1314#
1315#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1316#
1317#exceptions.IOError invalid argument Errno 22
1318       
1319      #try:
1320      #  pickler.dump(T,file,bin)
1321      #except IOError, e:
1322      #  print e
1323      #  msg = 'Could not store to %s, bin=%s' %(file, bin)
1324      #  raise msg
1325     
1326
1327# -----------------------------------------------------------------------------
1328
1329
1330def Xmyhash(T, ids=None):
1331    import pickle as pickler # Use non-C version here
1332    return hash(pickler.dumps(T,0))
1333
1334   
1335def myhash(T, ids=None):
1336  """Compute hashed integer from a range of inputs.
1337  If T is not hashable being e.g. a tuple T, myhash will recursively
1338  hash the values individually
1339
1340  USAGE:
1341    myhash(T)
1342
1343  ARGUMENTS:
1344    T -- Anything
1345  """
1346
1347  from types import TupleType, ListType, DictType, InstanceType 
1348  from Numeric import ArrayType, average
1349 
1350  #if type(T) in [TupleType, ListType, DictType, InstanceType]: 
1351  #if type(T) in [ListType, DictType, InstanceType]: 
1352  if type(T) == InstanceType: 
1353  #if False:
1354      # Keep track of unique id's to protect against infinite recursion
1355      if ids is None: ids = []
1356
1357      # Check if T has already been encountered
1358      i = id(T) 
1359 
1360      if i in ids:
1361          # T has been hashed already     
1362     
1363          # FIXME (Ole): It seems that different objects get the same id
1364          # For example id(D.items()) is the same as id(D.values()).
1365
1366          #print 'T has already been hashed:', mkargstr(T, 132), id(T), type(T)
1367          return 0
1368      else:
1369          #print 'Appending', T, id(T)
1370          ids.append(i)
1371   
1372
1373  # Start hashing 
1374 
1375 
1376  # On some architectures None, False and True gets different hash values
1377  if T is None:
1378      return(-1)
1379  if T is False:
1380      return(0)
1381  if T is True:
1382      return(1)
1383
1384  # Get hash values for hashable entries
1385  if type(T) in [TupleType, ListType]:
1386      hvals = []
1387      for t in T:
1388          h = myhash(t, ids)
1389          hvals.append(h)
1390      val = hash(tuple(hvals))
1391  elif type(T) == DictType:
1392      # Make dictionary ordering unique 
1393      I = T.items()
1394      I.sort()   
1395      val = myhash(I, ids)
1396  elif type(T) == ArrayType:
1397      val = hash(average(T.flat))  # Use mean value for efficiency
1398      #val = myhash(tuple(T), ids)
1399  elif type(T) == InstanceType:
1400      val = myhash(T.__dict__, ids)
1401     
1402      #print 'Hashed instance:'
1403      #print T.__dict__.keys()
1404      #print mkargstr(T.__dict__, 132)
1405      #print 'with value', val
1406
1407  else:
1408      try:
1409          val = hash(T)
1410      except:
1411          val = 1
1412
1413  return(val)
1414
1415# -----------------------------------------------------------------------------
1416
1417
1418def compare(A, B, ids=None):
1419    """Safe comparison of general objects
1420
1421    USAGE:
1422      compare(A,B)
1423
1424    DESCRIPTION:
1425      Return 1 if A and B they are identical, 0 otherwise
1426    """
1427
1428    if A == B:
1429        identical = True
1430    else: 
1431        # Use pickle to compare data
1432        # The native pickler must be used
1433        # since the faster cPickle does not
1434        # guarantee a unique translation
1435       
1436        import pickle as pickler # Use non-C version here
1437        identical = (pickler.dumps(A,0) == pickler.dumps(B,0))
1438
1439    return(identical)
1440
1441
1442def Xcompare(A, B, ids=None):
1443    """Safe comparison of general objects
1444
1445    USAGE:
1446      compare(A,B)
1447
1448    DESCRIPTION:
1449      Return 1 if A and B they are identical, 0 otherwise
1450    """
1451
1452    from types import TupleType, ListType, DictType, InstanceType
1453   
1454   
1455    # Keep track of unique id's to protect against infinite recursion
1456    if ids is None: ids = {}
1457
1458
1459    # Check if T has already been encountered
1460    iA = id(A) 
1461    iB = id(B)     
1462   
1463    if ids.has_key((iA, iB)):
1464        # A and B have been compared already
1465        #print 'Found', (iA, iB), A, B
1466        return ids[(iA, iB)]
1467    else:
1468        ids[(iA, iB)] = True
1469   
1470   
1471    #print 'Comparing', A, B, (iA, iB)
1472    #print ids
1473    #raw_input()
1474   
1475    if type(A) in [TupleType, ListType] and type(B) in [TupleType, ListType]:
1476        N = len(A)
1477        if len(B) != N: 
1478            identical = False
1479        else:
1480            identical = True
1481            for i in range(N):
1482                if not compare(A[i], B[i], ids): 
1483                    identical = False; break
1484               
1485    elif type(A) == DictType and type(B) == DictType:
1486        if len(A) != len(B):
1487            identical = False
1488        else:   
1489            identical = True
1490            for key in A.keys():
1491                if not B.has_key(key):
1492                    identical = False; break
1493         
1494                if not compare(A[key], B[key], ids): 
1495                    identical = False; break     
1496   
1497    elif type(A) == type(B) == types.InstanceType:   
1498        # Take care of special case where elements are instances           
1499        # Base comparison on attributes
1500               
1501        a = A.__dict__                 
1502        b = B.__dict__                             
1503               
1504        identical = compare(a, b, ids)               
1505
1506   
1507       
1508    else:       
1509        # Fall back to general code
1510        try:
1511            identical = (A == B)
1512        except:
1513            import pickle # Use non-C version here
1514            try:
1515                identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
1516            except:
1517                identical = 0
1518
1519    # Record result of comparison and return           
1520    ids[(iA, iB)] = identical
1521   
1522    return(identical)
1523
1524# -----------------------------------------------------------------------------
1525
1526def nospace(s):
1527  """Replace spaces in string s with underscores
1528
1529  USAGE:
1530    nospace(s)
1531
1532  ARGUMENTS:
1533    s -- string
1534  """
1535
1536  import string
1537
1538  newstr = ''
1539  for i in range(len(s)):
1540    if s[i] == ' ':
1541      newstr = newstr+'_'
1542    else:
1543      newstr = newstr+s[i]
1544
1545  return(newstr)
1546
1547# -----------------------------------------------------------------------------
1548
1549def get_funcname(func):
1550  """Retrieve name of function object func (depending on its type)
1551
1552  USAGE:
1553    get_funcname(func)
1554  """
1555
1556  import types, string
1557
1558  if type(func) == types.FunctionType:
1559    funcname = func.func_name
1560  elif type(func) == types.BuiltinFunctionType:
1561    funcname = func.__name__
1562  else:
1563    tab = string.maketrans("<>'","   ")
1564    tmp = string.translate(`func`,tab)
1565    tmp = string.split(tmp)
1566    funcname = string.join(tmp)
1567
1568    # Truncate memory address as in
1569    # class __main__.Dummy at 0x00A915D0
1570    index = funcname.find('at 0x')
1571    if index >= 0:
1572      funcname = funcname[:index+5] # Keep info that there is an address
1573
1574  funcname = nospace(funcname)
1575  return(funcname)
1576
1577# -----------------------------------------------------------------------------
1578
1579def get_bytecode(func):
1580  """ Get bytecode from function object.
1581
1582  USAGE:
1583    get_bytecode(func)
1584  """
1585
1586  import types
1587
1588  if type(func) == types.FunctionType:
1589    bytecode = func.func_code.co_code
1590    consts = func.func_code.co_consts
1591    argcount = func.func_code.co_argcount   
1592    defaults = func.func_defaults     
1593  elif type(func) == types.MethodType:
1594    bytecode = func.im_func.func_code.co_code
1595    consts =  func.im_func.func_code.co_consts
1596    argcount =  func.im_func.func_code.co_argcount   
1597    defaults = func.im_func.func_defaults         
1598  else:
1599    #raise Exception  #Test only
1600    bytecode = None   #Built-in functions are assumed not to change
1601    consts = 0
1602    argcount = 0
1603    defaults = 0
1604
1605  return (bytecode, consts, argcount, defaults)
1606
1607# -----------------------------------------------------------------------------
1608
1609def get_depstats(dependencies):
1610  """ Build dictionary of dependency files and their size, mod. time and ctime.
1611
1612  USAGE:
1613    get_depstats(dependencies):
1614  """
1615
1616  import types
1617
1618  #print 'Caching DEBUG: Dependencies are', dependencies
1619  d = {}
1620  if dependencies:
1621
1622    #Expand any wildcards
1623    import glob
1624    expanded_dependencies = []
1625    for FN in dependencies:
1626      expanded_FN = glob.glob(FN)
1627
1628      if expanded_FN == []:
1629        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1630        raise Exception, errmsg     
1631
1632      expanded_dependencies += expanded_FN
1633
1634   
1635    for FN in expanded_dependencies:
1636      if not type(FN) == types.StringType:
1637        errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1638        errmsg += '                   Dependency given: %s' %FN
1639        raise Exception, errmsg     
1640      if not os.access(FN,os.F_OK):
1641        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1642        raise Exception, errmsg
1643      (size,atime,mtime,ctime) = filestat(FN)
1644
1645      # We don't use atime because that would cause recomputation every time.
1646      # We don't use ctime because that is irrelevant and confusing for users.
1647      d.update({FN : (size,mtime)})
1648
1649  return(d)
1650
1651# -----------------------------------------------------------------------------
1652
1653def filestat(FN):
1654  """A safe wrapper using os.stat to get basic file statistics
1655     The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1656
1657  USAGE:
1658    filestat(FN)
1659
1660  DESCRIPTION:
1661     Must compile Python with
1662     CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1663              configure
1664     as given in section 8.1.1 Large File Support in the Libray Reference
1665  """
1666
1667  import os, time
1668
1669  try:
1670    stats = os.stat(FN)
1671    size  = stats[6]
1672    atime = stats[7]
1673    mtime = stats[8]
1674    ctime = stats[9]
1675  except:
1676
1677    # Hack to get the results anyway (works only on Unix at the moment)
1678    #
1679    print 'Hack to get os.stat when files are too large'
1680
1681    if unix:
1682      tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1683      # Unique filename, FIXME: Use random number
1684
1685      # Get size and access time (atime)
1686      #
1687      exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1688      (size,atime) = get_lsline(tmp)
1689
1690      # Get size and modification time (mtime)
1691      #
1692      exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1693      (size,mtime) = get_lsline(tmp)
1694
1695      # Get size and ctime
1696      #
1697      exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1698      (size,ctime) = get_lsline(tmp)
1699
1700      try:
1701        exitcode=os.system('rm '+tmp)
1702        # FIXME: Gives error if file doesn't exist
1703      except:
1704        pass
1705    else:
1706      pass
1707      raise Exception  # FIXME: Windows case
1708
1709  return(long(size),atime,mtime,ctime)
1710
1711# -----------------------------------------------------------------------------
1712
1713def get_lsline(FN):
1714  """get size and time for filename
1715
1716  USAGE:
1717    get_lsline(file_name)
1718
1719  DESCRIPTION:
1720    Read in one line 'ls -la' item from file (generated by filestat) and
1721    convert time to seconds since epoch. Return file size and time.
1722  """
1723
1724  import string, time
1725
1726  f = open(FN,'r')
1727  info = f.read()
1728  info = string.split(info)
1729
1730  size = info[4]
1731  week = info[5]
1732  mon  = info[6]
1733  day  = info[7]
1734  hour = info[8]
1735  year = info[9]
1736
1737  str = week+' '+mon+' '+day+' '+hour+' '+year
1738  timetup = time.strptime(str)
1739  t = time.mktime(timetup)
1740  return(size, t)
1741
1742# -----------------------------------------------------------------------------
1743
1744def checkdir(CD,verbose=None):
1745  """Check or create caching directory
1746
1747  USAGE:
1748    checkdir(CD,verbose):
1749
1750  ARGUMENTS:
1751    CD -- Directory
1752    verbose -- Flag verbose output (default: None)
1753
1754  DESCRIPTION:
1755    If CD does not exist it will be created if possible
1756  """
1757
1758  import os
1759  import os.path
1760
1761  if CD[-1] != os.sep: 
1762    CD = CD + os.sep  # Add separator for directories
1763
1764  CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1765  if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1766    try:
1767      exitcode=os.mkdir(CD)
1768
1769      # Change access rights if possible
1770      #
1771      if unix:
1772        exitcode=os.system('chmod 777 '+CD)
1773      else:
1774        pass  # FIXME: What about acces rights under Windows?
1775      if verbose: print 'MESSAGE: Directory', CD, 'created.'
1776    except:
1777      print 'WARNING: Directory', CD, 'could not be created.'
1778      if unix:
1779        CD = '/tmp/'
1780      else:
1781        CD = 'C:' 
1782      print 'Using directory %s instead' %CD
1783
1784  return(CD)
1785
1786#==============================================================================
1787# Statistics
1788#==============================================================================
1789
1790def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
1791                 compression):
1792  """Add stats entry
1793
1794  USAGE:
1795    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1796
1797  DESCRIPTION:
1798    Make one entry in the stats file about one cache hit recording time saved
1799    and other statistics. The data are used by the function cachestat.
1800  """
1801
1802  import os, time
1803
1804  try:
1805    TimeTuple = time.localtime(time.time())
1806    extension = time.strftime('%b%Y',TimeTuple)
1807    SFN = CD+statsfile+'.'+extension
1808    #statfile = open(SFN,'a')
1809    (statfile, dummy) = myopen(SFN,'a',compression=0)
1810
1811    # Change access rights if possible
1812    #
1813    #if unix:
1814    #  try:
1815    #    exitcode=os.system('chmod 666 '+SFN)
1816    #  except:
1817    #    pass
1818  except:
1819    print 'Warning: Stat file could not be opened'
1820
1821  try:
1822    if os.environ.has_key('USER'):
1823      user = os.environ['USER']
1824    else:
1825      user = 'Nobody'
1826
1827    date = time.asctime(TimeTuple)
1828
1829    if Retrieved:
1830      hit = '1'
1831    else:
1832      hit = '0'
1833
1834    # Get size of result file
1835    #   
1836    if compression:
1837      stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1838    else:
1839      stats = os.stat(CD+FN+'_'+file_types[0])
1840 
1841    if stats: 
1842      size = stats[6]
1843    else:
1844      size = -1  # Error condition, but don't crash. This is just statistics 
1845
1846    # Build entry
1847    # 
1848    entry = date             + ',' +\
1849            user             + ',' +\
1850            FN               + ',' +\
1851            str(int(size))   + ',' +\
1852            str(compression) + ',' +\
1853            hit              + ',' +\
1854            str(reason)      + ',' +\
1855            str(round(comptime,4)) + ',' +\
1856            str(round(loadtime,4)) +\
1857            CR
1858           
1859    statfile.write(entry)
1860    statfile.close()
1861  except:
1862    print 'Warning: Writing of stat file failed'
1863
1864# -----------------------------------------------------------------------------
1865
1866# FIXME: should take cachedir as an optional arg
1867#
1868def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1869  """  List caching statistics.
1870
1871  USAGE:
1872    __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1873
1874      Generate statistics of caching efficiency.
1875      The parameter sortidx determines by what field lists are sorted.
1876      If the optional keyword period is set to -1,
1877      all available caching history is used.
1878      If it is 0 only the current month is used.
1879      Future versions will include more than one month....
1880      OMN 20/8/2000
1881  """
1882
1883  import os
1884  import os.path
1885  from string import split, rstrip, find
1886  from time import strptime, localtime, strftime, mktime, ctime
1887
1888  # sortidx = 4    # Index into Fields[1:]. What to sort by.
1889
1890  Fields = ['Name', 'Hits', 'Exec(s)', \
1891            'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1892  Widths = [25,7,9,9,9,9,13]
1893  #Types = ['s','d','d','d','d','.2f','d']
1894  Types = ['s','d','.2f','.2f','.2f','.2f','d'] 
1895
1896  Dictnames = ['Function', 'User']
1897
1898  if not cachedir:
1899    cachedir = checkdir(options['cachedir'])
1900
1901  SD = os.path.expanduser(cachedir)  # Expand ~ or ~user in pathname
1902
1903  if period == -1:  # Take all available stats
1904    SFILENAME = statsfile
1905  else:  # Only stats from current month 
1906       # MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1907    TimeTuple = localtime(time())
1908    extension = strftime('%b%Y',TimeTuple)
1909    SFILENAME = statsfile+'.'+extension
1910
1911  DIRLIST = os.listdir(SD)
1912  SF = []
1913  for FN in DIRLIST:
1914    if find(FN,SFILENAME) >= 0:
1915      SF.append(FN)
1916
1917  blocksize = 15000000
1918  total_read = 0
1919  total_hits = 0
1920  total_discarded = 0
1921  firstday = mktime(strptime('2030','%Y'))
1922             # FIXME: strptime don't exist in WINDOWS ?
1923  lastday = 0
1924
1925  FuncDict = {}
1926  UserDict = {}
1927  for FN in SF:
1928    input = open(SD+FN,'r')
1929    print 'Reading file ', SD+FN
1930
1931    while 1:
1932      A = input.readlines(blocksize)
1933      if len(A) == 0: break
1934      total_read = total_read + len(A)
1935      for record in A:
1936        record = tuple(split(rstrip(record),','))
1937        #print record, len(record)
1938
1939        if len(record) == 9:
1940          timestamp = record[0]
1941       
1942          try:
1943            t = mktime(strptime(timestamp))
1944          except:
1945            total_discarded = total_discarded + 1         
1946            continue   
1947             
1948          if t > lastday:
1949            lastday = t
1950          if t < firstday:
1951            firstday = t
1952
1953          user     = record[1]
1954          func     = record[2]
1955
1956          # Strip hash-stamp off
1957          #
1958          i = find(func,'[')
1959          func = func[:i]
1960
1961          size        = float(record[3])
1962
1963          # Compression kepword can be Boolean
1964          if record[4] in ['True', '1']:
1965            compression = 1
1966          elif record[4] in ['False', '0']: 
1967            compression = 0
1968          else:
1969            print 'Unknown value of compression', record[4]
1970            print record
1971            total_discarded = total_discarded + 1           
1972            continue
1973
1974          #compression = int(record[4]) # Can be Boolean
1975          hit         = int(record[5])
1976          reason      = int(record[6])   # Not used here   
1977          cputime     = float(record[7])
1978          loadtime    = float(record[8])
1979
1980          if hit:
1981            total_hits = total_hits + 1
1982            saving = cputime-loadtime
1983
1984            if cputime != 0:
1985              rel_saving = round(100.0*saving/cputime,2)
1986            else:
1987              #rel_saving = round(1.0*saving,2)
1988              rel_saving = 100.0 - round(1.0*saving,2)  # A bit of a hack
1989
1990            info = [1,cputime,loadtime,saving,rel_saving,size]
1991
1992            UpdateDict(UserDict,user,info)
1993            UpdateDict(FuncDict,func,info)
1994          else:
1995            pass #Stats on recomputations and their reasons could go in here
1996             
1997        else:
1998          #print 'Record discarded'
1999          #print record
2000          total_discarded = total_discarded + 1
2001
2002    input.close()
2003
2004  # Compute averages of all sums and write list
2005  #
2006
2007  if total_read == 0:
2008    printline(Widths,'=')
2009    print 'CACHING STATISTICS: No valid records read'
2010    printline(Widths,'=')
2011    return
2012
2013  print
2014  printline(Widths,'=')
2015  print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
2016  printline(Widths,'=')
2017  #print '  Period:', ctime(firstday), 'to', ctime(lastday)
2018  print '  Total number of valid records', total_read
2019  print '  Total number of discarded records', total_discarded
2020  print '  Total number of hits', total_hits
2021  print
2022
2023  print '  Fields', Fields[2:], 'are averaged over number of hits'
2024  print '  Time is measured in seconds and size in bytes'
2025  print '  Tables are sorted by', Fields[1:][sortidx]
2026
2027  # printline(Widths,'-')
2028
2029  if showuser:
2030    Dictionaries = [FuncDict, UserDict]
2031  else:
2032    Dictionaries = [FuncDict]
2033
2034  i = 0
2035  for Dict in Dictionaries:
2036    for key in Dict.keys():
2037      rec = Dict[key]
2038      for n in range(len(rec)):
2039        if n > 0:
2040          rec[n] = round(1.0*rec[n]/rec[0],2)
2041      Dict[key] = rec
2042
2043    # Sort and output
2044    #
2045    keylist = SortDict(Dict,sortidx)
2046
2047    # Write Header
2048    #
2049    print
2050    #print Dictnames[i], 'statistics:'; i=i+1
2051    printline(Widths,'-')
2052    n = 0
2053    for s in Fields:
2054      if s == Fields[0]:  # Left justify
2055        s = Dictnames[i] + ' ' + s; i=i+1
2056        exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
2057      else:
2058        exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
2059    print
2060    printline(Widths,'-')
2061
2062    # Output Values
2063    #
2064    for key in keylist:
2065      rec = Dict[key]
2066      n = 0
2067      if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
2068      exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
2069      for val in rec:
2070        exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
2071      print
2072    print
2073
2074#==============================================================================
2075# Auxiliary stats functions
2076#==============================================================================
2077
2078def UpdateDict(Dict,key,info):
2079  """Update dictionary by adding new values to existing.
2080
2081  USAGE:
2082    UpdateDict(Dict,key,info)
2083  """
2084
2085  if Dict.has_key(key):
2086    dinfo = Dict[key]
2087    for n in range(len(dinfo)):
2088      dinfo[n] = info[n] + dinfo[n]
2089  else:
2090    dinfo = info[:]  # Make a copy of info list
2091
2092  Dict[key] = dinfo
2093  return Dict
2094
2095# -----------------------------------------------------------------------------
2096
2097def SortDict(Dict,sortidx=0):
2098  """Sort dictionary
2099
2100  USAGE:
2101    SortDict(Dict,sortidx):
2102
2103  DESCRIPTION:
2104    Sort dictionary of lists according field number 'sortidx'
2105  """
2106
2107  import types
2108
2109  sortlist  = []
2110  keylist = Dict.keys()
2111  for key in keylist:
2112    rec = Dict[key]
2113    if not type(rec) in [types.ListType, types.TupleType]:
2114      rec = [rec]
2115
2116    if sortidx > len(rec)-1:
2117      if options['verbose']:
2118        print 'ERROR: Sorting index to large, sortidx = ', sortidx
2119      raise IndexError
2120
2121    val = rec[sortidx]
2122    sortlist.append(val)
2123
2124  A = map(None,sortlist,keylist)
2125  A.sort()
2126  keylist = map(lambda x: x[1], A)  # keylist sorted by sortidx
2127
2128  return(keylist)
2129
2130# -----------------------------------------------------------------------------
2131
2132def printline(Widths,char):
2133  """Print textline in fixed field.
2134
2135  USAGE:
2136    printline(Widths,char)
2137  """
2138
2139  s = ''
2140  for n in range(len(Widths)):
2141    s = s+Widths[n]*char
2142    if n > 0:
2143      s = s+char
2144
2145  print s
2146
2147#==============================================================================
2148# Messages
2149#==============================================================================
2150
2151def msg1(funcname,args,kwargs,reason):
2152  """Message 1
2153
2154  USAGE:
2155    msg1(funcname,args,kwargs,reason):
2156  """
2157
2158  import string
2159  #print 'MESSAGE (caching.py): Evaluating function', funcname,
2160
2161  print_header_box('Evaluating function %s' %funcname)
2162 
2163  msg7(args,kwargs)
2164  msg8(reason) 
2165 
2166  print_footer()
2167 
2168  #
2169  # Old message
2170  #
2171  #args_present = 0
2172  #if args:
2173  #  if len(args) == 1:
2174  #    print 'with argument', mkargstr(args[0], textwidth2),
2175  #  else:
2176  #    print 'with arguments', mkargstr(args, textwidth2),
2177  #  args_present = 1     
2178  #   
2179  #if kwargs:
2180  #  if args_present:
2181  #    word = 'and'
2182  #  else:
2183  #    word = 'with'
2184  #     
2185  #  if len(kwargs) == 1:
2186  #    print word + ' keyword argument', mkargstr(kwargs, textwidth2)
2187  #  else:
2188  #    print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
2189  #  args_present = 1           
2190  #else:
2191  #  print    # Newline when no keyword args present
2192  #       
2193  #if not args_present:   
2194  #  print '',  # Default if no args or kwargs present
2195   
2196   
2197
2198# -----------------------------------------------------------------------------
2199
2200def msg2(funcname,args,kwargs,comptime,reason):
2201  """Message 2
2202
2203  USAGE:
2204    msg2(funcname,args,kwargs,comptime,reason)
2205  """
2206
2207  import string
2208
2209  #try:
2210  #  R = Reason_msg[reason]
2211  #except:
2212  #  R = 'Unknown reason' 
2213 
2214  #print_header_box('Caching statistics (storing) - %s' %R)
2215  print_header_box('Caching statistics (storing)') 
2216 
2217  msg6(funcname,args,kwargs)
2218  msg8(reason)
2219
2220  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2221
2222# -----------------------------------------------------------------------------
2223
2224def msg3(savetime, CD, FN, deps,compression):
2225  """Message 3
2226
2227  USAGE:
2228    msg3(savetime, CD, FN, deps,compression)
2229  """
2230
2231  import string
2232  print string.ljust('| Loading time:', textwidth1) + str(round(savetime,2)) + \
2233                     ' seconds (estimated)'
2234  msg5(CD,FN,deps,compression)
2235
2236# -----------------------------------------------------------------------------
2237
2238def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
2239  """Message 4
2240
2241  USAGE:
2242    msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
2243  """
2244
2245  import string
2246
2247  print_header_box('Caching statistics (retrieving)')
2248 
2249  msg6(funcname,args,kwargs)
2250  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2251  print string.ljust('| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
2252  print string.ljust('| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
2253        ' seconds'
2254  msg5(CD,FN,deps,compression)
2255
2256# -----------------------------------------------------------------------------
2257
2258def msg5(CD,FN,deps,compression):
2259  """Message 5
2260
2261  USAGE:
2262    msg5(CD,FN,deps,compression)
2263
2264  DESCRIPTION:
2265   Print dependency stats. Used by msg3 and msg4
2266  """
2267
2268  import os, time, string
2269
2270  print '|'
2271  print string.ljust('| Caching dir: ', textwidth1) + CD
2272
2273  if compression:
2274    suffix = '.z'
2275    bytetext = 'bytes, compressed'
2276  else:
2277    suffix = ''
2278    bytetext = 'bytes'
2279
2280  for file_type in file_types:
2281    file_name = FN + '_' + file_type + suffix
2282    print string.ljust('| ' + file_type + ' file: ', textwidth1) + file_name,
2283    stats = os.stat(CD+file_name)
2284    print '('+ str(stats[6]) + ' ' + bytetext + ')'
2285
2286  print '|'
2287  if len(deps) > 0:
2288    print '| Dependencies:  '
2289    dependencies  = deps.keys()
2290    dlist = []; maxd = 0
2291    tlist = []; maxt = 0
2292    slist = []; maxs = 0
2293    for d in dependencies:
2294      stats = deps[d]
2295      t = time.ctime(stats[1])
2296      s = str(stats[0])
2297      #if s[-1] == 'L':
2298      #  s = s[:-1]  # Strip rightmost 'long integer' L off.
2299      #              # FIXME: Unnecessary in versions later than 1.5.2
2300
2301      if len(d) > maxd: maxd = len(d)
2302      if len(t) > maxt: maxt = len(t)
2303      if len(s) > maxs: maxs = len(s)
2304      dlist.append(d)
2305      tlist.append(t)
2306      slist.append(s)
2307
2308    for n in range(len(dlist)):
2309      d = string.ljust(dlist[n]+':', maxd+1)
2310      t = string.ljust(tlist[n], maxt)
2311      s = string.rjust(slist[n], maxs)
2312
2313      print '| ', d, t, ' ', s, 'bytes'
2314  else:
2315    print '| No dependencies'
2316  print_footer()
2317
2318# -----------------------------------------------------------------------------
2319
2320def msg6(funcname,args,kwargs):
2321  """Message 6
2322
2323  USAGE:
2324    msg6(funcname,args,kwargs)
2325  """
2326
2327  import string
2328  print string.ljust('| Function:', textwidth1) + funcname
2329
2330  msg7(args,kwargs)
2331 
2332# -----------------------------------------------------------------------------   
2333
2334def msg7(args,kwargs):
2335  """Message 7
2336 
2337  USAGE:
2338    msg7(args,kwargs):
2339  """
2340 
2341  import string
2342 
2343  args_present = 0 
2344  if args:
2345    if len(args) == 1:
2346      print string.ljust('| Argument:', textwidth1) + mkargstr(args[0], \
2347                         textwidth2)
2348    else:
2349      print string.ljust('| Arguments:', textwidth1) + \
2350            mkargstr(args, textwidth2)
2351    args_present = 1
2352           
2353  if kwargs:
2354    if len(kwargs) == 1:
2355      print string.ljust('| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
2356                         textwidth2)
2357    else:
2358      print string.ljust('| Keyword Args:', textwidth1) + \
2359            mkargstr(kwargs, textwidth2)
2360    args_present = 1
2361
2362  if not args_present:               
2363    print '| No arguments' # Default if no args or kwargs present
2364
2365# -----------------------------------------------------------------------------
2366
2367def msg8(reason):
2368  """Message 8
2369 
2370  USAGE:
2371    msg8(reason):
2372  """
2373 
2374  import string
2375   
2376  try:
2377    R = Reason_msg[reason]
2378  except:
2379    R = 'Unknown' 
2380 
2381  print string.ljust('| Reason:', textwidth1) + R
2382   
2383# -----------------------------------------------------------------------------
2384
2385def print_header_box(line):
2386  """Print line in a nice box.
2387 
2388  USAGE:
2389    print_header_box(line)
2390
2391  """
2392  global textwidth3
2393
2394  import time
2395
2396  time_stamp = time.ctime(time.time())
2397  line = time_stamp + '. ' + line
2398   
2399  N = len(line) + 1
2400  s = '+' + '-'*N + CR
2401
2402  print s + '| ' + line + CR + s,
2403
2404  textwidth3 = N
2405
2406# -----------------------------------------------------------------------------
2407   
2408def print_footer():
2409  """Print line same width as that of print_header_box.
2410  """
2411 
2412  N = textwidth3
2413  s = '+' + '-'*N + CR   
2414     
2415  print s     
2416     
2417# -----------------------------------------------------------------------------
2418
2419def mkargstr(args, textwidth, argstr = ''):
2420  """ Generate a string containing first textwidth characters of arguments.
2421
2422  USAGE:
2423    mkargstr(args, textwidth, argstr = '')
2424
2425  DESCRIPTION:
2426    Exactly the same as str(args) possibly followed by truncation,
2427    but faster if args is huge.
2428  """
2429
2430  import types
2431
2432  WasTruncated = 0
2433
2434  if not type(args) in [types.TupleType, types.ListType, types.DictType]:
2435    if type(args) == types.StringType:
2436      argstr = argstr + "'"+str(args)+"'"
2437    else:
2438      # Truncate large Numeric arrays before using str()
2439      import Numeric
2440      if type(args) == Numeric.ArrayType:
2441#        if len(args.flat) > textwidth: 
2442#        Changed by Duncan and Nick 21/2/07 .flat has problems with
2443#        non-contigous arrays and ravel is equal to .flat except it
2444#        can work with non-contiguous  arrays
2445        if len(Numeric.ravel(args)) > textwidth:
2446          args = 'Array: ' + str(args.shape)
2447
2448      argstr = argstr + str(args)
2449  else:
2450    if type(args) == types.DictType:
2451      argstr = argstr + "{"
2452      for key in args.keys():
2453        argstr = argstr + mkargstr(key, textwidth) + ": " + \
2454                 mkargstr(args[key], textwidth) + ", "
2455        if len(argstr) > textwidth:
2456          WasTruncated = 1
2457          break
2458      argstr = argstr[:-2]  # Strip off trailing comma     
2459      argstr = argstr + "}"
2460
2461    else:
2462      if type(args) == types.TupleType:
2463        lc = '('
2464        rc = ')'
2465      else:
2466        lc = '['
2467        rc = ']'
2468      argstr = argstr + lc
2469      for arg in args:
2470        argstr = argstr + mkargstr(arg, textwidth) + ', '
2471        if len(argstr) > textwidth:
2472          WasTruncated = 1
2473          break
2474
2475      # Strip off trailing comma and space unless singleton tuple
2476      #
2477      if type(args) == types.TupleType and len(args) == 1:
2478        argstr = argstr[:-1]   
2479      else:
2480        argstr = argstr[:-2]
2481      argstr = argstr + rc
2482
2483  if len(argstr) > textwidth:
2484    WasTruncated = 1
2485
2486  if WasTruncated:
2487    argstr = argstr[:textwidth]+'...'
2488  return(argstr)
2489
2490# -----------------------------------------------------------------------------
2491
2492def test_OK(msg):
2493  """Print OK msg if test is OK.
2494 
2495  USAGE
2496    test_OK(message)
2497  """
2498
2499  import string
2500   
2501  print string.ljust(msg, textwidth4) + ' - OK' 
2502 
2503  #raise StandardError
2504 
2505# -----------------------------------------------------------------------------
2506
2507def test_error(msg):
2508  """Print error if test fails.
2509 
2510  USAGE
2511    test_error(message)
2512  """
2513 
2514  print 'ERROR (caching.test): %s' %msg
2515  print 'Please send this code example and output to '
2516  print 'Ole.Nielsen@anu.edu.au'
2517  print
2518  print
2519 
2520  #import sys
2521  #sys.exit()
2522  raise StandardError
2523
2524#-------------------------------------------------------------
2525if __name__ == "__main__":
2526  pass
Note: See TracBrowser for help on using the repository browser.