source: trunk/anuga_core/source/anuga/caching/caching.py @ 8492

Last change on this file since 8492 was 8125, checked in by wilsonr, 14 years ago

Changes to address ticket 360.
File size: 70.1 KB
Line 
1# =============================================================================
2# caching.py - Supervised caching of function results.
3# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4# Australian National University (1999-2003)
5# Geoscience Australia (2003-present)
6#
7#    This program is free software; you can redistribute it and/or modify
8#    it under the terms of the GNU General Public License as published by
9#    the Free Software Foundation; either version 2 of the License, or
10#    (at your option) any later version.
11#
12#    This program is distributed in the hope that it will be useful,
13#    but WITHOUT ANY WARRANTY; without even the implied warranty of
14#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15#    GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16#    for more details.
17#
18#    You should have received a copy of the GNU General Public License
19#    along with this program; if not, write to the Free Software
20#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
21#
22#
23# Contact address: Ole.Nielsen@ga.gov.au
24#
25# Version 1.5.6 February 2002
26# =============================================================================
27 
28"""Module caching.py - Supervised caching of function results.
29
30Public functions:
31
32cache(my_F,args) -- Cache values returned from callable object my_F given args.
33cachestat() --      Reports statistics about cache hits and time saved.
34test() --       Conducts a basic test of the caching functionality.
35
36See doc strings of individual functions for detailed documentation.
37"""
38
39# -----------------------------------------------------------------------------
40# Initialisation code
41
42# Determine platform
43#
44from os import getenv
45import types
46
47import os
48if os.name in ['nt', 'dos', 'win32', 'what else?']:
49  unix = False
50else:
51  unix = True
52
53import anuga.utilities.log as log
54
55import numpy as num
56
57#from future
58
59cache_dir = '.python_cache'
60
61# Make default caching directory name
62# We are changing the 'data directory' environment variable from
63# INUNDATIONHOME to ANUGADATA - this gives a changeover.
64if unix:
65    homedir = getenv('ANUGADATA')
66    if not homedir:
67        homedir = getenv('INUNDATIONHOME')
68
69    if not homedir:
70        homedir = '~'
71    else:
72        # Since homedir will be a group area, individually label the caches
73        user = getenv('LOGNAME')
74        if not user:
75            cache_dir += '_' + user
76   
77    CR = '\n'
78else:
79    homedir = 'c:'
80    CR = '\r\n'  #FIXME: Not tested under windows
81 
82cachedir = os.path.join(homedir, cache_dir)
83
84# -----------------------------------------------------------------------------
85# Options directory with default values - to be set by user
86#
87
88options = { 
89  'cachedir': cachedir,  # Default cache directory
90  'maxfiles': 1000000,   # Maximum number of cached files
91  'savestat': True,      # Log caching info to stats file
92  'verbose': True,       # Write messages to standard output
93  'bin': True,           # Use binary format (more efficient)
94  'compression': True,   # Use zlib compression
95  'bytecode': True,      # Recompute if bytecode has changed
96  'expire': False        # Automatically remove files that have been accessed
97                         # least recently
98}
99
100# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
101
102def set_option(key, value):
103  """Function to set values in the options directory.
104
105  USAGE:
106    set_option(key, value)
107
108  ARGUMENTS:
109    key --   Key in options dictionary. (Required)
110    value -- New value for key. (Required)
111
112  DESCRIPTION:
113    Function to set values in the options directory.
114    Raises an exception if key is not in options.
115  """
116
117  if key in options:
118    options[key] = value
119  else:
120    raise KeyError(key)  # Key not found, raise an exception
121
122# -----------------------------------------------------------------------------
123# Function cache - the main routine
124
125def cache(my_F, 
126          args=(), 
127          kwargs={}, 
128          dependencies=None, 
129          cachedir=None,
130          verbose=None, 
131          compression=None, 
132          evaluate=False, 
133          test=False, 
134          clear=False,
135          return_filename=False):
136  """Supervised caching of function results. Also known as memoization.
137
138  USAGE:
139    result = cache(my_F, args, kwargs, dependencies, cachedir, verbose,
140                   compression, evaluate, test, return_filename)
141
142  ARGUMENTS:
143    my_F --            Callable object (Required)
144    args --            Arguments to my_F (Default: ())
145    kwargs --          Keyword arguments to my_F (Default: {})   
146    dependencies --    Filenames that my_F depends on (Default: None)
147    cachedir --        Directory for cache files (Default: options['cachedir'])
148    verbose --         Flag verbose output to stdout
149                       (Default: options['verbose'])
150    compression --     Flag zlib compression (Default: options['compression'])
151    evaluate --        Flag forced evaluation of my_F (Default: False)
152    test --            Flag test for cached results (Default: False)
153    clear --           Flag delete cached results (Default: False)   
154    return_filename -- Flag return of cache filename (Default: False)   
155
156  DESCRIPTION:
157    A Python function call of the form
158
159      result = my_F(arg1,...,argn)
160
161    can be replaced by
162
163      from caching import cache
164      result = cache(my_F,(arg1,...,argn))
165
166  The latter form returns the same output as the former but reuses cached
167  results if the function has been computed previously in the same context.
168  'result' and the arguments can be simple types, tuples, list, dictionaries or
169  objects, but not unhashable types such as functions or open file objects.
170  The function 'my_F' may be a member function of an object or a module.
171
172  This type of caching is particularly useful for computationally intensive
173  functions with few frequently used combinations of input arguments. Note that
174  if the inputs or output are very large caching might not save time because
175  disc access may dominate the execution time.
176
177  If the function definition changes after a result has been cached it will be
178  detected by examining the functions bytecode (co_code, co_consts,
179  func_defaults, co_argcount) and it will be recomputed.
180
181  LIMITATIONS:
182    1 Caching uses function(*args, **kwargs) to evaluate and will work
183      with anything that can be pickled, so any limitation in function(,)
184      or pickle extends to caching.
185    2 A function to be cached should not depend on global variables
186      as wrong results may occur if globals are changed after a result has
187      been cached.
188
189  -----------------------------------------------------------------------------
190  Additional functionality:
191
192  Keyword args
193    Keyword arguments (kwargs) can be added as a dictionary of keyword: value
194    pairs, following Python's 'extended call syntax'.
195   
196    A Python function call of the form
197   
198      result = my_F(arg1,...,argn, kwarg1=val1,...,kwargm=valm)   
199
200    is then cached as follows
201
202      from caching import cache
203      result = cache(my_F,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
204   
205    The default value of kwargs is {} 
206
207  Explicit dependencies:
208    The call
209      cache(my_F,(arg1,...,argn), dependencies = <list of filenames>)
210    Checks the size, creation time and modification time of each listed file.
211    If any file has changed the function is recomputed and the results stored
212    again.
213
214  Specify caching directory:
215    The call
216      cache(my_F,(arg1,...,argn), cachedir = <cachedir>)
217    designates <cachedir> where cached data are stored. Use ~ to indicate users
218    home directory - not $HOME. The default is ~/.python_cache on a UNIX
219    platform and c:/.python_cache on a Win platform.
220
221  Silent operation:
222    The call
223      cache(my_F,(arg1,...,argn), verbose=False)
224    suppresses messages to standard output.
225
226  Compression:
227    The call
228      cache(my_F,(arg1,...,argn), compression=False)
229    disables compression. (Default: compression=True). If the requested compressed
230    or uncompressed file is not there, it'll try the other version.
231
232  Forced evaluation:
233    The call
234      cache(my_F,(arg1,...,argn), evaluate=True)
235    forces the function to evaluate even though cached data may exist.
236
237  Testing for presence of cached result:
238    The call
239      cache(my_F,(arg1,...,argn), test=True)
240    retrieves cached result if it exists, otherwise None. The function will not
241    be evaluated. If both evaluate and test are switched on, evaluate takes
242    precedence.
243    ??NOTE: In case of hash collisions, this may return the wrong result as
244    ??it only checks if *a* cached result is present.
245    # I think this was due to the bytecode option being False for some reason. (23/1/2009).
246   
247  Obtain cache filenames:
248    The call   
249      cache(my_F,(arg1,...,argn), return_filename=True)
250    returns the hashed base filename under which this function and its
251    arguments would be cached
252
253  Clearing cached results:
254    The call
255      cache(my_F,'clear')
256    clears all cached data for 'my_F' and
257      cache('clear')
258    clears all cached data.
259 
260    NOTE: The string 'clear' can be passed an *argument* to my_F using
261      cache(my_F,('clear',)) or cache(my_F,tuple(['clear'])).
262
263    New form of clear:
264      cache(my_F,(arg1,...,argn), clear=True)
265    clears cached data for particular combination my_F and args
266     
267  """
268
269  # Imports and input checks
270  #
271  import time, string
272
273  if not cachedir:
274    cachedir = options['cachedir']
275
276  if verbose == None:  # Do NOT write 'if not verbose:', it could be zero.
277    verbose = options['verbose']
278
279  if compression == None:  # Do NOT write 'if not compression:',
280                           # it could be zero.
281    compression = options['compression']
282
283  # Create cache directory if needed
284  CD = checkdir(cachedir,verbose)
285
286  # Handle the case cache('clear')
287  if isinstance(my_F, basestring):
288    if string.lower(my_F) == 'clear':
289      clear_cache(CD,verbose=verbose)
290      return
291
292  # Handle the case cache(my_F, 'clear')
293  if isinstance(args, basestring):
294    if string.lower(args) == 'clear':
295      clear_cache(CD,my_F,verbose=verbose)
296      return
297
298  # Force singleton arg into a tuple
299  if not isinstance(args, tuple):
300    args = tuple([args])
301 
302  # Check that kwargs is a dictionary
303  if not isinstance(kwargs, dict):
304    raise TypeError   
305   
306  # Hash arguments (and keyword args) to integer
307  arghash = myhash((args, kwargs))
308 
309  # Get sizes and timestamps for files listed in dependencies.
310  # Force singletons into a tuple.
311  if dependencies and not isinstance(dependencies, (tuple, list)):
312    dependencies = tuple([dependencies])
313  deps = get_depstats(dependencies)
314
315  # Extract function name from my_F object
316  funcname = get_funcname(my_F)
317
318  # Create cache filename
319  FN = funcname+'['+`arghash`+']'  # The symbol '(' does not work under unix
320
321  if return_filename:
322    return(FN)
323
324  if clear:
325    for file_type in file_types:
326      file_name = CD+FN+'_'+file_type
327      for fn in [file_name, file_name + '.z']:
328        if os.access(fn, os.F_OK):             
329          if unix:
330            os.remove(fn)
331          else:
332            # FIXME: os.remove doesn't work under windows       
333            os.system('del '+fn)
334          if verbose is True:
335            log.critical('MESSAGE (caching): File %s deleted' % fn)
336        ##else:
337        ##  log.critical('%s was not accessed' % fn)
338    return None
339
340
341  #-------------------------------------------------------------------       
342 
343  # Check if previous computation has been cached
344  if evaluate is True:
345    Retrieved = None  # Force evaluation of my_F regardless of caching status.
346    reason = 5
347  else:
348    T, FN, Retrieved, reason, comptime, loadtime, compressed = \
349        CacheLookup(CD, FN, my_F, 
350                    args, kwargs, 
351                    deps, 
352                    verbose, 
353                    compression,
354                    dependencies)
355
356  if not Retrieved:
357    if test:  # Do not attempt to evaluate function
358      T = None
359    else:  # Evaluate function and save to cache
360      if verbose is True:
361       
362        msg1(funcname, args, kwargs,reason)
363
364      # Remove expired files automatically
365      if options['expire']:
366        DeleteOldFiles(CD,verbose)
367       
368      # Save args before function is evaluated in case
369      # they are modified by function
370      save_args_to_cache(CD,FN,args,kwargs,compression)
371
372      # Execute and time function with supplied arguments
373      t0 = time.time()
374
375      T = my_F(*args, **kwargs) # Built-in 'apply' deprecated in Py3K   
376     
377      #comptime = round(time.time()-t0)
378      comptime = time.time()-t0
379
380      if verbose is True:
381        msg2(funcname,args,kwargs,comptime,reason)
382
383      # Save results and estimated loading time to cache
384      loadtime = save_results_to_cache(T, CD, FN, my_F, deps, comptime, \
385                                       funcname, dependencies, compression)
386      if verbose is True:
387        msg3(loadtime, CD, FN, deps, compression)
388      compressed = compression
389
390  if options['savestat'] and (not test or Retrieved):
391  ##if options['savestat']:
392    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
393
394  return(T)  # Return results in all cases
395
396# -----------------------------------------------------------------------------
397
398def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
399  """Generate statistics of caching efficiency.
400
401  USAGE:
402    cachestat(sortidx, period, showuser, cachedir)
403
404  ARGUMENTS:
405    sortidx --  Index of field by which lists are (default: 4)
406                Legal values are
407                 0: 'Name'
408                 1: 'Hits'
409                 2: 'CPU'
410                 3: 'Time Saved'
411                 4: 'Gain(%)'
412                 5: 'Size'
413    period --   If set to -1 all available caching history is used.
414                If set 0 only the current month is used (default -1).
415    showuser -- Flag for additional table showing user statistics
416                (default: None).
417    cachedir -- Directory for cache files (default: options['cachedir']).
418
419  DESCRIPTION:
420    Logged caching statistics is converted into summaries of the form
421    --------------------------------------------------------------------------
422    Function Name   Hits   Exec(s)  Cache(s)  Saved(s)   Gain(%)      Size
423    --------------------------------------------------------------------------
424  """
425
426  __cachestat(sortidx, period, showuser, cachedir)
427  return
428
429# -----------------------------------------------------------------------------
430
431# Has mostly been moved to proper unit test.
432# What remains here includes example of the
433# cache statistics form.
434def test(cachedir=None, verbose=False, compression=None):
435  """Test the functionality of caching.
436
437  USAGE:
438    test(verbose)
439
440  ARGUMENTS:
441    verbose --     Flag whether caching will output its statistics (default=False)
442    cachedir --    Directory for cache files (Default: options['cachedir'])
443    compression -- Flag zlib compression (Default: options['compression'])
444  """
445   
446  import string, time
447
448  # Initialise
449  #
450  #import caching
451  #reload(caching)
452
453  if not cachedir:
454    cachedir = options['cachedir']
455
456  if verbose is None:  # Do NOT write 'if not verbose:', it could be zero.
457    verbose = options['verbose']
458 
459  if compression == None:  # Do NOT write 'if not compression:',
460                           # it could be zero.
461    compression = options['compression']
462  else:
463    try:
464      set_option('compression', compression)
465    except:
466      test_error('Set option failed')     
467
468  try:
469    import zlib
470  except:
471    log.critical()
472    log.critical('*** Could not find zlib, default to no-compression      ***')
473    log.critical('*** Installing zlib will improve performance of caching ***')
474    log.critical()
475    compression = 0       
476    set_option('compression', compression)   
477 
478  log.critical('\nTesting caching module - please stand by\n')
479
480  # Define a test function to be cached
481  #
482  def f(a,b,c,N,x=0,y='abcdefg'):
483    """f(a,b,c,N)
484       Do something time consuming and produce a complex result.
485    """
486
487    import string
488
489    B = []
490    for n in range(N):
491      s = str(n+2.0/(n + 4.0))+'.a'*10
492      B.append((a,b,c,s,n,x,y))
493    return(B)
494   
495  # Check that default cachedir is OK
496  #     
497  CD = checkdir(cachedir,verbose)   
498   
499   
500  # Make a dependency file
501  #   
502  try:
503    DepFN = CD + 'testfile.tmp'
504    DepFN_wildcard = CD + 'test*.tmp'
505    Depfile = open(DepFN,'w')
506    Depfile.write('We are the knights who say NI!')
507    Depfile.close()
508    test_OK('Wrote file %s' %DepFN)
509  except:
510    test_error('Could not open file %s for writing - check your environment' \
511               % DepFN)
512
513  # Check set_option (and switch stats off
514  #   
515  try:
516    set_option('savestat',0)
517    assert(options['savestat'] == 0)
518    test_OK('Set option')
519  except:
520    test_error('Set option failed')   
521   
522  # Make some test input arguments
523  #
524  N = 5000  #Make N fairly small here
525
526  a = [1,2]
527  b = ('Thou shalt count the number three',4)
528  c = {'Five is right out': 6, (7,8): 9}
529  x = 3
530  y = 'holy hand granate'
531
532  # Test caching
533  #
534  if compression:
535    comprange = 2
536  else:
537    comprange = 1
538
539  for comp in range(comprange):
540 
541    # Evaluate and store
542    #
543    try:
544      T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
545                   verbose=verbose, compression=comp)
546      if comp:                   
547        test_OK('Caching evaluation with compression')
548      else:     
549        test_OK('Caching evaluation without compression')     
550    except:
551      if comp:
552        test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
553      else:
554        test_error('Caching evaluation failed - try caching.test(verbose=1)')
555
556    # Retrieve
557    #                           
558    try:                         
559      T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
560                   compression=comp) 
561
562      if comp:                   
563        test_OK('Caching retrieval with compression')
564      else:     
565        test_OK('Caching retrieval without compression')     
566    except:
567      if comp:
568        test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
569      else:                                     
570        test_error('Caching retrieval failed - try caching.test(verbose=1)')
571
572    # Reference result
573    #   
574    T3 = f(a,b,c,N,x=x,y=y)  # Compute without caching
575   
576    if T1 == T2 and T2 == T3:
577      if comp:
578        test_OK('Basic caching functionality (with compression)')
579      else:
580        test_OK('Basic caching functionality (without compression)')
581    else:
582      test_error('Cached result does not match computed result')
583
584
585  # Test return_filename
586  #   
587  try:
588    FN = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
589                 return_filename=1)   
590    assert(FN[:2] == 'f[')
591    test_OK('Return of cache filename')
592  except:
593    test_error('Return of cache filename failed')
594
595  # Test existence of cachefiles
596  # 
597  try:
598    (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
599    (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
600    (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
601    test_OK('Presence of cache files')
602    datafile.close()
603    argsfile.close()
604    admfile.close()
605  except:
606    test_error('Expected cache files did not exist') 
607             
608  # Test 'test' function when cache is present
609  #     
610  try:
611    #T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
612    #                   evaluate=1) 
613    T4 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
614    assert(T1 == T4)
615
616    test_OK("Option 'test' when cache file present")
617  except:
618    test_error("Option 'test' when cache file present failed")     
619
620  # Test that 'clear' works
621  #
622  #try:
623  #  cache(f,'clear',verbose=verbose)
624  #  test_OK('Clearing of cache files')
625  #except:
626  #  test_error('Clear does not work')
627  try:
628    cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)   
629    test_OK('Clearing of cache files')
630  except:
631    test_error('Clear does not work') 
632
633 
634
635  # Test 'test' function when cache is absent
636  #     
637  try:
638    T4 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
639    assert(T4 is None)
640    test_OK("Option 'test' when cache absent")
641  except:
642    test_error("Option 'test' when cache absent failed")     
643         
644  # Test dependencies
645  #
646  T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
647               dependencies=DepFN) 
648  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
649               dependencies=DepFN)                     
650                       
651  if T1 == T2:
652    test_OK('Basic dependencies functionality')
653  else:
654    test_error('Dependencies do not work')
655
656  # Test basic wildcard dependency
657  #
658  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
659               dependencies=DepFN_wildcard)                     
660   
661  if T1 == T3:
662    test_OK('Basic dependencies with wildcard functionality')
663  else:
664    test_error('Dependencies with wildcards do not work')
665
666
667  # Test that changed timestamp in dependencies triggers recomputation
668 
669  # Modify dependency file
670  Depfile = open(DepFN,'a')
671  Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
672  Depfile.close()
673 
674  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
675               dependencies=DepFN, test = 1)                     
676 
677  if T3 is None:
678    test_OK('Changed dependencies recognised')
679  else:
680    test_error('Changed dependencies not recognised')   
681 
682  # Test recomputation when dependencies have changed
683  #
684  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
685               dependencies=DepFN)                       
686  if T1 == T3:
687    test_OK('Recomputed value with changed dependencies')
688  else:
689    test_error('Recomputed value with changed dependencies failed')
690
691  # Performance test (with statistics)
692  # Don't really rely on this as it will depend on specific computer.
693  #
694
695  set_option('savestat',1)
696
697  N = 20*N   #Should be large on fast computers...
698  tt = time.time()
699  T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
700  t1 = time.time() - tt
701 
702  tt = time.time()
703  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
704  t2 = time.time() - tt
705 
706  if T1 == T2:
707    if t1 > t2:
708      test_OK('Performance test: relative time saved = %s pct' \
709              %str(round((t1-t2)*100/t1,2)))
710  else:       
711    test_error('Basic caching failed for new problem')
712           
713  # Test presence of statistics file
714  #
715  try: 
716    DIRLIST = os.listdir(CD)
717    SF = []
718    for FN in DIRLIST:
719      if string.find(FN,statsfile) >= 0:
720        fid = open(CD+FN,'r')
721        fid.close()
722    test_OK('Statistics files present') 
723  except:
724    test_OK('Statistics files cannot be opened')         
725     
726  print_header_box('Show sample output of the caching function:')
727 
728  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
729  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
730  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
731 
732  print_header_box('Show sample output of cachestat():')
733  if unix:
734    cachestat()   
735  else:
736    try:
737      import time
738      t = time.strptime('2030','%Y')
739      cachestat()
740    except: 
741      log.critical('cachestat() does not work here, because it relies on '
742                   'time.strptime() which is unavailable in Windows')
743     
744  test_OK('Caching self test completed')   
745     
746           
747  # Test setoption (not yet implemented)
748  #
749
750 
751#==============================================================================
752# Auxiliary functions
753#==============================================================================
754
755# Import pickler
756# cPickle is used by functions mysave, myload, and compare
757#
758import cPickle  # 10 to 100 times faster than pickle
759pickler = cPickle
760
761# Local immutable constants
762#
763comp_level = 1              # Compression level for zlib.
764                            # comp_level = 1 works well.
765textwidth1 = 16             # Text width of key fields in report forms.
766#textwidth2 = 132            # Maximal width of textual representation of
767textwidth2 = 300            # Maximal width of textual representation of
768                            # arguments.
769textwidth3 = 16             # Initial width of separation lines. Is modified.
770textwidth4 = 50             # Text width in test_OK()
771statsfile  = '.cache_stat'  # Basefilename for cached statistics.
772                            # It will reside in the chosen cache directory.
773
774file_types = ['Result',     # File name extension for cached function results.
775              'Args',       # File name extension for stored function args.
776              'Admin']      # File name extension for administrative info.
777
778Reason_msg = ['OK',         # Verbose reasons for recomputation
779              'No cached result', 
780              'Dependencies have changed', 
781              'Arguments have changed',
782              'Bytecode has changed',
783              'Recomputation was requested by caller',
784              'Cached file was unreadable']             
785             
786# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
787
788def CacheLookup(CD, FN, my_F, args, kwargs, deps, verbose, compression, 
789                dependencies):
790  """Determine whether cached result exists and return info.
791
792  USAGE:
793    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \ 
794    CacheLookup(CD, FN, my_F, args, kwargs, deps, verbose, compression, \
795                dependencies)
796
797  INPUT ARGUMENTS:
798    CD --            Cache Directory
799    FN --            Suggested cache file name
800    my_F --          Callable object
801    args --          Tuple of arguments
802    kwargs --        Dictionary of keyword arguments   
803    deps --          Dependencies time stamps
804    verbose --       Flag text output
805    compression --   Flag zlib compression
806    dependencies --  Given list of dependencies
807   
808  OUTPUT ARGUMENTS:
809    T --             Cached result if present otherwise None
810    FN --            File name under which new results must be saved
811    Retrieved --     True if a valid cached result was found
812    reason --        0: OK (if Retrieved),
813                     1: No cached result,
814                     2: Dependencies have changed,
815                     3: Arguments have changed
816                     4: Bytecode has changed
817                     5: Recomputation was forced
818                     6: Unreadable file
819    comptime --      Number of seconds it took to computed cachged result
820    loadtime --      Number of seconds it took to load cached result
821    compressed --    Flag (0,1) if cached results were compressed or not
822
823  DESCRIPTION:
824    Determine if cached result exists as follows:
825    Load in saved arguments and bytecode stored under hashed filename.
826    If they are identical to current arguments and bytecode and if dependencies
827    have not changed their time stamp, then return cached result.
828
829    Otherwise return filename under which new results should be cached.
830    Hash collisions are handled recursively by calling CacheLookup again with a
831    modified filename.
832  """
833
834  import time, string
835
836  # Assess whether cached result exists - compressed or not.
837  #
838  if verbose:
839    log.critical('Caching: looking for cached files %s_{%s,%s,%s}.z'
840                 % (CD+FN, file_types[0], file_types[1], file_types[2]))
841  (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
842  (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
843  (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
844
845  if verbose is True and deps is not None:
846    log.critical('Caching: Dependencies are %s' % deps.keys())
847
848  if not (argsfile and datafile and admfile) or \
849     not (compressed0 == compressed1 and compressed0 == compressed2):
850    # Cached result does not exist or files were compressed differently
851    #
852    # This will ensure that evaluation will take place unless all files are
853    # present.
854
855    reason = 1
856    return(None,FN,None,reason,None,None,None) #Recompute using same filename
857
858  compressed = compressed0  # Remember if compressed files were actually used
859  datafile.close()
860
861  # Retrieve arguments and adm. info
862  #
863  R, reason = myload(argsfile, compressed)  # The original arguments
864  argsfile.close()
865   
866  if reason > 0:
867      # Recompute using same filename   
868      return(None, FN, None, reason, None, None, None)
869  else:   
870      (argsref, kwargsref) = R
871
872  R, reason = myload(admfile, compressed)
873  admfile.close() 
874
875  if reason > 0:
876    return(None,FN,None,reason,None,None,None) #Recompute using same filename
877
878 
879  depsref  = R[0]  # Dependency statistics
880  comptime = R[1]  # The computation time
881  coderef  = R[2]  # The byte code
882  funcname = R[3]  # The function name
883
884  # Check if dependencies have changed
885  #
886  if dependencies and not compare(depsref, deps):
887    if verbose:
888      log.critical('Dependencies %s have changed - recomputing' % dependencies)
889
890    # Don't use cached file - recompute
891    reason = 2
892    return(None, FN, None, reason, None, None, None)
893
894  # Get bytecode from my_F
895  #
896  bytecode = get_bytecode(my_F)
897
898  # Check if arguments or bytecode have changed
899  if compare(argsref, args) and compare(kwargsref, kwargs) and \
900     (not options['bytecode'] or compare(bytecode, coderef)):
901
902    # Arguments and dependencies match. Get cached results
903    T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
904    if reason > 0:
905        # Recompute using same FN     
906        return(None, FN, None, reason, None, None, None)
907
908    Retrieved = 1
909    reason = 0
910
911    if verbose:
912      msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
913
914      if loadtime >= comptime:
915        log.critical('Caching did not yield any gain.')
916        log.critical('Consider executing function %s without caching.'
917                     % funcname)
918  else:
919
920    # Non matching arguments or bytecodes signify a hash-collision.
921    # This is resolved by recursive search of cache filenames
922    # until either a matching or an unused filename is found.
923    #
924    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
925        CacheLookup(CD, FN+'x', my_F, args, kwargs, deps, 
926                    verbose, compression, dependencies)
927
928    # The real reason is that args or bytecodes have changed.
929    # Not that the recursive seach has found an unused filename
930    if not Retrieved:
931      if not compare(bytecode, coderef):
932        reason = 4 # Bytecode has changed
933      else:   
934        reason = 3 # Arguments have changed
935       
936   
937  return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
938
939# -----------------------------------------------------------------------------
940
941def clear_cache(CD, my_F=None, verbose=None):
942  """Clear cache for my_F.
943
944  USAGE:
945     clear(CD, my_F, verbose)
946
947  ARGUMENTS:
948     CD --       Caching directory (required)
949     my_F --     Function object (default: None)
950     verbose --  Flag verbose output (default: None)
951
952  DESCRIPTION:
953
954    If my_F == None, clear everything,
955    otherwise clear only files pertaining to my_F.
956  """
957
958  import os, re
959   
960  if CD[-1] != os.sep:
961    CD = CD+os.sep
962 
963  if verbose == None:
964    verbose = options['verbose']
965
966  # FIXME: Windows version needs to be tested
967
968  if my_F:
969    funcname = get_funcname(my_F)
970    if verbose:
971      log.critical('Clearing %s' % CD+funcname+'*')
972
973    file_names = os.listdir(CD)
974    for file_name in file_names:
975      #RE = re.search('^' + funcname,file_name)  #Inefficient
976      #if RE:
977      if file_name[:len(funcname)] == funcname:
978        if unix:
979          os.remove(CD+file_name)
980        else:
981          os.system('del '+CD+file_name)
982          # FIXME: os.remove doesn't work under windows
983  else:
984    file_names = os.listdir(CD)
985    if len(file_names) > 0:
986      if verbose:
987        log.critical('Remove the following files:')
988        for file_name in file_names:
989            log.critical('     ' + file_name)
990
991        A = raw_input('Delete (Y/N)[N] ?')
992      else:
993        A = 'Y' 
994       
995      if A == 'Y' or A == 'y':
996        for file_name in file_names:
997          if unix:
998            os.remove(CD+file_name)
999          else:
1000            os.system('del '+CD+file_name)
1001            # FIXME: os.remove doesn't work under windows
1002          #exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
1003
1004# -----------------------------------------------------------------------------
1005
1006def DeleteOldFiles(CD,verbose=None):
1007  """Remove expired files
1008
1009  USAGE:
1010    DeleteOldFiles(CD,verbose=None)
1011  """
1012
1013  if verbose == None:
1014    verbose = options['verbose']
1015
1016  maxfiles = options['maxfiles']
1017
1018  # FIXME: Windows version
1019
1020  import os
1021  block = 1000  # How many files to delete per invokation
1022  Files = os.listdir(CD)
1023  numfiles = len(Files)
1024  if not unix: return  # FIXME: Windows case ?
1025
1026  if numfiles > maxfiles:
1027    delfiles = numfiles-maxfiles+block
1028    if verbose:
1029      log.critical('Deleting %d expired files:' % delfiles)
1030      os.system('ls -lur '+CD+'* | head -' + `delfiles`)            # List them
1031    os.system('ls -ur '+CD+'* | head -' + `delfiles` + ' | xargs /bin/rm')
1032                                                                  # Delete them
1033    # FIXME: Replace this with os.listdir and os.remove
1034
1035# -----------------------------------------------------------------------------
1036
1037def save_args_to_cache(CD, FN, args, kwargs, compression):
1038  """Save arguments to cache
1039
1040  USAGE:
1041    save_args_to_cache(CD,FN,args,kwargs,compression)
1042  """
1043
1044  import time, os, sys
1045
1046  (argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1047
1048  if argsfile is None:
1049    msg = 'ERROR (caching): Could not open argsfile for writing: %s' %FN
1050    raise IOError(msg)
1051
1052  mysave((args,kwargs),argsfile,compression)  # Save args and kwargs to cache
1053  argsfile.close()
1054
1055  # Change access rights if possible
1056  #
1057  #if unix:
1058  #  try:
1059  #    exitcode=os.system('chmod 666 '+argsfile.name)
1060  #  except:
1061  #    pass
1062  #else:
1063  #  pass  # FIXME: Take care of access rights under Windows
1064
1065  return
1066
1067# -----------------------------------------------------------------------------
1068
1069def save_results_to_cache(T, CD, FN, my_F, deps, comptime, funcname,
1070                          dependencies, compression):
1071  """Save computed results T and admin info to cache
1072
1073  USAGE:
1074    save_results_to_cache(T, CD, FN, my_F, deps, comptime, funcname,
1075                          dependencies, compression)
1076  """
1077
1078  import time, os, sys
1079
1080  (datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1081  (admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1082
1083  if not datafile:
1084    if verbose:
1085        log.critical('ERROR: Could not open %s' % datafile.name)
1086    raise IOError
1087
1088  if not admfile:
1089    if verbose:
1090        log.critical('ERROR: Could not open %s' % admfile.name)
1091    raise IOError
1092
1093  t0 = time.time()
1094
1095  mysave(T,datafile,compression)  # Save data to cache
1096  datafile.close()
1097  #savetime = round(time.time()-t0,2)
1098  savetime = time.time()-t0 
1099
1100  bytecode = get_bytecode(my_F)  # Get bytecode from function object
1101  admtup = (deps, comptime, bytecode, funcname)  # Gather admin info
1102
1103  mysave(admtup,admfile,compression)  # Save admin info to cache
1104  admfile.close()
1105
1106  # Change access rights if possible
1107  #
1108  #if unix:
1109  #  try:
1110  #    exitcode=os.system('chmod 666 '+datafile.name)
1111  #    exitcode=os.system('chmod 666 '+admfile.name)
1112  #  except:
1113  #    pass
1114  #else:
1115  #  pass  # FIXME: Take care of access rights under Windows
1116
1117  return(savetime)
1118
1119# -----------------------------------------------------------------------------
1120
1121def load_from_cache(CD, FN, compression):
1122  """Load previously cached data from file FN
1123
1124  USAGE:
1125    load_from_cache(CD,FN,compression)
1126  """
1127
1128  import time
1129
1130  (datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1131  t0 = time.time()
1132  T, reason = myload(datafile,compressed)
1133
1134  loadtime = time.time()-t0
1135  datafile.close() 
1136
1137  return T, loadtime, compressed, reason
1138
1139# -----------------------------------------------------------------------------
1140
1141def myopen(FN, mode, compression=True):
1142  """Open file FN using given mode
1143
1144  USAGE:
1145    myopen(FN, mode, compression=True)
1146
1147  ARGUMENTS:
1148    FN --           File name to be opened
1149    mode --         Open mode (as in open)
1150    compression --  Flag zlib compression
1151
1152  DESCRIPTION:
1153     if compression
1154       Attempt first to open FN + '.z'
1155       If this fails try to open FN
1156     else do the opposite
1157     Return file handle plus info about whether it was compressed or not.
1158  """
1159
1160  import string
1161
1162  # Determine if file exists already (if writing was requested)
1163  # This info is only used to determine if access modes should be set
1164  #
1165  if 'w' in mode or 'a' in mode:
1166    try:
1167      file = open(FN+'.z','r')
1168      file.close()
1169      new_file = 0
1170    except:
1171      try:
1172        file = open(FN,'r') 
1173        file.close()
1174        new_file = 0
1175      except:
1176        new_file = 1
1177  else:
1178    new_file = 0 #Assume it exists if mode was not 'w'
1179 
1180
1181  compressed = 0
1182  if compression:
1183    try:
1184      file = open(FN+'.z',mode)
1185      compressed = 1
1186    except:
1187      try:
1188        file = open(FN,mode)
1189      except:
1190        file = None
1191  else:
1192    try:
1193      file = open(FN,mode)
1194    except:
1195      try:
1196        file = open(FN+'.z',mode)
1197        compressed = 1
1198      except:
1199        file = None
1200
1201  # Now set access rights if it is a new file
1202  #
1203  if file and new_file:
1204    if unix:
1205      exitcode=os.system('chmod 666 '+file.name)
1206    else:
1207      pass  # FIXME: Take care of access rights under Windows
1208
1209  return(file,compressed)
1210
1211# -----------------------------------------------------------------------------
1212
1213def myload(file, compressed):
1214  """Load data from file
1215
1216  USAGE:
1217    myload(file, compressed)
1218  """
1219
1220  reason = 0
1221  try:
1222    if compressed:
1223      import zlib
1224
1225      RsC = file.read()
1226      try:
1227        Rs  = zlib.decompress(RsC)
1228      except:
1229        #  File "./caching.py", line 1032, in load_from_cache
1230        #  T = myload(datafile,compressed)
1231        #  File "./caching.py", line 1124, in myload
1232        #  Rs  = zlib.decompress(RsC)
1233        #  zlib.error: Error -5 while decompressing data
1234        #raise Exception
1235        reason = 6  # Unreadable file
1236        return None, reason 
1237     
1238     
1239      del RsC  # Free up some space
1240      R = pickler.loads(Rs)
1241    else:
1242      try:
1243        R = pickler.load(file)
1244      #except EOFError, e:
1245      except:
1246        #Catch e.g., file with 0 length or corrupted
1247        reason = 6  # Unreadable file
1248        return None, reason
1249     
1250  except MemoryError:
1251    import sys
1252    if options['verbose']:
1253      log.critical('ERROR: Out of memory while loading %s, aborting'
1254                   % file.name)
1255
1256    # Raise the error again for now
1257    #
1258    raise MemoryError
1259
1260  return R, reason
1261
1262# -----------------------------------------------------------------------------
1263
1264def mysave(T, file, compression):
1265  """Save data T to file
1266
1267  USAGE:
1268    mysave(T, file, compression)
1269
1270  """
1271
1272  bin = options['bin']
1273
1274  if compression:
1275    try:
1276      import zlib
1277    except:
1278      log.critical()
1279      log.critical('*** Could not find zlib ***')
1280      log.critical('*** Try to run caching with compression off ***')
1281      log.critical("*** caching.set_option('compression', 0) ***")
1282      raise Exception
1283     
1284
1285    try:
1286      Ts  = pickler.dumps(T, bin)
1287    except MemoryError:
1288      msg = '****WARNING (caching.py): Could not pickle data for compression.'
1289      msg += ' Try using compression = False'
1290      raise MemoryError(msg)
1291    else: 
1292      # Compressed pickling     
1293      TsC = zlib.compress(Ts, comp_level)
1294      file.write(TsC)
1295  else:
1296      # Uncompressed pickling
1297      pickler.dump(T, file, bin)
1298
1299      # FIXME: This may not work on Windoze network drives.
1300      # The error msg is IOError: [Errno 22] Invalid argument
1301      # Testing with small files was OK, though.
1302      # I think this is an OS problem.
1303
1304      # Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1305     
1306# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1307#
1308#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1309#
1310#The following extract is taken from the site http://www.python.org:
1311#
1312#---------------------------------------------------------------------------------------------
1313#exception IOError
1314#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1315#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1316#---------------------------------------------------------------------------------------------
1317#
1318#The error code(s) that accompany exceptions are described at:
1319#http://www.python.org/dev/doc/devel//lib/module-errno.html
1320#
1321#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1322#
1323#exceptions.IOError invalid argument Errno 22
1324       
1325      #try:
1326      #  pickler.dump(T,file,bin)
1327      #except IOError, e:
1328      #  print e
1329      #  msg = 'Could not store to %s, bin=%s' %(file, bin)
1330      #  raise Exception(msg)
1331     
1332
1333# -----------------------------------------------------------------------------
1334
1335   
1336def myhash(T, ids=None):
1337  """Compute hashed integer from a range of inputs.
1338  If T is not hashable being e.g. a tuple T, myhash will recursively
1339  hash the values individually
1340
1341  USAGE:
1342    myhash(T)
1343
1344  ARGUMENTS:
1345    T -- Anything
1346  """
1347
1348  from types import TupleType, ListType, DictType, InstanceType 
1349   
1350  if type(T) in [TupleType, ListType, DictType, InstanceType]: 
1351      # Keep track of unique id's to protect against infinite recursion
1352      if ids is None: ids = []
1353
1354      # Check if T has already been encountered
1355      i = id(T) 
1356 
1357      if i in ids:
1358          return 0 # T has been hashed already     
1359      else:
1360          ids.append(i)
1361   
1362
1363   
1364  # Start hashing 
1365 
1366  # On some architectures None, False and True gets different hash values
1367  if T is None:
1368      return(-1)
1369  if T is False:
1370      return(0)
1371  if T is True:
1372      return(1)
1373
1374  # Get hash values for hashable entries
1375  if type(T) in [TupleType, ListType]:
1376      hvals = []
1377      for t in T:
1378          h = myhash(t, ids)
1379          hvals.append(h)
1380      val = hash(tuple(hvals))
1381  elif type(T) == DictType:
1382      # Make dictionary ordering unique 
1383     
1384      # FIXME(Ole): Need new way of doing this in Python 3.0
1385      I = T.items()
1386      I.sort()   
1387      val = myhash(I, ids)
1388  elif isinstance(T, num.ndarray):
1389      T = num.array(T) # Ensure array is contiguous
1390
1391      # Use mean value for efficiency
1392      val = hash(num.average(T.flat))
1393  elif type(T) == InstanceType:
1394      # Use the attribute values
1395      val = myhash(T.__dict__, ids)
1396  else:
1397      try:
1398          val = hash(T)
1399      except:
1400          val = 1
1401
1402  return(val)
1403
1404
1405
1406def compare(A, B, ids=None):
1407    """Safe comparison of general objects
1408
1409    USAGE:
1410      compare(A,B)
1411
1412    DESCRIPTION:
1413      Return 1 if A and B they are identical, 0 otherwise
1414    """
1415
1416    # Keep track of unique id's to protect against infinite recursion
1417    if ids is None: ids = {}
1418
1419    # Check if T has already been encountered
1420    iA = id(A) 
1421    iB = id(B)     
1422   
1423    if (iA, iB) in ids:
1424        # A and B have been compared already
1425        return ids[(iA, iB)]
1426    else:
1427        ids[(iA, iB)] = True
1428   
1429   
1430    # Check if arguments are of same type
1431    if type(A) != type(B):
1432        return False
1433       
1434 
1435    # Compare recursively based on argument type
1436    if isinstance(A, (tuple, list)):
1437        N = len(A)
1438        if len(B) != N: 
1439            identical = False
1440        else:
1441            identical = True
1442            for i in range(N):
1443                if not compare(A[i], B[i], ids): 
1444                    identical = False; break
1445                   
1446    elif isinstance(A, dict):
1447        if len(A) != len(B):
1448            identical = False
1449        else:                       
1450            # Make dictionary ordering unique
1451            a = A.items(); a.sort()   
1452            b = B.items(); b.sort()
1453           
1454            identical = compare(a, b, ids)
1455           
1456    elif isinstance(A, num.ndarray):
1457        # Use element by element comparison
1458        identical = num.alltrue(A==B)
1459
1460    elif type(A) == types.InstanceType:
1461        # Take care of special case where elements are instances           
1462        # Base comparison on attributes     
1463        identical = compare(A.__dict__, 
1464                            B.__dict__, 
1465                            ids)
1466    else:       
1467        # Fall back to general code
1468        try:
1469            identical = (A == B)
1470        except:
1471            import pickle
1472            # Use pickle to compare data
1473            # The native pickler must be used
1474            # since the faster cPickle does not
1475            # guarantee a unique translation           
1476            try:
1477                identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
1478            except:
1479                identical = False
1480
1481    # Record result of comparison and return           
1482    ids[(iA, iB)] = identical
1483   
1484    return(identical)
1485
1486   
1487# -----------------------------------------------------------------------------
1488
1489def nospace(s):
1490  """Replace spaces in string s with underscores
1491
1492  USAGE:
1493    nospace(s)
1494
1495  ARGUMENTS:
1496    s -- string
1497  """
1498
1499  import string
1500
1501  newstr = ''
1502  for i in range(len(s)):
1503    if s[i] == ' ':
1504      newstr = newstr+'_'
1505    else:
1506      newstr = newstr+s[i]
1507
1508  return(newstr)
1509
1510# -----------------------------------------------------------------------------
1511
1512def get_funcname(my_F):
1513  """Retrieve name of function object func (depending on its type)
1514
1515  USAGE:
1516    get_funcname(my_F)
1517  """
1518
1519  import string
1520
1521  if type(my_F) == types.FunctionType:
1522    funcname = my_F.func_name
1523  elif type(my_F) == types.BuiltinFunctionType:
1524    funcname = my_F.__name__
1525  else:
1526    tab = string.maketrans("<>'","   ")
1527    tmp = string.translate(repr(my_F), tab)
1528    tmp = string.split(tmp)
1529    funcname = string.join(tmp)
1530
1531    # Truncate memory address as in
1532    # class __main__.Dummy at 0x00A915D0
1533    index = funcname.find('at 0x')
1534    if index >= 0:
1535      funcname = funcname[:index+5] # Keep info that there is an address
1536
1537  funcname = nospace(funcname)
1538  return(funcname)
1539
1540# -----------------------------------------------------------------------------
1541
1542def get_bytecode(my_F):
1543  """ Get bytecode from function object.
1544
1545  USAGE:
1546    get_bytecode(my_F)
1547  """
1548
1549  if type(my_F) == types.FunctionType:
1550    return get_func_code_details(my_F)
1551  elif type(my_F) == types.MethodType:
1552    return get_func_code_details(my_F.im_func)
1553  elif type(my_F) == types.InstanceType:   
1554    if hasattr(my_F, '__call__'):
1555      # Get bytecode from __call__ method
1556      bytecode = get_func_code_details(my_F.__call__.im_func)
1557     
1558      # Add hash value of object to detect attribute changes
1559      return bytecode + (myhash(my_F),) 
1560    else:
1561      msg = 'Instance %s was passed into caching in the role of a function ' % str(my_F)
1562      msg = ' but it was not callable.'
1563      raise Exception(msg)
1564  elif type(my_F) in [types.BuiltinFunctionType, types.BuiltinMethodType]:     
1565    # Built-in functions are assumed not to change 
1566    return None, 0, 0, 0
1567  elif type(my_F) == types.ClassType:
1568      # Get bytecode from __init__ method
1569      bytecode = get_func_code_details(my_F.__init__.im_func)   
1570      return bytecode     
1571  else:
1572    msg = 'Unknown function type: %s' % type(my_F)
1573    raise Exception(msg)
1574
1575
1576 
1577 
1578def get_func_code_details(my_F):
1579  """Extract co_code, co_consts, co_argcount, func_defaults
1580  """
1581 
1582  bytecode = my_F.func_code.co_code
1583  consts = my_F.func_code.co_consts
1584  argcount = my_F.func_code.co_argcount   
1585  defaults = my_F.func_defaults       
1586 
1587  return bytecode, consts, argcount, defaults 
1588
1589# -----------------------------------------------------------------------------
1590
1591def get_depstats(dependencies):
1592  """ Build dictionary of dependency files and their size, mod. time and ctime.
1593
1594  USAGE:
1595    get_depstats(dependencies):
1596  """
1597
1598  d = {}
1599  if dependencies:
1600
1601    #Expand any wildcards
1602    import glob
1603    expanded_dependencies = []
1604    for FN in dependencies:
1605      expanded_FN = glob.glob(FN)
1606
1607      if expanded_FN == []:
1608        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1609        raise Exception(errmsg)
1610
1611      expanded_dependencies += expanded_FN
1612
1613   
1614    for FN in expanded_dependencies:
1615      if not isinstance(FN, basestring):
1616        errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1617        errmsg += '                   Dependency given: %s' %FN
1618        raise Exception(errmsg)
1619      if not os.access(FN,os.F_OK):
1620        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1621        raise Exception(errmsg)
1622      (size,atime,mtime,ctime) = filestat(FN)
1623
1624      # We don't use atime because that would cause recomputation every time.
1625      # We don't use ctime because that is irrelevant and confusing for users.
1626      d.update({FN : (size,mtime)})
1627
1628  return(d)
1629
1630# -----------------------------------------------------------------------------
1631
1632def filestat(FN):
1633  """A safe wrapper using os.stat to get basic file statistics
1634     The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1635
1636  USAGE:
1637    filestat(FN)
1638
1639  DESCRIPTION:
1640     Must compile Python with
1641     CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1642              configure
1643     as given in section 8.1.1 Large File Support in the Libray Reference
1644  """
1645
1646  import os, time
1647
1648  try:
1649    stats = os.stat(FN)
1650    size  = stats[6]
1651    atime = stats[7]
1652    mtime = stats[8]
1653    ctime = stats[9]
1654  except:
1655
1656    # Hack to get the results anyway (works only on Unix at the moment)
1657    #
1658    log.critical('Hack to get os.stat when files are too large')
1659
1660    if unix:
1661      tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1662      # Unique filename, FIXME: Use random number
1663
1664      # Get size and access time (atime)
1665      #
1666      exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1667      (size,atime) = get_lsline(tmp)
1668
1669      # Get size and modification time (mtime)
1670      #
1671      exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1672      (size,mtime) = get_lsline(tmp)
1673
1674      # Get size and ctime
1675      #
1676      exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1677      (size,ctime) = get_lsline(tmp)
1678
1679      try:
1680        exitcode=os.system('rm '+tmp)
1681        # FIXME: Gives error if file doesn't exist
1682      except:
1683        pass
1684    else:
1685      pass
1686      raise Exception  # FIXME: Windows case
1687
1688  return(long(size),atime,mtime,ctime)
1689
1690# -----------------------------------------------------------------------------
1691
1692def get_lsline(FN):
1693  """get size and time for filename
1694
1695  USAGE:
1696    get_lsline(file_name)
1697
1698  DESCRIPTION:
1699    Read in one line 'ls -la' item from file (generated by filestat) and
1700    convert time to seconds since epoch. Return file size and time.
1701  """
1702
1703  import string, time
1704
1705  f = open(FN,'r')
1706  info = f.read()
1707  info = string.split(info)
1708
1709  size = info[4]
1710  week = info[5]
1711  mon  = info[6]
1712  day  = info[7]
1713  hour = info[8]
1714  year = info[9]
1715
1716  str = week+' '+mon+' '+day+' '+hour+' '+year
1717  timetup = time.strptime(str)
1718  t = time.mktime(timetup)
1719  return(size, t)
1720
1721# -----------------------------------------------------------------------------
1722
1723def checkdir(CD, verbose=None, warn=False):
1724  """Check or create caching directory
1725
1726  USAGE:
1727    checkdir(CD,verbose):
1728
1729  ARGUMENTS:
1730    CD -- Directory
1731    verbose -- Flag verbose output (default: None)
1732
1733  DESCRIPTION:
1734    If CD does not exist it will be created if possible
1735  """
1736
1737  import os
1738  import os.path
1739
1740  if CD[-1] != os.sep: 
1741    CD = CD + os.sep  # Add separator for directories
1742
1743  CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1744  if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1745    try:
1746      exitcode=os.mkdir(CD)
1747
1748      # Change access rights if possible
1749      #
1750      if unix:
1751        exitcode=os.system('chmod 777 '+CD)
1752      else:
1753        pass  # FIXME: What about acces rights under Windows?
1754      if verbose: log.critical('MESSAGE: Directory %s created.' % CD)
1755    except:
1756      if warn is True:
1757        log.critical('WARNING: Directory %s could not be created.' % CD)
1758      if unix:
1759        CD = '/tmp/'
1760      else:
1761        CD = 'C:' 
1762      if warn is True:
1763        log.critical('Using directory %s instead' % CD)
1764
1765  return(CD)
1766
1767checkdir(cachedir, warn=True)
1768
1769#==============================================================================
1770# Statistics
1771#==============================================================================
1772
1773def addstatsline(CD, funcname, FN, Retrieved, reason, comptime, loadtime,
1774                 compression):
1775  """Add stats entry
1776
1777  USAGE:
1778    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1779
1780  DESCRIPTION:
1781    Make one entry in the stats file about one cache hit recording time saved
1782    and other statistics. The data are used by the function cachestat.
1783  """
1784
1785  import os, time
1786
1787  try:
1788    TimeTuple = time.localtime(time.time())
1789    extension = time.strftime('%b%Y',TimeTuple)
1790    SFN = CD+statsfile+'.'+extension
1791    #statfile = open(SFN,'a')
1792    (statfile, dummy) = myopen(SFN,'a',compression=0)
1793
1794    # Change access rights if possible
1795    #
1796    #if unix:
1797    #  try:
1798    #    exitcode=os.system('chmod 666 '+SFN)
1799    #  except:
1800    #    pass
1801  except:
1802    log.critical('Warning: Stat file could not be opened')
1803
1804  try:
1805    if os.environ.has_key('USER'):
1806      user = os.environ['USER']
1807    else:
1808      user = 'Nobody'
1809
1810    date = time.asctime(TimeTuple)
1811
1812    if Retrieved:
1813      hit = '1'
1814    else:
1815      hit = '0'
1816
1817    # Get size of result file
1818    #   
1819    if compression:
1820      stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1821    else:
1822      stats = os.stat(CD+FN+'_'+file_types[0])
1823 
1824    if stats: 
1825      size = stats[6]
1826    else:
1827      size = -1  # Error condition, but don't crash. This is just statistics 
1828
1829    # Build entry
1830    # 
1831    entry = date             + ',' +\
1832            user             + ',' +\
1833            FN               + ',' +\
1834            str(int(size))   + ',' +\
1835            str(compression) + ',' +\
1836            hit              + ',' +\
1837            str(reason)      + ',' +\
1838            str(round(comptime,4)) + ',' +\
1839            str(round(loadtime,4)) +\
1840            CR
1841           
1842    statfile.write(entry)
1843    statfile.close()
1844  except:
1845    log.critical('Warning: Writing of stat file failed')
1846
1847# -----------------------------------------------------------------------------
1848
1849# FIXME: should take cachedir as an optional arg
1850#
1851def __cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
1852  """  List caching statistics.
1853
1854  USAGE:
1855    __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1856
1857      Generate statistics of caching efficiency.
1858      The parameter sortidx determines by what field lists are sorted.
1859      If the optional keyword period is set to -1,
1860      all available caching history is used.
1861      If it is 0 only the current month is used.
1862      Future versions will include more than one month....
1863      OMN 20/8/2000
1864  """
1865
1866  import os
1867  import os.path
1868  from string import split, rstrip, find
1869  from time import strptime, localtime, strftime, mktime, ctime
1870
1871  # sortidx = 4    # Index into Fields[1:]. What to sort by.
1872
1873  Fields = ['Name', 'Hits', 'Exec(s)', \
1874            'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1875  Widths = [25,7,9,9,9,9,13]
1876  #Types = ['s','d','d','d','d','.2f','d']
1877  Types = ['s','d','.2f','.2f','.2f','.2f','d'] 
1878
1879  Dictnames = ['Function', 'User']
1880
1881  if not cachedir:
1882    cachedir = checkdir(options['cachedir'])
1883
1884  SD = os.path.expanduser(cachedir)  # Expand ~ or ~user in pathname
1885
1886  if period == -1:  # Take all available stats
1887    SFILENAME = statsfile
1888  else:  # Only stats from current month 
1889       # MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1890    TimeTuple = localtime(time())
1891    extension = strftime('%b%Y',TimeTuple)
1892    SFILENAME = statsfile+'.'+extension
1893
1894  DIRLIST = os.listdir(SD)
1895  SF = []
1896  for FN in DIRLIST:
1897    if find(FN,SFILENAME) >= 0:
1898      SF.append(FN)
1899
1900  blocksize = 15000000
1901  total_read = 0
1902  total_hits = 0
1903  total_discarded = 0
1904  firstday = mktime(strptime('2030','%Y'))
1905             # FIXME: strptime don't exist in WINDOWS ?
1906  lastday = 0
1907
1908  FuncDict = {}
1909  UserDict = {}
1910  for FN in SF:
1911    input = open(SD+FN,'r')
1912    log.critical('Reading file %s' % SD+FN)
1913
1914    while True:
1915      A = input.readlines(blocksize)
1916      if len(A) == 0: break
1917      total_read = total_read + len(A)
1918      for record in A:
1919        record = tuple(split(rstrip(record),','))
1920
1921        if len(record) == 9:
1922          timestamp = record[0]
1923       
1924          try:
1925            t = mktime(strptime(timestamp))
1926          except:
1927            total_discarded = total_discarded + 1         
1928            continue   
1929             
1930          if t > lastday:
1931            lastday = t
1932          if t < firstday:
1933            firstday = t
1934
1935          user     = record[1]
1936          my_F     = record[2]
1937
1938          # Strip hash-stamp off
1939          #
1940          i = find(my_F,'[')
1941          my_F = my_F[:i]
1942
1943          size        = float(record[3])
1944
1945          # Compression kepword can be Boolean
1946          if record[4] in ['True', '1']:
1947            compression = 1
1948          elif record[4] in ['False', '0']: 
1949            compression = 0
1950          else:
1951            log.critical('Unknown value of compression %s' % str(record[4]))
1952            log.critical(str(record))
1953            total_discarded = total_discarded + 1           
1954            continue
1955
1956          #compression = int(record[4]) # Can be Boolean
1957          hit         = int(record[5])
1958          reason      = int(record[6])   # Not used here   
1959          cputime     = float(record[7])
1960          loadtime    = float(record[8])
1961
1962          if hit:
1963            total_hits = total_hits + 1
1964            saving = cputime-loadtime
1965
1966            if cputime != 0:
1967              rel_saving = round(100.0*saving/cputime,2)
1968            else:
1969              #rel_saving = round(1.0*saving,2)
1970              rel_saving = 100.0 - round(1.0*saving,2)  # A bit of a hack
1971
1972            info = [1,cputime,loadtime,saving,rel_saving,size]
1973
1974            UpdateDict(UserDict,user,info)
1975            UpdateDict(FuncDict,my_F,info)
1976          else:
1977            pass #Stats on recomputations and their reasons could go in here
1978             
1979        else:
1980          total_discarded = total_discarded + 1
1981
1982    input.close()
1983
1984  # Compute averages of all sums and write list
1985  #
1986
1987  if total_read == 0:
1988    printline(Widths,'=')
1989    log.critical('CACHING STATISTICS: No valid records read')
1990    printline(Widths,'=')
1991    return
1992
1993  log.critical()
1994  printline(Widths,'=')
1995  log.critical('CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday))
1996  printline(Widths,'=')
1997  log.critical('  Total number of valid records %d' % total_read)
1998  log.critical('  Total number of discarded records %d' % total_discarded)
1999  log.critical('  Total number of hits %d' % total_hits)
2000  log.critical()
2001
2002  log.critical('  Fields %s are averaged over number of hits' % Fields[2:])
2003  log.critical('  Time is measured in seconds and size in bytes')
2004  log.critical('  Tables are sorted by %s' % Fields[1:][sortidx])
2005
2006  if showuser:
2007    Dictionaries = [FuncDict, UserDict]
2008  else:
2009    Dictionaries = [FuncDict]
2010
2011  i = 0
2012  for Dict in Dictionaries:
2013    for key in Dict.keys():
2014      rec = Dict[key]
2015      for n in range(len(rec)):
2016        if n > 0:
2017          rec[n] = round(1.0*rec[n]/rec[0],2)
2018      Dict[key] = rec
2019
2020    # Sort and output
2021    #
2022    keylist = SortDict(Dict,sortidx)
2023
2024    # Write Header
2025    #
2026    log.critical()
2027    printline(Widths,'-')
2028    n = 0
2029    for s in Fields:
2030      if s == Fields[0]:  # Left justify
2031        s = Dictnames[i] + ' ' + s; i=i+1
2032        #exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
2033        log.critical('%-*s' % (Widths[n], s))
2034        n += 1
2035      else:
2036        #exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
2037        log.critical('%*s' % (Widths[n], s))
2038        n += 1
2039    log.critical()
2040    printline(Widths,'-')
2041
2042    # Output Values
2043    #
2044    for key in keylist:
2045      rec = Dict[key]
2046      n = 0
2047      if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
2048      #exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
2049      log.critical('%-*s' % (Widths[n], str(key)))
2050      n += 1
2051      for val in rec:
2052        #exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
2053        log.critical('%*s' % (Widths[n], str(key)))
2054        n += 1
2055      log.critical()
2056    log.critical()
2057
2058#==============================================================================
2059# Auxiliary stats functions
2060#==============================================================================
2061
2062def UpdateDict(Dict, key, info):
2063  """Update dictionary by adding new values to existing.
2064
2065  USAGE:
2066    UpdateDict(Dict,key,info)
2067  """
2068
2069  if Dict.has_key(key):
2070    dinfo = Dict[key]
2071    for n in range(len(dinfo)):
2072      dinfo[n] = info[n] + dinfo[n]
2073  else:
2074    dinfo = info[:]  # Make a copy of info list
2075
2076  Dict[key] = dinfo
2077  return Dict
2078
2079# -----------------------------------------------------------------------------
2080
2081def SortDict(Dict, sortidx=0):
2082  """Sort dictionary
2083
2084  USAGE:
2085    SortDict(Dict,sortidx):
2086
2087  DESCRIPTION:
2088    Sort dictionary of lists according field number 'sortidx'
2089  """
2090
2091  sortlist  = []
2092  keylist = Dict.keys()
2093  for key in keylist:
2094    rec = Dict[key]
2095    if not isinstance(rec, (list, tuple)):
2096      rec = [rec]
2097
2098    if sortidx > len(rec)-1:
2099      msg = 'ERROR: Sorting index too large, sortidx = %s' % str(sortidx)
2100      raise IndexError(msg)
2101
2102    val = rec[sortidx]
2103    sortlist.append(val)
2104
2105  A = map(None,sortlist,keylist)
2106  A.sort()
2107  keylist = map(lambda x: x[1], A)  # keylist sorted by sortidx
2108
2109  return(keylist)
2110
2111# -----------------------------------------------------------------------------
2112
2113def printline(Widths,char):
2114  """Print textline in fixed field.
2115
2116  USAGE:
2117    printline(Widths,char)
2118  """
2119
2120  s = ''
2121  for n in range(len(Widths)):
2122    s = s+Widths[n]*char
2123    if n > 0:
2124      s = s+char
2125
2126  log.critical(s)
2127
2128#==============================================================================
2129# Messages
2130#==============================================================================
2131
2132def msg1(funcname, args, kwargs, reason):
2133  """Message 1
2134
2135  USAGE:
2136    msg1(funcname, args, kwargs, reason):
2137  """
2138
2139  import string
2140
2141  print_header_box('Evaluating function %s' %funcname)
2142 
2143  msg7(args, kwargs)
2144  msg8(reason) 
2145 
2146  print_footer()
2147 
2148# -----------------------------------------------------------------------------
2149
2150def msg2(funcname, args, kwargs, comptime, reason):
2151  """Message 2
2152
2153  USAGE:
2154    msg2(funcname, args, kwargs, comptime, reason)
2155  """
2156
2157  import string
2158
2159  #try:
2160  #  R = Reason_msg[reason]
2161  #except:
2162  #  R = 'Unknown reason' 
2163 
2164  #print_header_box('Caching statistics (storing) - %s' %R)
2165  print_header_box('Caching statistics (storing)') 
2166 
2167  msg6(funcname,args,kwargs)
2168  msg8(reason)
2169
2170  log.critical(string.ljust('| CPU time:', textwidth1) +
2171               str(round(comptime,2)) + ' seconds')
2172
2173# -----------------------------------------------------------------------------
2174
2175def msg3(savetime, CD, FN, deps, compression):
2176  """Message 3
2177
2178  USAGE:
2179    msg3(savetime, CD, FN, deps, compression)
2180  """
2181
2182  import string
2183  log.critical(string.ljust('| Loading time:', textwidth1) + 
2184               str(round(savetime,2)) + ' seconds (estimated)')
2185  msg5(CD,FN,deps,compression)
2186
2187# -----------------------------------------------------------------------------
2188
2189def msg4(funcname, args, kwargs, deps, comptime, loadtime, CD, FN, compression):
2190  """Message 4
2191
2192  USAGE:
2193    msg4(funcname, args, kwargs, deps, comptime, loadtime, CD, FN, compression)
2194  """
2195
2196  import string
2197
2198  print_header_box('Caching statistics (retrieving)')
2199 
2200  msg6(funcname,args,kwargs)
2201  log.critical(string.ljust('| CPU time:', textwidth1) +
2202               str(round(comptime,2)) + ' seconds')
2203  log.critical(string.ljust('| Loading time:', textwidth1) +
2204               str(round(loadtime,2)) + ' seconds')
2205  log.critical(string.ljust('| Time saved:', textwidth1) +
2206               str(round(comptime-loadtime,2)) + ' seconds')
2207  msg5(CD,FN,deps,compression)
2208
2209# -----------------------------------------------------------------------------
2210
2211def msg5(CD, FN, deps, compression):
2212  """Message 5
2213
2214  USAGE:
2215    msg5(CD, FN, deps, compression)
2216
2217  DESCRIPTION:
2218   Print dependency stats. Used by msg3 and msg4
2219  """
2220
2221  import os, time, string
2222
2223  log.critical('|')
2224  log.critical(string.ljust('| Caching dir: ', textwidth1) + CD)
2225
2226  if compression:
2227    suffix = '.z'
2228    bytetext = 'bytes, compressed'
2229  else:
2230    suffix = ''
2231    bytetext = 'bytes'
2232
2233  for file_type in file_types:
2234    file_name = FN + '_' + file_type + suffix
2235    stats = os.stat(CD+file_name)
2236    log.critical(string.ljust('| ' + file_type + ' file: ', textwidth1) +
2237                 file_name + '('+ str(stats[6]) + ' ' + bytetext + ')')
2238
2239  log.critical('|')
2240  if len(deps) > 0:
2241    log.critical('| Dependencies:  ')
2242    dependencies  = deps.keys()
2243    dlist = []; maxd = 0
2244    tlist = []; maxt = 0
2245    slist = []; maxs = 0
2246    for d in dependencies:
2247      stats = deps[d]
2248      t = time.ctime(stats[1])
2249      s = str(stats[0])
2250      #if s[-1] == 'L':
2251      #  s = s[:-1]  # Strip rightmost 'long integer' L off.
2252      #              # FIXME: Unnecessary in versions later than 1.5.2
2253
2254      if len(d) > maxd: maxd = len(d)
2255      if len(t) > maxt: maxt = len(t)
2256      if len(s) > maxs: maxs = len(s)
2257      dlist.append(d)
2258      tlist.append(t)
2259      slist.append(s)
2260
2261    for n in range(len(dlist)):
2262      d = string.ljust(dlist[n]+':', maxd+1)
2263      t = string.ljust(tlist[n], maxt)
2264      s = string.rjust(slist[n], maxs)
2265
2266      log.critical('| %s %s %s bytes' % (d, t, s))
2267  else:
2268    log.critical('| No dependencies')
2269  print_footer()
2270
2271# -----------------------------------------------------------------------------
2272
2273def msg6(funcname, args, kwargs):
2274  """Message 6
2275
2276  USAGE:
2277    msg6(funcname, args, kwargs)
2278  """
2279
2280  import string
2281  log.critical(string.ljust('| Function:', textwidth1) + funcname)
2282
2283  msg7(args, kwargs)
2284 
2285# -----------------------------------------------------------------------------   
2286
2287def msg7(args, kwargs):
2288  """Message 7
2289 
2290  USAGE:
2291    msg7(args, kwargs):
2292  """
2293 
2294  import string
2295 
2296  args_present = 0 
2297  if args:
2298    if len(args) == 1:
2299      log.critical(string.ljust('| Argument:', textwidth1) +
2300                   mkargstr(args[0], textwidth2))
2301    else:
2302      log.critical(string.ljust('| Arguments:', textwidth1) +
2303                   mkargstr(args, textwidth2))
2304    args_present = 1
2305           
2306  if kwargs:
2307    if len(kwargs) == 1:
2308      log.critical(string.ljust('| Keyword Arg:', textwidth1) +
2309                   mkargstr(kwargs, textwidth2))
2310    else:
2311      log.critical(string.ljust('| Keyword Args:', textwidth1) +
2312                   mkargstr(kwargs, textwidth2))
2313    args_present = 1
2314
2315  if not args_present:               
2316    log.critical('| No arguments')    # Default if no args or kwargs present
2317
2318# -----------------------------------------------------------------------------
2319
2320def msg8(reason):
2321  """Message 8
2322 
2323  USAGE:
2324    msg8(reason):
2325  """
2326 
2327  import string
2328   
2329  try:
2330    R = Reason_msg[reason]
2331  except:
2332    R = 'Unknown' 
2333 
2334  log.critical(string.ljust('| Reason:', textwidth1) + R)
2335   
2336# -----------------------------------------------------------------------------
2337
2338def print_header_box(line):
2339  """Print line in a nice box.
2340 
2341  USAGE:
2342    print_header_box(line)
2343
2344  """
2345  global textwidth3
2346
2347  import time
2348
2349  time_stamp = time.ctime(time.time())
2350  line = time_stamp + '. ' + line
2351   
2352  N = len(line) + 1
2353  s = '+' + '-'*N + CR
2354
2355  log.critical(s + '| ' + line + CR + s)
2356
2357  textwidth3 = N
2358
2359# -----------------------------------------------------------------------------
2360   
2361def print_footer():
2362  """Print line same width as that of print_header_box.
2363  """
2364 
2365  N = textwidth3
2366  s = '+' + '-'*N + CR   
2367     
2368  log.critical(s)
2369     
2370# -----------------------------------------------------------------------------
2371
2372def mkargstr(args, textwidth, argstr = '', level=0):
2373  """ Generate a string containing first textwidth characters of arguments.
2374
2375  USAGE:
2376    mkargstr(args, textwidth, argstr = '', level=0)
2377
2378  DESCRIPTION:
2379    Exactly the same as str(args) possibly followed by truncation,
2380    but faster if args is huge.
2381  """
2382
2383  if level > 10:
2384      # Protect against circular structures
2385      return '...'
2386 
2387  WasTruncated = 0
2388
2389  if not isinstance(args, (tuple, list, dict)):
2390    if isinstance(args, basestring):
2391      argstr = argstr + "'"+str(args)+"'"
2392    else:
2393      # Truncate large numeric arrays before using str()
2394      if isinstance(args, num.ndarray):
2395#        if len(args.flat) > textwidth: 
2396#        Changed by Duncan and Nick 21/2/07 .flat has problems with
2397#        non-contigous arrays and ravel is equal to .flat except it
2398#        can work with non-contiguous  arrays
2399        if len(num.ravel(args)) > textwidth:
2400          args = 'Array: ' + str(args.shape)
2401
2402      argstr = argstr + str(args)
2403  else:
2404    if isinstance(args, dict):
2405      argstr = argstr + "{"
2406      for key in args.keys():
2407        argstr = argstr + mkargstr(key, textwidth, level=level+1) + ": " + \
2408                 mkargstr(args[key], textwidth, level=level+1) + ", "
2409        if len(argstr) > textwidth:
2410          WasTruncated = 1
2411          break
2412      argstr = argstr[:-2]  # Strip off trailing comma     
2413      argstr = argstr + "}"
2414
2415    else:
2416      if isinstance(args, tuple):
2417        lc = '('
2418        rc = ')'
2419      else:
2420        lc = '['
2421        rc = ']'
2422      argstr = argstr + lc
2423      for arg in args:
2424        argstr = argstr + mkargstr(arg, textwidth, level=level+1) + ', '
2425        if len(argstr) > textwidth:
2426          WasTruncated = 1
2427          break
2428
2429      # Strip off trailing comma and space unless singleton tuple
2430      #
2431      if isinstance(args, tuple) and len(args) == 1:
2432        argstr = argstr[:-1]   
2433      else:
2434        argstr = argstr[:-2]
2435      argstr = argstr + rc
2436
2437  if len(argstr) > textwidth:
2438    WasTruncated = 1
2439
2440  if WasTruncated:
2441    argstr = argstr[:textwidth]+'...'
2442  return(argstr)
2443
2444# -----------------------------------------------------------------------------
2445
2446def test_OK(msg):
2447  """Print OK msg if test is OK.
2448 
2449  USAGE
2450    test_OK(message)
2451  """
2452
2453  import string
2454   
2455  log.critical(string.ljust(msg, textwidth4) + ' - OK' )
2456 
2457  #raise StandardError
2458 
2459# -----------------------------------------------------------------------------
2460
2461def test_error(msg):
2462  """Print error if test fails.
2463 
2464  USAGE
2465    test_error(message)
2466  """
2467 
2468  log.critical('ERROR (caching.test): %s' % msg)
2469  log.critical('Please send this code example and output to ')
2470  log.critical('Ole.Nielsen@anu.edu.au')
2471  log.critical()
2472  log.critical()
2473 
2474  raise StandardError
2475
2476#-------------------------------------------------------------
2477if __name__ == "__main__":
2478  pass
Note: See TracBrowser for help on using the repository browser.