source: anuga_core/source/anuga/caching/caching.py @ 5860

Last change on this file since 5860 was 5860, checked in by ole, 16 years ago

Thought of testing hash-collision and found issue with comparison
This is now fixed and tests verified.

File size: 72.1 KB
Line 
1# =============================================================================
2# caching.py - Supervised caching of function results.
3# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4# Australian National University (1999-2003)
5# Geoscience Australia (2003-present)
6#
7#    This program is free software; you can redistribute it and/or modify
8#    it under the terms of the GNU General Public License as published by
9#    the Free Software Foundation; either version 2 of the License, or
10#    (at your option) any later version.
11#
12#    This program is distributed in the hope that it will be useful,
13#    but WITHOUT ANY WARRANTY; without even the implied warranty of
14#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15#    GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16#    for more details.
17#
18#    You should have received a copy of the GNU General Public License
19#    along with this program; if not, write to the Free Software
20#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
21#
22#
23# Contact address: Ole.Nielsen@ga.gov.au
24#
25# Version 1.5.6 February 2002
26# =============================================================================
27 
28"""Module caching.py - Supervised caching of function results.
29
30Public functions:
31
32cache(func,args) -- Cache values returned from func given args.
33cachestat() --      Reports statistics about cache hits and time saved.
34test() --       Conducts a basic test of the caching functionality.
35
36See doc strings of individual functions for detailed documentation.
37"""
38
39# -----------------------------------------------------------------------------
40# Initialisation code
41
42# Determine platform
43#
44from os import getenv
45import types
46
47import os
48if os.name in ['nt', 'dos', 'win32', 'what else?']:
49  unix = 0
50else:
51  unix = 1
52
53cache_dir = '.python_cache'
54# Make default caching directory name
55#
56if unix:
57  homedir = getenv('INUNDATIONHOME')
58  if homedir is None:
59    homedir = '~'
60  else:
61    homedir = homedir + os.sep + '.cache'
62    # Since homedir will be a group area, individually label the caches
63    user = getenv('LOGNAME')
64    if user is not None:
65      cache_dir += '_' + user
66   
67  CR = '\n'
68else:
69  homedir = 'c:'
70  CR = '\r\n'  #FIXME: Not tested under windows
71 
72cachedir = homedir + os.sep + cache_dir + os.sep
73
74# -----------------------------------------------------------------------------
75# Options directory with default values - to be set by user
76#
77
78options = { 
79  'cachedir': cachedir,  # Default cache directory
80  'maxfiles': 1000000,   # Maximum number of cached files
81  'savestat': 1,         # Log caching info to stats file
82  'verbose': 1,          # Write messages to standard output
83  'bin': 1,              # Use binary format (more efficient)
84  'compression': 1,      # Use zlib compression
85  'bytecode': 0,         # Recompute if bytecode has changed
86  'expire': 0            # Automatically remove files that have been accessed
87                         # least recently
88}
89
90# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
91
92def set_option(key, value):
93  """Function to set values in the options directory.
94
95  USAGE:
96    set_option(key, value)
97
98  ARGUMENTS:
99    key --   Key in options dictionary. (Required)
100    value -- New value for key. (Required)
101
102  DESCRIPTION:
103    Function to set values in the options directory.
104    Raises an exception if key is not in options.
105  """
106
107  if options.has_key(key):
108    options[key] = value
109  else:
110    raise KeyError(key)  # Key not found, raise an exception
111
112# -----------------------------------------------------------------------------
113# Function cache - the main routine
114
115def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
116          verbose=None, compression=None, evaluate=0, test=0, clear=0,
117          return_filename=0):
118  """Supervised caching of function results.
119
120  USAGE:
121    result = cache(func, args, kwargs, dependencies, cachedir, verbose,
122                   compression, evaluate, test, return_filename)
123
124  ARGUMENTS:
125    func --            Function object (Required)
126    args --            Arguments to func (Default: ())
127    kwargs --          Keyword arguments to func (Default: {})   
128    dependencies --    Filenames that func depends on (Default: None)
129    cachedir --        Directory for cache files (Default: options['cachedir'])
130    verbose --         Flag verbose output to stdout
131                       (Default: options['verbose'])
132    compression --     Flag zlib compression (Default: options['compression'])
133    evaluate --        Flag forced evaluation of func (Default: 0)
134    test --            Flag test for cached results (Default: 0)
135    clear --           Flag delete cached results (Default: 0)   
136    return_filename -- Flag return of cache filename (Default: 0)   
137
138  DESCRIPTION:
139    A Python function call of the form
140
141      result = func(arg1,...,argn)
142
143    can be replaced by
144
145      from caching import cache
146      result = cache(func,(arg1,...,argn))
147
148  The latter form returns the same output as the former but reuses cached
149  results if the function has been computed previously in the same context.
150  'result' and the arguments can be simple types, tuples, list, dictionaries or
151  objects, but not unhashable types such as functions or open file objects.
152  The function 'func' may be a member function of an object or a module.
153
154  This type of caching is particularly useful for computationally intensive
155  functions with few frequently used combinations of input arguments. Note that
156  if the inputs or output are very large caching might not save time because
157  disc access may dominate the execution time.
158
159  If the function definition changes after a result has been cached it will be
160  detected by examining the functions bytecode (co_code, co_consts,
161  func_defualts, co_argcount) and it will be recomputed.
162
163  LIMITATIONS:
164    1 Caching uses the apply function and will work with anything that can be
165      pickled, so any limitation in apply or pickle extends to caching.
166    2 A function to be cached should not depend on global variables
167      as wrong results may occur if globals are changed after a result has
168      been cached.
169
170  -----------------------------------------------------------------------------
171  Additional functionality:
172
173  Keyword args
174    Keyword arguments (kwargs) can be added as a dictionary of keyword: value
175    pairs, following the syntax of the built-in function apply().
176    A Python function call of the form
177   
178      result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)   
179
180    is then cached as follows
181
182      from caching import cache
183      result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
184   
185    The default value of kwargs is {} 
186
187  Explicit dependencies:
188    The call
189      cache(func,(arg1,...,argn),dependencies = <list of filenames>)
190    Checks the size, creation time and modification time of each listed file.
191    If any file has changed the function is recomputed and the results stored
192    again.
193
194  Specify caching directory:
195    The call
196      cache(func,(arg1,...,argn), cachedir = <cachedir>)
197    designates <cachedir> where cached data are stored. Use ~ to indicate users
198    home directory - not $HOME. The default is ~/.python_cache on a UNIX
199    platform and c:/.python_cache on a Win platform.
200
201  Silent operation:
202    The call
203      cache(func,(arg1,...,argn),verbose=0)
204    suppresses messages to standard output.
205
206  Compression:
207    The call
208      cache(func,(arg1,...,argn),compression=0)
209    disables compression. (Default: compression=1). If the requested compressed
210    or uncompressed file is not there, it'll try the other version.
211
212  Forced evaluation:
213    The call
214      cache(func,(arg1,...,argn),evaluate=1)
215    forces the function to evaluate even though cached data may exist.
216
217  Testing for presence of cached result:
218    The call
219      cache(func,(arg1,...,argn),test=1)
220    retrieves cached result if it exists, otherwise None. The function will not
221    be evaluated. If both evaluate and test are switched on, evaluate takes
222    precedence.
223    ??NOTE: In case of hash collisions, this may return the wrong result as
224    ??it only checks if *a* cached result is present.
225   
226  Obtain cache filenames:
227    The call   
228      cache(func,(arg1,...,argn),return_filename=1)
229    returns the hashed base filename under which this function and its
230    arguments would be cached
231
232  Clearing cached results:
233    The call
234      cache(func,'clear')
235    clears all cached data for 'func' and
236      cache('clear')
237    clears all cached data.
238 
239    NOTE: The string 'clear' can be passed an *argument* to func using
240      cache(func,('clear',)) or cache(func,tuple(['clear'])).
241
242    New form of clear:
243      cache(func,(arg1,...,argn),clear=1)
244    clears cached data for particular combination func and args
245     
246  """
247
248  # Imports and input checks
249  #
250  import types, time, string
251
252  if not cachedir:
253    cachedir = options['cachedir']
254
255  if verbose == None:  # Do NOT write 'if not verbose:', it could be zero.
256    verbose = options['verbose']
257
258  if compression == None:  # Do NOT write 'if not compression:',
259                           # it could be zero.
260    compression = options['compression']
261
262  # Create cache directory if needed
263  CD = checkdir(cachedir,verbose)
264
265  # Handle the case cache('clear')
266  if type(func) == types.StringType:
267    if string.lower(func) == 'clear':
268      clear_cache(CD,verbose=verbose)
269      return
270
271  # Handle the case cache(func, 'clear')
272  if type(args) == types.StringType:
273    if string.lower(args) == 'clear':
274      clear_cache(CD,func,verbose=verbose)
275      return
276
277  # Force singleton arg into a tuple
278  if type(args) != types.TupleType:
279    args = tuple([args])
280 
281  # Check that kwargs is a dictionary
282  if type(kwargs) != types.DictType:
283    raise TypeError   
284   
285  # Hash arguments (and keyword args) to integer
286  arghash = myhash((args, kwargs))
287 
288  # Get sizes and timestamps for files listed in dependencies.
289  # Force singletons into a tuple.
290  if dependencies and type(dependencies) != types.TupleType \
291                  and type(dependencies) != types.ListType:
292    dependencies = tuple([dependencies])
293  deps = get_depstats(dependencies)
294
295  # Extract function name from func object
296  funcname = get_funcname(func)
297
298  # Create cache filename
299  FN = funcname+'['+`arghash`+']'  # The symbol '(' does not work under unix
300
301  if return_filename:
302    return(FN)
303
304  if clear:
305    for file_type in file_types:
306      file_name = CD+FN+'_'+file_type
307      for fn in [file_name, file_name + '.z']:
308        if os.access(fn, os.F_OK):             
309          if unix:
310            os.remove(fn)
311          else:
312            # FIXME: os.remove doesn't work under windows       
313            os.system('del '+fn)
314          if verbose is True:
315            print 'MESSAGE (caching): File %s deleted' %fn
316        ##else:
317        ##  print '%s was not accessed' %fn
318    return None
319
320
321  #-------------------------------------------------------------------       
322 
323  # Check if previous computation has been cached
324  if evaluate:
325    Retrieved = None  # Force evaluation of func regardless of caching status.
326    reason = 5
327  else:
328    T, FN, Retrieved, reason, comptime, loadtime, compressed = \
329        CacheLookup(CD, FN, func, 
330                    args, kwargs, 
331                    deps, 
332                    verbose, 
333                    compression,
334                    dependencies)
335
336  if not Retrieved:
337    if test:  # Do not attempt to evaluate function
338      T = None
339    else:  # Evaluate function and save to cache
340      if verbose:
341       
342        msg1(funcname, args, kwargs,reason)
343
344      # Remove expired files automatically
345      if options['expire']:
346        DeleteOldFiles(CD,verbose)
347       
348      # Save args before function is evaluated in case
349      # they are modified by function
350      save_args_to_cache(CD,FN,args,kwargs,compression)
351
352      # Execute and time function with supplied arguments
353      t0 = time.time()
354      T = apply(func,args,kwargs)
355      #comptime = round(time.time()-t0)
356      comptime = time.time()-t0
357
358      if verbose:
359        msg2(funcname,args,kwargs,comptime,reason)
360
361      # Save results and estimated loading time to cache
362      loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
363                                       funcname, dependencies, compression)
364      if verbose:
365        msg3(loadtime, CD, FN, deps, compression)
366      compressed = compression
367
368  if options['savestat'] and (not test or Retrieved):
369  ##if options['savestat']:
370    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
371
372  return(T)  # Return results in all cases
373
374# -----------------------------------------------------------------------------
375
376def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
377  """Generate statistics of caching efficiency.
378
379  USAGE:
380    cachestat(sortidx, period, showuser, cachedir)
381
382  ARGUMENTS:
383    sortidx --  Index of field by which lists are (default: 4)
384                Legal values are
385                 0: 'Name'
386                 1: 'Hits'
387                 2: 'CPU'
388                 3: 'Time Saved'
389                 4: 'Gain(%)'
390                 5: 'Size'
391    period --   If set to -1 all available caching history is used.
392                If set 0 only the current month is used (default -1).
393    showuser -- Flag for additional table showing user statistics
394                (default: None).
395    cachedir -- Directory for cache files (default: options['cachedir']).
396
397  DESCRIPTION:
398    Logged caching statistics is converted into summaries of the form
399    --------------------------------------------------------------------------
400    Function Name   Hits   Exec(s)  Cache(s)  Saved(s)   Gain(%)      Size
401    --------------------------------------------------------------------------
402  """
403
404  __cachestat(sortidx, period, showuser, cachedir)
405  return
406
407# -----------------------------------------------------------------------------
408
409#Has mostly been moved to proper unit test
410def test(cachedir=None,verbose=0,compression=None):
411  """Test the functionality of caching.
412
413  USAGE:
414    test(verbose)
415
416  ARGUMENTS:
417    verbose --     Flag whether caching will output its statistics (default=0)
418    cachedir --    Directory for cache files (Default: options['cachedir'])
419    compression -- Flag zlib compression (Default: options['compression'])
420  """
421   
422  import string, time
423
424  # Initialise
425  #
426  import caching
427  reload(caching)
428
429  if not cachedir:
430    cachedir = options['cachedir']
431
432  if verbose is None:  # Do NOT write 'if not verbose:', it could be zero.
433    verbose = options['verbose']
434 
435  if compression == None:  # Do NOT write 'if not compression:',
436                           # it could be zero.
437    compression = options['compression']
438  else:
439    try:
440      set_option('compression', compression)
441    except:
442      test_error('Set option failed')     
443
444  try:
445    import zlib
446  except:
447    print
448    print '*** Could not find zlib, default to no-compression      ***'
449    print '*** Installing zlib will improve performance of caching ***'
450    print
451    compression = 0       
452    set_option('compression', compression)   
453 
454  print 
455  print_header_box('Testing caching module - please stand by')
456  print   
457
458  # Define a test function to be cached
459  #
460  def f(a,b,c,N,x=0,y='abcdefg'):
461    """f(a,b,c,N)
462       Do something time consuming and produce a complex result.
463    """
464
465    import string
466
467    B = []
468    for n in range(N):
469      s = str(n+2.0/(n + 4.0))+'.a'*10
470      B.append((a,b,c,s,n,x,y))
471    return(B)
472   
473  # Check that default cachedir is OK
474  #     
475  CD = checkdir(cachedir,verbose)   
476   
477   
478  # Make a dependency file
479  #   
480  try:
481    DepFN = CD + 'testfile.tmp'
482    DepFN_wildcard = CD + 'test*.tmp'
483    Depfile = open(DepFN,'w')
484    Depfile.write('We are the knights who say NI!')
485    Depfile.close()
486    test_OK('Wrote file %s' %DepFN)
487  except:
488    test_error('Could not open file %s for writing - check your environment' \
489               % DepFN)
490
491  # Check set_option (and switch stats off
492  #   
493  try:
494    set_option('savestat',0)
495    assert(options['savestat'] == 0)
496    test_OK('Set option')
497  except:
498    test_error('Set option failed')   
499   
500  # Make some test input arguments
501  #
502  N = 5000  #Make N fairly small here
503
504  a = [1,2]
505  b = ('Thou shalt count the number three',4)
506  c = {'Five is right out': 6, (7,8): 9}
507  x = 3
508  y = 'holy hand granate'
509
510  # Test caching
511  #
512  if compression:
513    comprange = 2
514  else:
515    comprange = 1
516
517  for comp in range(comprange):
518 
519    # Evaluate and store
520    #
521    try:
522      T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
523                         verbose=verbose, compression=comp)
524      if comp:                   
525        test_OK('Caching evaluation with compression')
526      else:     
527        test_OK('Caching evaluation without compression')     
528    except:
529      if comp:
530        test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
531      else:
532        test_error('Caching evaluation failed - try caching.test(verbose=1)')
533
534    # Retrieve
535    #                           
536    try:                         
537      T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
538                         compression=comp) 
539
540      if comp:                   
541        test_OK('Caching retrieval with compression')
542      else:     
543        test_OK('Caching retrieval without compression')     
544    except:
545      if comp:
546        test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
547      else:                                     
548        test_error('Caching retrieval failed - try caching.test(verbose=1)')
549
550    # Reference result
551    #   
552    T3 = f(a,b,c,N,x=x,y=y)  # Compute without caching
553   
554    if T1 == T2 and T2 == T3:
555      if comp:
556        test_OK('Basic caching functionality (with compression)')
557      else:
558        test_OK('Basic caching functionality (without compression)')
559    else:
560      test_error('Cached result does not match computed result')
561
562
563  # Test return_filename
564  #   
565  try:
566    FN = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
567                       return_filename=1)   
568    assert(FN[:2] == 'f[')
569    test_OK('Return of cache filename')
570  except:
571    test_error('Return of cache filename failed')
572
573  # Test existence of cachefiles
574 
575  try:
576    (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
577    (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
578    (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
579    test_OK('Presence of cache files')
580    datafile.close()
581    argsfile.close()
582    admfile.close()
583  except:
584    test_error('Expected cache files did not exist') 
585             
586  # Test 'test' function when cache is present
587  #     
588  try:
589    #T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
590    #                   evaluate=1) 
591    T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
592    assert(T1 == T4)
593
594    test_OK("Option 'test' when cache file present")
595  except:
596    test_error("Option 'test' when cache file present failed")     
597
598  # Test that 'clear' works
599  #
600  #try:
601  #  caching.cache(f,'clear',verbose=verbose)
602  #  test_OK('Clearing of cache files')
603  #except:
604  #  test_error('Clear does not work')
605  try:
606    caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)   
607    test_OK('Clearing of cache files')
608  except:
609    test_error('Clear does not work') 
610
611 
612
613  # Test 'test' function when cache is absent
614  #     
615  try:
616    T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
617    assert(T4 is None)
618    test_OK("Option 'test' when cache absent")
619  except:
620    test_error("Option 'test' when cache absent failed")     
621         
622  # Test dependencies
623  #
624  T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
625                       dependencies=DepFN) 
626  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
627                       dependencies=DepFN)                     
628                       
629  if T1 == T2:
630    test_OK('Basic dependencies functionality')
631  else:
632    test_error('Dependencies do not work')
633
634  # Test basic wildcard dependency
635  #
636  T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
637                       dependencies=DepFN_wildcard)                     
638   
639  if T1 == T3:
640    test_OK('Basic dependencies with wildcard functionality')
641  else:
642    test_error('Dependencies with wildcards do not work')
643
644
645  # Test that changed timestamp in dependencies triggers recomputation
646 
647  # Modify dependency file
648  Depfile = open(DepFN,'a')
649  Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
650  Depfile.close()
651 
652  T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
653                       dependencies=DepFN, test = 1)                     
654 
655  if T3 is None:
656    test_OK('Changed dependencies recognised')
657  else:
658    test_error('Changed dependencies not recognised')   
659 
660  # Test recomputation when dependencies have changed
661  #
662  T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
663                       dependencies=DepFN)                       
664  if T1 == T3:
665    test_OK('Recomputed value with changed dependencies')
666  else:
667    test_error('Recomputed value with changed dependencies failed')
668
669  # Performance test (with statistics)
670  # Don't really rely on this as it will depend on specific computer.
671  #
672
673  set_option('savestat',1)
674
675  N = 20*N   #Should be large on fast computers...
676  tt = time.time()
677  T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
678  t1 = time.time() - tt
679 
680  tt = time.time()
681  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
682  t2 = time.time() - tt
683 
684  if T1 == T2:
685    if t1 > t2:
686      test_OK('Performance test: relative time saved = %s pct' \
687              %str(round((t1-t2)*100/t1,2)))
688    #else:
689    #  print 'WARNING: Performance a bit low - this could be specific to current platform'
690  else:       
691    test_error('Basic caching failed for new problem')
692           
693  # Test presence of statistics file
694  #
695  try: 
696    DIRLIST = os.listdir(CD)
697    SF = []
698    for FN in DIRLIST:
699      if string.find(FN,statsfile) >= 0:
700        fid = open(CD+FN,'r')
701        fid.close()
702    test_OK('Statistics files present') 
703  except:
704    test_OK('Statistics files cannot be opened')         
705     
706  print_header_box('Show sample output of the caching function:')
707 
708  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
709  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
710  T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
711 
712  print_header_box('Show sample output of cachestat():')
713  if unix:
714    cachestat()   
715  else:
716    try:
717      import time
718      t = time.strptime('2030','%Y')
719      cachestat()
720    except: 
721      print 'caching.cachestat() does not work here, because it'
722      print 'relies on time.strptime() which is unavailable in Windows'
723     
724  print
725  test_OK('Caching self test completed')   
726     
727           
728  # Test setoption (not yet implemented)
729  #
730
731 
732#==============================================================================
733# Auxiliary functions
734#==============================================================================
735
736# Import pickler
737# cPickle is used by functions mysave, myload, and compare
738#
739import cPickle  # 10 to 100 times faster than pickle
740pickler = cPickle
741
742# Local immutable constants
743#
744comp_level = 1              # Compression level for zlib.
745                            # comp_level = 1 works well.
746textwidth1 = 16             # Text width of key fields in report forms.
747#textwidth2 = 132            # Maximal width of textual representation of
748textwidth2 = 300            # Maximal width of textual representation of
749                            # arguments.
750textwidth3 = 16             # Initial width of separation lines. Is modified.
751textwidth4 = 50             # Text width in test_OK()
752statsfile  = '.cache_stat'  # Basefilename for cached statistics.
753                            # It will reside in the chosen cache directory.
754
755file_types = ['Result',     # File name extension for cached function results.
756              'Args',       # File name extension for stored function args.
757              'Admin']      # File name extension for administrative info.
758
759Reason_msg = ['OK',         # Verbose reasons for recomputation
760              'No cached result', 
761              'Dependencies have changed', 
762              'Arguments have changed',
763              'Bytecode has changed',
764              'Recomputation was requested by caller',
765              'Cached file was unreadable']             
766             
767# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
768
769def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, 
770                dependencies):
771  """Determine whether cached result exists and return info.
772
773  USAGE:
774    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \ 
775    CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
776                dependencies)
777
778  INPUT ARGUMENTS:
779    CD --            Cache Directory
780    FN --            Suggested cache file name
781    func --          Function object
782    args --          Tuple of arguments
783    kwargs --        Dictionary of keyword arguments   
784    deps --          Dependencies time stamps
785    verbose --       Flag text output
786    compression --   Flag zlib compression
787    dependencies --  Given list of dependencies
788   
789  OUTPUT ARGUMENTS:
790    T --             Cached result if present otherwise None
791    FN --            File name under which new results must be saved
792    Retrieved --     True if a valid cached result was found
793    reason --        0: OK (if Retrieved),
794                     1: No cached result,
795                     2: Dependencies have changed,
796                     3: Arguments have changed
797                     4: Bytecode has changed
798                     5: Recomputation was forced
799                     6: Unreadable file
800    comptime --      Number of seconds it took to computed cachged result
801    loadtime --      Number of seconds it took to load cached result
802    compressed --    Flag (0,1) if cached results were compressed or not
803
804  DESCRIPTION:
805    Determine if cached result exists as follows:
806    Load in saved arguments and bytecode stored under hashed filename.
807    If they are identical to current arguments and bytecode and if dependencies
808    have not changed their time stamp, then return cached result.
809
810    Otherwise return filename under which new results should be cached.
811    Hash collisions are handled recursively by calling CacheLookup again with a
812    modified filename.
813  """
814
815  import time, string, types
816
817  # Assess whether cached result exists - compressed or not.
818  #
819  if verbose:
820    print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
821           %(CD+FN, file_types[0], file_types[1], file_types[2])
822  (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
823  (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
824  (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
825
826  if verbose is True and deps is not None:
827    print 'Caching: Dependencies are', deps.keys()
828
829  if not (argsfile and datafile and admfile) or \
830     not (compressed0 == compressed1 and compressed0 == compressed2):
831    # Cached result does not exist or files were compressed differently
832    #
833    # This will ensure that evaluation will take place unless all files are
834    # present.
835
836    reason = 1
837    return(None,FN,None,reason,None,None,None) #Recompute using same filename
838
839  compressed = compressed0  # Remember if compressed files were actually used
840  datafile.close()
841
842  # Retrieve arguments and adm. info
843  #
844  R, reason = myload(argsfile, compressed)  # The original arguments
845  argsfile.close()
846   
847  if reason > 0:
848      # Recompute using same filename   
849      return(None, FN, None, reason, None, None, None)
850  else:   
851      (argsref, kwargsref) = R
852
853  R, reason = myload(admfile, compressed)
854  admfile.close() 
855
856  if reason > 0:
857    return(None,FN,None,reason,None,None,None) #Recompute using same filename
858
859 
860  depsref  = R[0]  # Dependency statistics
861  comptime = R[1]  # The computation time
862  coderef  = R[2]  # The byte code
863  funcname = R[3]  # The function name
864
865  # Check if dependencies have changed
866  #
867  if dependencies and not compare(depsref, deps):
868    if verbose:
869      print 'MESSAGE (caching.py): Dependencies', dependencies, \
870            'have changed - recomputing'
871    # Don't use cached file - recompute
872    reason = 2
873    return(None, FN, None, reason, None, None, None)
874
875  # Get bytecode from func
876  #
877  bytecode = get_bytecode(func)
878
879  #print compare(argsref,args),   
880  #print compare(kwargsref,kwargs),
881  #print compare(bytecode,coderef)
882
883  # Check if arguments or bytecode have changed
884  if compare(argsref, args) and compare(kwargsref, kwargs) and \
885     (not options['bytecode'] or compare(bytecode, coderef)):
886
887    # Arguments and dependencies match. Get cached results
888    T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
889    if reason > 0:
890        # Recompute using same FN     
891        return(None, FN, None, reason, None, None, None)
892
893    Retrieved = 1
894    reason = 0
895
896    if verbose:
897      msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
898
899      if loadtime >= comptime:
900        print 'WARNING (caching.py): Caching did not yield any gain.'
901        print '                      Consider executing function ',
902        print '('+funcname+') without caching.'
903  else:
904
905    # Non matching arguments or bytecodes signify a hash-collision.
906    # This is resolved by recursive search of cache filenames
907    # until either a matching or an unused filename is found.
908    #
909    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
910        CacheLookup(CD, FN+'x', func, args, kwargs, deps, 
911                    verbose, compression, dependencies)
912
913    # DEBUGGING
914    # if not Retrieved:
915    #   print 'Arguments did not match'
916    # else:
917    #   print 'Match found !'
918   
919    # The real reason is that args or bytecodes have changed.
920    # Not that the recursive seach has found an unused filename
921    if not Retrieved:
922      if not compare(bytecode, coderef):
923        reason = 4 # Bytecode has changed
924      else:   
925        reason = 3 # Arguments have changed
926       
927   
928  return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
929
930# -----------------------------------------------------------------------------
931
932def clear_cache(CD,func=None, verbose=None):
933  """Clear cache for func.
934
935  USAGE:
936     clear(CD, func, verbose)
937
938  ARGUMENTS:
939     CD --       Caching directory (required)
940     func --     Function object (default: None)
941     verbose --  Flag verbose output (default: None)
942
943  DESCRIPTION:
944
945    If func == None, clear everything,
946    otherwise clear only files pertaining to func.
947  """
948
949  import os, re
950   
951  if CD[-1] != os.sep:
952    CD = CD+os.sep
953 
954  if verbose == None:
955    verbose = options['verbose']
956
957  # FIXME: Windows version needs to be tested
958
959  if func:
960    funcname = get_funcname(func)
961    if verbose:
962      print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
963
964    file_names = os.listdir(CD)
965    for file_name in file_names:
966      #RE = re.search('^' + funcname,file_name)  #Inefficient
967      #if RE:
968      if file_name[:len(funcname)] == funcname:
969        if unix:
970          os.remove(CD+file_name)
971        else:
972          os.system('del '+CD+file_name)
973          # FIXME: os.remove doesn't work under windows
974  else:
975    file_names = os.listdir(CD)
976    if len(file_names) > 0:
977      if verbose:
978        print 'MESSAGE (caching.py): Remove the following files:'
979        for file_name in file_names:
980            print file_name
981
982        A = raw_input('Delete (Y/N)[N] ?')
983      else:
984        A = 'Y' 
985       
986      if A == 'Y' or A == 'y':
987        for file_name in file_names:
988          if unix:
989            os.remove(CD+file_name)
990          else:
991            os.system('del '+CD+file_name)
992            # FIXME: os.remove doesn't work under windows
993          #exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
994
995# -----------------------------------------------------------------------------
996
997def DeleteOldFiles(CD,verbose=None):
998  """Remove expired files
999
1000  USAGE:
1001    DeleteOldFiles(CD,verbose=None)
1002  """
1003
1004  if verbose == None:
1005    verbose = options['verbose']
1006
1007  maxfiles = options['maxfiles']
1008
1009  # FIXME: Windows version
1010
1011  import os
1012  block = 1000  # How many files to delete per invokation
1013  Files = os.listdir(CD)
1014  numfiles = len(Files)
1015  if not unix: return  # FIXME: Windows case ?
1016
1017  if numfiles > maxfiles:
1018    delfiles = numfiles-maxfiles+block
1019    if verbose:
1020      print 'Deleting '+`delfiles`+' expired files:'
1021      os.system('ls -lur '+CD+'* | head -' + `delfiles`)            # List them
1022    os.system('ls -ur '+CD+'* | head -' + `delfiles` + ' | xargs /bin/rm')
1023                                                                  # Delete them
1024    # FIXME: Replace this with os.listdir and os.remove
1025
1026# -----------------------------------------------------------------------------
1027
1028def save_args_to_cache(CD, FN, args, kwargs, compression):
1029  """Save arguments to cache
1030
1031  USAGE:
1032    save_args_to_cache(CD,FN,args,kwargs,compression)
1033  """
1034
1035  import time, os, sys, types
1036
1037  (argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1038
1039  if argsfile is None:
1040    msg = 'ERROR (caching): Could not open argsfile for writing: %s' %FN
1041    raise IOError, msg
1042
1043  mysave((args,kwargs),argsfile,compression)  # Save args and kwargs to cache
1044  argsfile.close()
1045
1046  # Change access rights if possible
1047  #
1048  #if unix:
1049  #  try:
1050  #    exitcode=os.system('chmod 666 '+argsfile.name)
1051  #  except:
1052  #    pass
1053  #else:
1054  #  pass  # FIXME: Take care of access rights under Windows
1055
1056  return
1057
1058# -----------------------------------------------------------------------------
1059
1060def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1061                          dependencies, compression):
1062  """Save computed results T and admin info to cache
1063
1064  USAGE:
1065    save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1066                          dependencies, compression)
1067  """
1068
1069  import time, os, sys, types
1070
1071  (datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1072  (admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1073
1074  if not datafile:
1075    if verbose:
1076      print 'ERROR (caching): Could not open %s' %datafile.name
1077    raise IOError
1078
1079  if not admfile:
1080    if verbose:
1081      print 'ERROR (caching): Could not open %s' %admfile.name
1082    raise IOError
1083
1084  t0 = time.time()
1085
1086  mysave(T,datafile,compression)  # Save data to cache
1087  datafile.close()
1088  #savetime = round(time.time()-t0,2)
1089  savetime = time.time()-t0 
1090
1091  bytecode = get_bytecode(func)  # Get bytecode from function object
1092  admtup = (deps, comptime, bytecode, funcname)  # Gather admin info
1093
1094  mysave(admtup,admfile,compression)  # Save admin info to cache
1095  admfile.close()
1096
1097  # Change access rights if possible
1098  #
1099  #if unix:
1100  #  try:
1101  #    exitcode=os.system('chmod 666 '+datafile.name)
1102  #    exitcode=os.system('chmod 666 '+admfile.name)
1103  #  except:
1104  #    pass
1105  #else:
1106  #  pass  # FIXME: Take care of access rights under Windows
1107
1108  return(savetime)
1109
1110# -----------------------------------------------------------------------------
1111
1112def load_from_cache(CD,FN,compression):
1113  """Load previously cached data from file FN
1114
1115  USAGE:
1116    load_from_cache(CD,FN,compression)
1117  """
1118
1119  import time
1120
1121  (datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1122  t0 = time.time()
1123  T, reason = myload(datafile,compressed)
1124
1125  loadtime = time.time()-t0
1126  datafile.close() 
1127
1128  return T, loadtime, compressed, reason
1129
1130# -----------------------------------------------------------------------------
1131
1132def myopen(FN,mode,compression=1):
1133  """Open file FN using given mode
1134
1135  USAGE:
1136    myopen(FN,mode,compression=1)
1137
1138  ARGUMENTS:
1139    FN --           File name to be opened
1140    mode --         Open mode (as in open)
1141    compression --  Flag zlib compression
1142
1143  DESCRIPTION:
1144     if compression
1145       Attempt first to open FN + '.z'
1146       If this fails try to open FN
1147     else do the opposite
1148     Return file handle plus info about whether it was compressed or not.
1149  """
1150
1151  import string
1152
1153  # Determine if file exists already (if writing was requested)
1154  # This info is only used to determine if access modes should be set
1155  #
1156  if 'w' in mode or 'a' in mode:
1157    try:
1158      file = open(FN+'.z','r')
1159      file.close()
1160      new_file = 0
1161    except:
1162      try:
1163        file = open(FN,'r') 
1164        file.close()
1165        new_file = 0
1166      except:
1167        new_file = 1
1168  else:
1169    new_file = 0 #Assume it exists if mode was not 'w'
1170 
1171
1172  compressed = 0
1173  if compression:
1174    try:
1175      file = open(FN+'.z',mode)
1176      compressed = 1
1177    except:
1178      try:
1179        file = open(FN,mode)
1180      except:
1181        file = None
1182  else:
1183    try:
1184      file = open(FN,mode)
1185    except:
1186      try:
1187        file = open(FN+'.z',mode)
1188        compressed = 1
1189      except:
1190        file = None
1191
1192  # Now set access rights if it is a new file
1193  #
1194  if file and new_file:
1195    if unix:
1196      exitcode=os.system('chmod 666 '+file.name)
1197    else:
1198      pass  # FIXME: Take care of access rights under Windows
1199
1200  return(file,compressed)
1201
1202# -----------------------------------------------------------------------------
1203
1204def myload(file, compressed):
1205  """Load data from file
1206
1207  USAGE:
1208    myload(file, compressed)
1209  """
1210
1211  reason = 0
1212  try:
1213    if compressed:
1214      import zlib
1215
1216      RsC = file.read()
1217      try:
1218        Rs  = zlib.decompress(RsC)
1219      except:
1220        #  File "./caching.py", line 1032, in load_from_cache
1221        #  T = myload(datafile,compressed)
1222        #  File "./caching.py", line 1124, in myload
1223        #  Rs  = zlib.decompress(RsC)
1224        #  zlib.error: Error -5 while decompressing data
1225        #print 'ERROR (caching): Could not decompress ', file.name
1226        #raise Exception
1227        reason = 6  # Unreadable file
1228        return None, reason 
1229     
1230     
1231      del RsC  # Free up some space
1232      R = pickler.loads(Rs)
1233    else:
1234      try:
1235        R = pickler.load(file)
1236      #except EOFError, e:
1237      except:
1238        #Catch e.g., file with 0 length or corrupted
1239        reason = 6  # Unreadable file
1240        return None, reason
1241     
1242  except MemoryError:
1243    import sys
1244    if options['verbose']:
1245      print 'ERROR (caching): Out of memory while loading %s, aborting' \
1246            %(file.name)
1247
1248    # Raise the error again for now
1249    #
1250    raise MemoryError
1251
1252  return R, reason
1253
1254# -----------------------------------------------------------------------------
1255
1256def mysave(T,file,compression):
1257  """Save data T to file
1258
1259  USAGE:
1260    mysave(T,file,compression)
1261
1262  """
1263
1264  bin = options['bin']
1265
1266  if compression:
1267    try:
1268      import zlib
1269    except:
1270      print
1271      print '*** Could not find zlib ***'
1272      print '*** Try to run caching with compression off ***'
1273      print "*** caching.set_option('compression', 0) ***"
1274      raise Exception
1275     
1276
1277    try:
1278      Ts  = pickler.dumps(T, bin)
1279    except MemoryError:
1280      msg = '****WARNING (caching.py): Could not pickle data for compression.'
1281      msg += ' Try using compression = False'
1282      raise MemoryError, msg
1283    else: 
1284      # Compressed pickling     
1285      TsC = zlib.compress(Ts, comp_level)
1286      file.write(TsC)
1287  else:
1288      # Uncompressed pickling
1289      pickler.dump(T, file, bin)
1290
1291      # FIXME: This may not work on Windoze network drives.
1292      # The error msg is IOError: [Errno 22] Invalid argument
1293      # Testing with small files was OK, though.
1294      # I think this is an OS problem.
1295
1296      # Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1297     
1298# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1299#
1300#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1301#
1302#The following extract is taken from the site http://www.python.org:
1303#
1304#---------------------------------------------------------------------------------------------
1305#exception IOError
1306#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1307#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1308#---------------------------------------------------------------------------------------------
1309#
1310#The error code(s) that accompany exceptions are described at:
1311#http://www.python.org/dev/doc/devel//lib/module-errno.html
1312#
1313#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1314#
1315#exceptions.IOError invalid argument Errno 22
1316       
1317      #try:
1318      #  pickler.dump(T,file,bin)
1319      #except IOError, e:
1320      #  print e
1321      #  msg = 'Could not store to %s, bin=%s' %(file, bin)
1322      #  raise msg
1323     
1324
1325# -----------------------------------------------------------------------------
1326
1327   
1328def myhash(T, ids=None):
1329  """Compute hashed integer from a range of inputs.
1330  If T is not hashable being e.g. a tuple T, myhash will recursively
1331  hash the values individually
1332
1333  USAGE:
1334    myhash(T)
1335
1336  ARGUMENTS:
1337    T -- Anything
1338  """
1339
1340  from types import TupleType, ListType, DictType, InstanceType 
1341  from Numeric import ArrayType, average
1342
1343   
1344  if type(T) in [TupleType, ListType, DictType, InstanceType]: 
1345      # Keep track of unique id's to protect against infinite recursion
1346      if ids is None: ids = []
1347
1348      # Check if T has already been encountered
1349      i = id(T) 
1350 
1351      if i in ids:
1352          return 0 # T has been hashed already     
1353      else:
1354          ids.append(i)
1355   
1356
1357   
1358  # Start hashing 
1359 
1360  # On some architectures None, False and True gets different hash values
1361  if T is None:
1362      return(-1)
1363  if T is False:
1364      return(0)
1365  if T is True:
1366      return(1)
1367
1368  # Get hash values for hashable entries
1369  if type(T) in [TupleType, ListType]:
1370      hvals = []
1371      for t in T:
1372          h = myhash(t, ids)
1373          hvals.append(h)
1374      val = hash(tuple(hvals))
1375  elif type(T) == DictType:
1376      # Make dictionary ordering unique 
1377      I = T.items()
1378      I.sort()   
1379      val = myhash(I, ids)
1380  elif type(T) == ArrayType:
1381      # Use mean value for efficiency 
1382      val = hash(average(T.flat))
1383  elif type(T) == InstanceType:
1384      val = myhash(T.__dict__, ids)
1385  else:
1386      try:
1387          val = hash(T)
1388      except:
1389          val = 1
1390
1391  return(val)
1392
1393
1394
1395def compare(A, B, ids=None):
1396    """Safe comparison of general objects
1397
1398    USAGE:
1399      compare(A,B)
1400
1401    DESCRIPTION:
1402      Return 1 if A and B they are identical, 0 otherwise
1403    """
1404
1405    from types import TupleType, ListType, DictType, InstanceType
1406    from Numeric import ArrayType, alltrue   
1407   
1408    # Keep track of unique id's to protect against infinite recursion
1409    if ids is None: ids = {}
1410
1411    # Check if T has already been encountered
1412    iA = id(A) 
1413    iB = id(B)     
1414   
1415    if ids.has_key((iA, iB)):
1416        # A and B have been compared already
1417        #print 'Found', (iA, iB), A, B
1418        return ids[(iA, iB)]
1419    else:
1420        ids[(iA, iB)] = True
1421   
1422   
1423    # Check if arguments are of same type
1424    if type(A) != type(B):
1425        return False
1426       
1427 
1428    # Compare recursively based on argument type
1429    if type(A) in [TupleType, ListType]:
1430        N = len(A)
1431        if len(B) != N: 
1432            identical = False
1433        else:
1434            identical = True
1435            for i in range(N):
1436                if not compare(A[i], B[i], ids): 
1437                    identical = False; break
1438                   
1439    elif type(A) == DictType:
1440        if len(A) != len(B):
1441            identical = False
1442        else:                       
1443            # Make dictionary ordering unique
1444            a = A.items(); a.sort()   
1445            b = B.items(); b.sort()
1446           
1447            identical = compare(a, b, ids)
1448           
1449    elif type(A) == ArrayType:
1450        # Use element by element comparison
1451        identical = alltrue(A==B)
1452
1453    elif type(A) == InstanceType:
1454        # Take care of special case where elements are instances           
1455        # Base comparison on attributes     
1456        identical = compare(A.__dict__, 
1457                            B.__dict__, 
1458                            ids)
1459    else:       
1460        # Fall back to general code
1461        try:
1462            identical = (A == B)
1463        except:
1464            import pickle
1465            # Use pickle to compare data
1466            # The native pickler must be used
1467            # since the faster cPickle does not
1468            # guarantee a unique translation           
1469            try:
1470                identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
1471            except:
1472                identical = False
1473
1474    # Record result of comparison and return           
1475    ids[(iA, iB)] = identical
1476   
1477    return(identical)
1478
1479   
1480# -----------------------------------------------------------------------------
1481
1482def nospace(s):
1483  """Replace spaces in string s with underscores
1484
1485  USAGE:
1486    nospace(s)
1487
1488  ARGUMENTS:
1489    s -- string
1490  """
1491
1492  import string
1493
1494  newstr = ''
1495  for i in range(len(s)):
1496    if s[i] == ' ':
1497      newstr = newstr+'_'
1498    else:
1499      newstr = newstr+s[i]
1500
1501  return(newstr)
1502
1503# -----------------------------------------------------------------------------
1504
1505def get_funcname(func):
1506  """Retrieve name of function object func (depending on its type)
1507
1508  USAGE:
1509    get_funcname(func)
1510  """
1511
1512  import types, string
1513
1514  if type(func) == types.FunctionType:
1515    funcname = func.func_name
1516  elif type(func) == types.BuiltinFunctionType:
1517    funcname = func.__name__
1518  else:
1519    tab = string.maketrans("<>'","   ")
1520    tmp = string.translate(`func`,tab)
1521    tmp = string.split(tmp)
1522    funcname = string.join(tmp)
1523
1524    # Truncate memory address as in
1525    # class __main__.Dummy at 0x00A915D0
1526    index = funcname.find('at 0x')
1527    if index >= 0:
1528      funcname = funcname[:index+5] # Keep info that there is an address
1529
1530  funcname = nospace(funcname)
1531  return(funcname)
1532
1533# -----------------------------------------------------------------------------
1534
1535def get_bytecode(func):
1536  """ Get bytecode from function object.
1537
1538  USAGE:
1539    get_bytecode(func)
1540  """
1541
1542  import types
1543
1544  if type(func) == types.FunctionType:
1545    bytecode = func.func_code.co_code
1546    consts = func.func_code.co_consts
1547    argcount = func.func_code.co_argcount   
1548    defaults = func.func_defaults     
1549  elif type(func) == types.MethodType:
1550    bytecode = func.im_func.func_code.co_code
1551    consts =  func.im_func.func_code.co_consts
1552    argcount =  func.im_func.func_code.co_argcount   
1553    defaults = func.im_func.func_defaults         
1554  else:
1555    #raise Exception  #Test only
1556    bytecode = None   #Built-in functions are assumed not to change
1557    consts = 0
1558    argcount = 0
1559    defaults = 0
1560
1561  return (bytecode, consts, argcount, defaults)
1562
1563# -----------------------------------------------------------------------------
1564
1565def get_depstats(dependencies):
1566  """ Build dictionary of dependency files and their size, mod. time and ctime.
1567
1568  USAGE:
1569    get_depstats(dependencies):
1570  """
1571
1572  import types
1573
1574  #print 'Caching DEBUG: Dependencies are', dependencies
1575  d = {}
1576  if dependencies:
1577
1578    #Expand any wildcards
1579    import glob
1580    expanded_dependencies = []
1581    for FN in dependencies:
1582      expanded_FN = glob.glob(FN)
1583
1584      if expanded_FN == []:
1585        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1586        raise Exception, errmsg     
1587
1588      expanded_dependencies += expanded_FN
1589
1590   
1591    for FN in expanded_dependencies:
1592      if not type(FN) == types.StringType:
1593        errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1594        errmsg += '                   Dependency given: %s' %FN
1595        raise Exception, errmsg     
1596      if not os.access(FN,os.F_OK):
1597        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1598        raise Exception, errmsg
1599      (size,atime,mtime,ctime) = filestat(FN)
1600
1601      # We don't use atime because that would cause recomputation every time.
1602      # We don't use ctime because that is irrelevant and confusing for users.
1603      d.update({FN : (size,mtime)})
1604
1605  return(d)
1606
1607# -----------------------------------------------------------------------------
1608
1609def filestat(FN):
1610  """A safe wrapper using os.stat to get basic file statistics
1611     The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1612
1613  USAGE:
1614    filestat(FN)
1615
1616  DESCRIPTION:
1617     Must compile Python with
1618     CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1619              configure
1620     as given in section 8.1.1 Large File Support in the Libray Reference
1621  """
1622
1623  import os, time
1624
1625  try:
1626    stats = os.stat(FN)
1627    size  = stats[6]
1628    atime = stats[7]
1629    mtime = stats[8]
1630    ctime = stats[9]
1631  except:
1632
1633    # Hack to get the results anyway (works only on Unix at the moment)
1634    #
1635    print 'Hack to get os.stat when files are too large'
1636
1637    if unix:
1638      tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1639      # Unique filename, FIXME: Use random number
1640
1641      # Get size and access time (atime)
1642      #
1643      exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1644      (size,atime) = get_lsline(tmp)
1645
1646      # Get size and modification time (mtime)
1647      #
1648      exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1649      (size,mtime) = get_lsline(tmp)
1650
1651      # Get size and ctime
1652      #
1653      exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1654      (size,ctime) = get_lsline(tmp)
1655
1656      try:
1657        exitcode=os.system('rm '+tmp)
1658        # FIXME: Gives error if file doesn't exist
1659      except:
1660        pass
1661    else:
1662      pass
1663      raise Exception  # FIXME: Windows case
1664
1665  return(long(size),atime,mtime,ctime)
1666
1667# -----------------------------------------------------------------------------
1668
1669def get_lsline(FN):
1670  """get size and time for filename
1671
1672  USAGE:
1673    get_lsline(file_name)
1674
1675  DESCRIPTION:
1676    Read in one line 'ls -la' item from file (generated by filestat) and
1677    convert time to seconds since epoch. Return file size and time.
1678  """
1679
1680  import string, time
1681
1682  f = open(FN,'r')
1683  info = f.read()
1684  info = string.split(info)
1685
1686  size = info[4]
1687  week = info[5]
1688  mon  = info[6]
1689  day  = info[7]
1690  hour = info[8]
1691  year = info[9]
1692
1693  str = week+' '+mon+' '+day+' '+hour+' '+year
1694  timetup = time.strptime(str)
1695  t = time.mktime(timetup)
1696  return(size, t)
1697
1698# -----------------------------------------------------------------------------
1699
1700def checkdir(CD,verbose=None):
1701  """Check or create caching directory
1702
1703  USAGE:
1704    checkdir(CD,verbose):
1705
1706  ARGUMENTS:
1707    CD -- Directory
1708    verbose -- Flag verbose output (default: None)
1709
1710  DESCRIPTION:
1711    If CD does not exist it will be created if possible
1712  """
1713
1714  import os
1715  import os.path
1716
1717  if CD[-1] != os.sep: 
1718    CD = CD + os.sep  # Add separator for directories
1719
1720  CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1721  if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1722    try:
1723      exitcode=os.mkdir(CD)
1724
1725      # Change access rights if possible
1726      #
1727      if unix:
1728        exitcode=os.system('chmod 777 '+CD)
1729      else:
1730        pass  # FIXME: What about acces rights under Windows?
1731      if verbose: print 'MESSAGE: Directory', CD, 'created.'
1732    except:
1733      print 'WARNING: Directory', CD, 'could not be created.'
1734      if unix:
1735        CD = '/tmp/'
1736      else:
1737        CD = 'C:' 
1738      print 'Using directory %s instead' %CD
1739
1740  return(CD)
1741
1742#==============================================================================
1743# Statistics
1744#==============================================================================
1745
1746def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
1747                 compression):
1748  """Add stats entry
1749
1750  USAGE:
1751    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1752
1753  DESCRIPTION:
1754    Make one entry in the stats file about one cache hit recording time saved
1755    and other statistics. The data are used by the function cachestat.
1756  """
1757
1758  import os, time
1759
1760  try:
1761    TimeTuple = time.localtime(time.time())
1762    extension = time.strftime('%b%Y',TimeTuple)
1763    SFN = CD+statsfile+'.'+extension
1764    #statfile = open(SFN,'a')
1765    (statfile, dummy) = myopen(SFN,'a',compression=0)
1766
1767    # Change access rights if possible
1768    #
1769    #if unix:
1770    #  try:
1771    #    exitcode=os.system('chmod 666 '+SFN)
1772    #  except:
1773    #    pass
1774  except:
1775    print 'Warning: Stat file could not be opened'
1776
1777  try:
1778    if os.environ.has_key('USER'):
1779      user = os.environ['USER']
1780    else:
1781      user = 'Nobody'
1782
1783    date = time.asctime(TimeTuple)
1784
1785    if Retrieved:
1786      hit = '1'
1787    else:
1788      hit = '0'
1789
1790    # Get size of result file
1791    #   
1792    if compression:
1793      stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1794    else:
1795      stats = os.stat(CD+FN+'_'+file_types[0])
1796 
1797    if stats: 
1798      size = stats[6]
1799    else:
1800      size = -1  # Error condition, but don't crash. This is just statistics 
1801
1802    # Build entry
1803   
1804    entry = date             + ',' +\
1805            user             + ',' +\
1806            FN               + ',' +\
1807            str(int(size))   + ',' +\
1808            str(compression) + ',' +\
1809            hit              + ',' +\
1810            str(reason)      + ',' +\
1811            str(round(comptime,4)) + ',' +\
1812            str(round(loadtime,4)) +\
1813            CR
1814           
1815    statfile.write(entry)
1816    statfile.close()
1817  except:
1818    print 'Warning: Writing of stat file failed'
1819
1820# -----------------------------------------------------------------------------
1821
1822# FIXME: should take cachedir as an optional arg
1823#
1824def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1825  """  List caching statistics.
1826
1827  USAGE:
1828    __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1829
1830      Generate statistics of caching efficiency.
1831      The parameter sortidx determines by what field lists are sorted.
1832      If the optional keyword period is set to -1,
1833      all available caching history is used.
1834      If it is 0 only the current month is used.
1835      Future versions will include more than one month....
1836      OMN 20/8/2000
1837  """
1838
1839  import os
1840  import os.path
1841  from string import split, rstrip, find
1842  from time import strptime, localtime, strftime, mktime, ctime
1843
1844  # sortidx = 4    # Index into Fields[1:]. What to sort by.
1845
1846  Fields = ['Name', 'Hits', 'Exec(s)', \
1847            'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1848  Widths = [25,7,9,9,9,9,13]
1849  #Types = ['s','d','d','d','d','.2f','d']
1850  Types = ['s','d','.2f','.2f','.2f','.2f','d'] 
1851
1852  Dictnames = ['Function', 'User']
1853
1854  if not cachedir:
1855    cachedir = checkdir(options['cachedir'])
1856
1857  SD = os.path.expanduser(cachedir)  # Expand ~ or ~user in pathname
1858
1859  if period == -1:  # Take all available stats
1860    SFILENAME = statsfile
1861  else:  # Only stats from current month 
1862       # MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1863    TimeTuple = localtime(time())
1864    extension = strftime('%b%Y',TimeTuple)
1865    SFILENAME = statsfile+'.'+extension
1866
1867  DIRLIST = os.listdir(SD)
1868  SF = []
1869  for FN in DIRLIST:
1870    if find(FN,SFILENAME) >= 0:
1871      SF.append(FN)
1872
1873  blocksize = 15000000
1874  total_read = 0
1875  total_hits = 0
1876  total_discarded = 0
1877  firstday = mktime(strptime('2030','%Y'))
1878             # FIXME: strptime don't exist in WINDOWS ?
1879  lastday = 0
1880
1881  FuncDict = {}
1882  UserDict = {}
1883  for FN in SF:
1884    input = open(SD+FN,'r')
1885    print 'Reading file ', SD+FN
1886
1887    while 1:
1888      A = input.readlines(blocksize)
1889      if len(A) == 0: break
1890      total_read = total_read + len(A)
1891      for record in A:
1892        record = tuple(split(rstrip(record),','))
1893        #print record, len(record)
1894
1895        if len(record) == 9:
1896          timestamp = record[0]
1897       
1898          try:
1899            t = mktime(strptime(timestamp))
1900          except:
1901            total_discarded = total_discarded + 1         
1902            continue   
1903             
1904          if t > lastday:
1905            lastday = t
1906          if t < firstday:
1907            firstday = t
1908
1909          user     = record[1]
1910          func     = record[2]
1911
1912          # Strip hash-stamp off
1913          #
1914          i = find(func,'[')
1915          func = func[:i]
1916
1917          size        = float(record[3])
1918
1919          # Compression kepword can be Boolean
1920          if record[4] in ['True', '1']:
1921            compression = 1
1922          elif record[4] in ['False', '0']: 
1923            compression = 0
1924          else:
1925            print 'Unknown value of compression', record[4]
1926            print record
1927            total_discarded = total_discarded + 1           
1928            continue
1929
1930          #compression = int(record[4]) # Can be Boolean
1931          hit         = int(record[5])
1932          reason      = int(record[6])   # Not used here   
1933          cputime     = float(record[7])
1934          loadtime    = float(record[8])
1935
1936          if hit:
1937            total_hits = total_hits + 1
1938            saving = cputime-loadtime
1939
1940            if cputime != 0:
1941              rel_saving = round(100.0*saving/cputime,2)
1942            else:
1943              #rel_saving = round(1.0*saving,2)
1944              rel_saving = 100.0 - round(1.0*saving,2)  # A bit of a hack
1945
1946            info = [1,cputime,loadtime,saving,rel_saving,size]
1947
1948            UpdateDict(UserDict,user,info)
1949            UpdateDict(FuncDict,func,info)
1950          else:
1951            pass #Stats on recomputations and their reasons could go in here
1952             
1953        else:
1954          #print 'Record discarded'
1955          #print record
1956          total_discarded = total_discarded + 1
1957
1958    input.close()
1959
1960  # Compute averages of all sums and write list
1961  #
1962
1963  if total_read == 0:
1964    printline(Widths,'=')
1965    print 'CACHING STATISTICS: No valid records read'
1966    printline(Widths,'=')
1967    return
1968
1969  print
1970  printline(Widths,'=')
1971  print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
1972  printline(Widths,'=')
1973  #print '  Period:', ctime(firstday), 'to', ctime(lastday)
1974  print '  Total number of valid records', total_read
1975  print '  Total number of discarded records', total_discarded
1976  print '  Total number of hits', total_hits
1977  print
1978
1979  print '  Fields', Fields[2:], 'are averaged over number of hits'
1980  print '  Time is measured in seconds and size in bytes'
1981  print '  Tables are sorted by', Fields[1:][sortidx]
1982
1983  # printline(Widths,'-')
1984
1985  if showuser:
1986    Dictionaries = [FuncDict, UserDict]
1987  else:
1988    Dictionaries = [FuncDict]
1989
1990  i = 0
1991  for Dict in Dictionaries:
1992    for key in Dict.keys():
1993      rec = Dict[key]
1994      for n in range(len(rec)):
1995        if n > 0:
1996          rec[n] = round(1.0*rec[n]/rec[0],2)
1997      Dict[key] = rec
1998
1999    # Sort and output
2000    #
2001    keylist = SortDict(Dict,sortidx)
2002
2003    # Write Header
2004    #
2005    print
2006    #print Dictnames[i], 'statistics:'; i=i+1
2007    printline(Widths,'-')
2008    n = 0
2009    for s in Fields:
2010      if s == Fields[0]:  # Left justify
2011        s = Dictnames[i] + ' ' + s; i=i+1
2012        exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
2013      else:
2014        exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
2015    print
2016    printline(Widths,'-')
2017
2018    # Output Values
2019    #
2020    for key in keylist:
2021      rec = Dict[key]
2022      n = 0
2023      if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
2024      exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
2025      for val in rec:
2026        exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
2027      print
2028    print
2029
2030#==============================================================================
2031# Auxiliary stats functions
2032#==============================================================================
2033
2034def UpdateDict(Dict,key,info):
2035  """Update dictionary by adding new values to existing.
2036
2037  USAGE:
2038    UpdateDict(Dict,key,info)
2039  """
2040
2041  if Dict.has_key(key):
2042    dinfo = Dict[key]
2043    for n in range(len(dinfo)):
2044      dinfo[n] = info[n] + dinfo[n]
2045  else:
2046    dinfo = info[:]  # Make a copy of info list
2047
2048  Dict[key] = dinfo
2049  return Dict
2050
2051# -----------------------------------------------------------------------------
2052
2053def SortDict(Dict,sortidx=0):
2054  """Sort dictionary
2055
2056  USAGE:
2057    SortDict(Dict,sortidx):
2058
2059  DESCRIPTION:
2060    Sort dictionary of lists according field number 'sortidx'
2061  """
2062
2063  import types
2064
2065  sortlist  = []
2066  keylist = Dict.keys()
2067  for key in keylist:
2068    rec = Dict[key]
2069    if not type(rec) in [types.ListType, types.TupleType]:
2070      rec = [rec]
2071
2072    if sortidx > len(rec)-1:
2073      if options['verbose']:
2074        print 'ERROR: Sorting index to large, sortidx = ', sortidx
2075      raise IndexError
2076
2077    val = rec[sortidx]
2078    sortlist.append(val)
2079
2080  A = map(None,sortlist,keylist)
2081  A.sort()
2082  keylist = map(lambda x: x[1], A)  # keylist sorted by sortidx
2083
2084  return(keylist)
2085
2086# -----------------------------------------------------------------------------
2087
2088def printline(Widths,char):
2089  """Print textline in fixed field.
2090
2091  USAGE:
2092    printline(Widths,char)
2093  """
2094
2095  s = ''
2096  for n in range(len(Widths)):
2097    s = s+Widths[n]*char
2098    if n > 0:
2099      s = s+char
2100
2101  print s
2102
2103#==============================================================================
2104# Messages
2105#==============================================================================
2106
2107def msg1(funcname,args,kwargs,reason):
2108  """Message 1
2109
2110  USAGE:
2111    msg1(funcname,args,kwargs,reason):
2112  """
2113
2114  import string
2115  #print 'MESSAGE (caching.py): Evaluating function', funcname,
2116
2117  print_header_box('Evaluating function %s' %funcname)
2118 
2119  msg7(args, kwargs)
2120  msg8(reason) 
2121 
2122  print_footer()
2123 
2124  #
2125  # Old message
2126  #
2127  #args_present = 0
2128  #if args:
2129  #  if len(args) == 1:
2130  #    print 'with argument', mkargstr(args[0], textwidth2),
2131  #  else:
2132  #    print 'with arguments', mkargstr(args, textwidth2),
2133  #  args_present = 1     
2134  #   
2135  #if kwargs:
2136  #  if args_present:
2137  #    word = 'and'
2138  #  else:
2139  #    word = 'with'
2140  #     
2141  #  if len(kwargs) == 1:
2142  #    print word + ' keyword argument', mkargstr(kwargs, textwidth2)
2143  #  else:
2144  #    print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
2145  #  args_present = 1           
2146  #else:
2147  #  print    # Newline when no keyword args present
2148  #       
2149  #if not args_present:   
2150  #  print '',  # Default if no args or kwargs present
2151   
2152   
2153
2154# -----------------------------------------------------------------------------
2155
2156def msg2(funcname,args,kwargs,comptime,reason):
2157  """Message 2
2158
2159  USAGE:
2160    msg2(funcname,args,kwargs,comptime,reason)
2161  """
2162
2163  import string
2164
2165  #try:
2166  #  R = Reason_msg[reason]
2167  #except:
2168  #  R = 'Unknown reason' 
2169 
2170  #print_header_box('Caching statistics (storing) - %s' %R)
2171  print_header_box('Caching statistics (storing)') 
2172 
2173  msg6(funcname,args,kwargs)
2174  msg8(reason)
2175
2176  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2177
2178# -----------------------------------------------------------------------------
2179
2180def msg3(savetime, CD, FN, deps,compression):
2181  """Message 3
2182
2183  USAGE:
2184    msg3(savetime, CD, FN, deps,compression)
2185  """
2186
2187  import string
2188  print string.ljust('| Loading time:', textwidth1) + str(round(savetime,2)) + \
2189                     ' seconds (estimated)'
2190  msg5(CD,FN,deps,compression)
2191
2192# -----------------------------------------------------------------------------
2193
2194def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
2195  """Message 4
2196
2197  USAGE:
2198    msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
2199  """
2200
2201  import string
2202
2203  print_header_box('Caching statistics (retrieving)')
2204 
2205  msg6(funcname,args,kwargs)
2206  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2207  print string.ljust('| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
2208  print string.ljust('| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
2209        ' seconds'
2210  msg5(CD,FN,deps,compression)
2211
2212# -----------------------------------------------------------------------------
2213
2214def msg5(CD,FN,deps,compression):
2215  """Message 5
2216
2217  USAGE:
2218    msg5(CD,FN,deps,compression)
2219
2220  DESCRIPTION:
2221   Print dependency stats. Used by msg3 and msg4
2222  """
2223
2224  import os, time, string
2225
2226  print '|'
2227  print string.ljust('| Caching dir: ', textwidth1) + CD
2228
2229  if compression:
2230    suffix = '.z'
2231    bytetext = 'bytes, compressed'
2232  else:
2233    suffix = ''
2234    bytetext = 'bytes'
2235
2236  for file_type in file_types:
2237    file_name = FN + '_' + file_type + suffix
2238    print string.ljust('| ' + file_type + ' file: ', textwidth1) + file_name,
2239    stats = os.stat(CD+file_name)
2240    print '('+ str(stats[6]) + ' ' + bytetext + ')'
2241
2242  print '|'
2243  if len(deps) > 0:
2244    print '| Dependencies:  '
2245    dependencies  = deps.keys()
2246    dlist = []; maxd = 0
2247    tlist = []; maxt = 0
2248    slist = []; maxs = 0
2249    for d in dependencies:
2250      stats = deps[d]
2251      t = time.ctime(stats[1])
2252      s = str(stats[0])
2253      #if s[-1] == 'L':
2254      #  s = s[:-1]  # Strip rightmost 'long integer' L off.
2255      #              # FIXME: Unnecessary in versions later than 1.5.2
2256
2257      if len(d) > maxd: maxd = len(d)
2258      if len(t) > maxt: maxt = len(t)
2259      if len(s) > maxs: maxs = len(s)
2260      dlist.append(d)
2261      tlist.append(t)
2262      slist.append(s)
2263
2264    for n in range(len(dlist)):
2265      d = string.ljust(dlist[n]+':', maxd+1)
2266      t = string.ljust(tlist[n], maxt)
2267      s = string.rjust(slist[n], maxs)
2268
2269      print '| ', d, t, ' ', s, 'bytes'
2270  else:
2271    print '| No dependencies'
2272  print_footer()
2273
2274# -----------------------------------------------------------------------------
2275
2276def msg6(funcname,args,kwargs):
2277  """Message 6
2278
2279  USAGE:
2280    msg6(funcname,args,kwargs)
2281  """
2282
2283  import string
2284  print string.ljust('| Function:', textwidth1) + funcname
2285
2286  msg7(args, kwargs)
2287 
2288# -----------------------------------------------------------------------------   
2289
2290def msg7(args, kwargs):
2291  """Message 7
2292 
2293  USAGE:
2294    msg7(args, kwargs):
2295  """
2296 
2297  import string
2298 
2299  args_present = 0 
2300  if args:
2301    if len(args) == 1:
2302      print string.ljust('| Argument:', textwidth1) + mkargstr(args[0], \
2303                         textwidth2)
2304    else:
2305      print string.ljust('| Arguments:', textwidth1) + \
2306            mkargstr(args, textwidth2)
2307    args_present = 1
2308           
2309  if kwargs:
2310    if len(kwargs) == 1:
2311      print string.ljust('| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
2312                         textwidth2)
2313    else:
2314      print string.ljust('| Keyword Args:', textwidth1) + \
2315            mkargstr(kwargs, textwidth2)
2316    args_present = 1
2317
2318  if not args_present:               
2319    print '| No arguments' # Default if no args or kwargs present
2320
2321# -----------------------------------------------------------------------------
2322
2323def msg8(reason):
2324  """Message 8
2325 
2326  USAGE:
2327    msg8(reason):
2328  """
2329 
2330  import string
2331   
2332  try:
2333    R = Reason_msg[reason]
2334  except:
2335    R = 'Unknown' 
2336 
2337  print string.ljust('| Reason:', textwidth1) + R
2338   
2339# -----------------------------------------------------------------------------
2340
2341def print_header_box(line):
2342  """Print line in a nice box.
2343 
2344  USAGE:
2345    print_header_box(line)
2346
2347  """
2348  global textwidth3
2349
2350  import time
2351
2352  time_stamp = time.ctime(time.time())
2353  line = time_stamp + '. ' + line
2354   
2355  N = len(line) + 1
2356  s = '+' + '-'*N + CR
2357
2358  print s + '| ' + line + CR + s,
2359
2360  textwidth3 = N
2361
2362# -----------------------------------------------------------------------------
2363   
2364def print_footer():
2365  """Print line same width as that of print_header_box.
2366  """
2367 
2368  N = textwidth3
2369  s = '+' + '-'*N + CR   
2370     
2371  print s     
2372     
2373# -----------------------------------------------------------------------------
2374
2375def mkargstr(args, textwidth, argstr = '', level=0):
2376  """ Generate a string containing first textwidth characters of arguments.
2377
2378  USAGE:
2379    mkargstr(args, textwidth, argstr = '', level=0)
2380
2381  DESCRIPTION:
2382    Exactly the same as str(args) possibly followed by truncation,
2383    but faster if args is huge.
2384  """
2385
2386  import types
2387
2388  if level > 10:
2389      # Protect against circular structures
2390      return '...'
2391 
2392  WasTruncated = 0
2393
2394  if not type(args) in [types.TupleType, types.ListType, types.DictType]:
2395    if type(args) == types.StringType:
2396      argstr = argstr + "'"+str(args)+"'"
2397    else:
2398      # Truncate large Numeric arrays before using str()
2399      import Numeric
2400      if type(args) == Numeric.ArrayType:
2401#        if len(args.flat) > textwidth: 
2402#        Changed by Duncan and Nick 21/2/07 .flat has problems with
2403#        non-contigous arrays and ravel is equal to .flat except it
2404#        can work with non-contiguous  arrays
2405        if len(Numeric.ravel(args)) > textwidth:
2406          args = 'Array: ' + str(args.shape)
2407
2408      argstr = argstr + str(args)
2409  else:
2410    if type(args) == types.DictType:
2411      argstr = argstr + "{"
2412      for key in args.keys():
2413        argstr = argstr + mkargstr(key, textwidth, level=level+1) + ": " + \
2414                 mkargstr(args[key], textwidth, level=level+1) + ", "
2415        if len(argstr) > textwidth:
2416          WasTruncated = 1
2417          break
2418      argstr = argstr[:-2]  # Strip off trailing comma     
2419      argstr = argstr + "}"
2420
2421    else:
2422      if type(args) == types.TupleType:
2423        lc = '('
2424        rc = ')'
2425      else:
2426        lc = '['
2427        rc = ']'
2428      argstr = argstr + lc
2429      for arg in args:
2430        argstr = argstr + mkargstr(arg, textwidth, level=level+1) + ', '
2431        if len(argstr) > textwidth:
2432          WasTruncated = 1
2433          break
2434
2435      # Strip off trailing comma and space unless singleton tuple
2436      #
2437      if type(args) == types.TupleType and len(args) == 1:
2438        argstr = argstr[:-1]   
2439      else:
2440        argstr = argstr[:-2]
2441      argstr = argstr + rc
2442
2443  if len(argstr) > textwidth:
2444    WasTruncated = 1
2445
2446  if WasTruncated:
2447    argstr = argstr[:textwidth]+'...'
2448  return(argstr)
2449
2450# -----------------------------------------------------------------------------
2451
2452def test_OK(msg):
2453  """Print OK msg if test is OK.
2454 
2455  USAGE
2456    test_OK(message)
2457  """
2458
2459  import string
2460   
2461  print string.ljust(msg, textwidth4) + ' - OK' 
2462 
2463  #raise StandardError
2464 
2465# -----------------------------------------------------------------------------
2466
2467def test_error(msg):
2468  """Print error if test fails.
2469 
2470  USAGE
2471    test_error(message)
2472  """
2473 
2474  print 'ERROR (caching.test): %s' %msg
2475  print 'Please send this code example and output to '
2476  print 'Ole.Nielsen@anu.edu.au'
2477  print
2478  print
2479 
2480  #import sys
2481  #sys.exit()
2482  raise StandardError
2483
2484#-------------------------------------------------------------
2485if __name__ == "__main__":
2486  pass
Note: See TracBrowser for help on using the repository browser.