source: anuga_core/source/anuga/caching/caching.py @ 7276

Last change on this file since 7276 was 7276, checked in by ole, 15 years ago

Merged numpy branch back into the trunk.

In ~/sandpit/anuga/anuga_core/source
svn merge -r 6246:HEAD ../../branches/numpy .

In ~/sandpit/anuga/anuga_validation
svn merge -r 6417:HEAD ../branches/numpy_anuga_validation .

In ~/sandpit/anuga/misc
svn merge -r 6809:HEAD ../branches/numpy_misc .

For all merges, I used numpy version where conflicts existed

The suites test_all.py (in source/anuga) and validate_all.py passed using Python2.5 with numpy on my Ubuntu Linux box.
File size: 70.9 KB
Line 
1# =============================================================================
2# caching.py - Supervised caching of function results.
3# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4# Australian National University (1999-2003)
5# Geoscience Australia (2003-present)
6#
7#    This program is free software; you can redistribute it and/or modify
8#    it under the terms of the GNU General Public License as published by
9#    the Free Software Foundation; either version 2 of the License, or
10#    (at your option) any later version.
11#
12#    This program is distributed in the hope that it will be useful,
13#    but WITHOUT ANY WARRANTY; without even the implied warranty of
14#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15#    GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16#    for more details.
17#
18#    You should have received a copy of the GNU General Public License
19#    along with this program; if not, write to the Free Software
20#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
21#
22#
23# Contact address: Ole.Nielsen@ga.gov.au
24#
25# Version 1.5.6 February 2002
26# =============================================================================
27 
28"""Module caching.py - Supervised caching of function results.
29
30Public functions:
31
32cache(func,args) -- Cache values returned from func given args.
33cachestat() --      Reports statistics about cache hits and time saved.
34test() --       Conducts a basic test of the caching functionality.
35
36See doc strings of individual functions for detailed documentation.
37"""
38
39# -----------------------------------------------------------------------------
40# Initialisation code
41
42# Determine platform
43#
44from os import getenv
45import types
46
47import os
48if os.name in ['nt', 'dos', 'win32', 'what else?']:
49  unix = False
50else:
51  unix = True
52
53import numpy as num
54
55
56cache_dir = '.python_cache'
57
58# Make default caching directory name
59# We are changing the 'data directory' environment variable from
60# INUNDATIONHOME to ANUGADATA - this gives a changeover.
61if unix:
62    homedir = getenv('ANUGADATA')
63    if not homedir:
64        homedir = getenv('INUNDATIONHOME')
65
66    if not homedir:
67        homedir = '~'
68    else:
69        # Since homedir will be a group area, individually label the caches
70        user = getenv('LOGNAME')
71        if not user:
72            cache_dir += '_' + user
73   
74    CR = '\n'
75else:
76    homedir = 'c:'
77    CR = '\r\n'  #FIXME: Not tested under windows
78 
79cachedir = os.path.join(homedir, cache_dir)
80
81# -----------------------------------------------------------------------------
82# Options directory with default values - to be set by user
83#
84
85options = { 
86  'cachedir': cachedir,  # Default cache directory
87  'maxfiles': 1000000,   # Maximum number of cached files
88  'savestat': True,      # Log caching info to stats file
89  'verbose': True,       # Write messages to standard output
90  'bin': True,           # Use binary format (more efficient)
91  'compression': True,   # Use zlib compression
92  'bytecode': True,      # Recompute if bytecode has changed
93  'expire': False        # Automatically remove files that have been accessed
94                         # least recently
95}
96
97# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
98
99def set_option(key, value):
100  """Function to set values in the options directory.
101
102  USAGE:
103    set_option(key, value)
104
105  ARGUMENTS:
106    key --   Key in options dictionary. (Required)
107    value -- New value for key. (Required)
108
109  DESCRIPTION:
110    Function to set values in the options directory.
111    Raises an exception if key is not in options.
112  """
113
114  if options.has_key(key):
115    options[key] = value
116  else:
117    raise KeyError(key)  # Key not found, raise an exception
118
119# -----------------------------------------------------------------------------
120# Function cache - the main routine
121
122def cache(func, 
123          args=(), 
124          kwargs={}, 
125          dependencies=None, 
126          cachedir=None,
127          verbose=None, 
128          compression=None, 
129          evaluate=False, 
130          test=False, 
131          clear=False,
132          return_filename=False):
133  """Supervised caching of function results. Also known as memoization.
134
135  USAGE:
136    result = cache(func, args, kwargs, dependencies, cachedir, verbose,
137                   compression, evaluate, test, return_filename)
138
139  ARGUMENTS:
140    func --            Function object (Required)
141    args --            Arguments to func (Default: ())
142    kwargs --          Keyword arguments to func (Default: {})   
143    dependencies --    Filenames that func depends on (Default: None)
144    cachedir --        Directory for cache files (Default: options['cachedir'])
145    verbose --         Flag verbose output to stdout
146                       (Default: options['verbose'])
147    compression --     Flag zlib compression (Default: options['compression'])
148    evaluate --        Flag forced evaluation of func (Default: False)
149    test --            Flag test for cached results (Default: False)
150    clear --           Flag delete cached results (Default: False)   
151    return_filename -- Flag return of cache filename (Default: False)   
152
153  DESCRIPTION:
154    A Python function call of the form
155
156      result = func(arg1,...,argn)
157
158    can be replaced by
159
160      from caching import cache
161      result = cache(func,(arg1,...,argn))
162
163  The latter form returns the same output as the former but reuses cached
164  results if the function has been computed previously in the same context.
165  'result' and the arguments can be simple types, tuples, list, dictionaries or
166  objects, but not unhashable types such as functions or open file objects.
167  The function 'func' may be a member function of an object or a module.
168
169  This type of caching is particularly useful for computationally intensive
170  functions with few frequently used combinations of input arguments. Note that
171  if the inputs or output are very large caching might not save time because
172  disc access may dominate the execution time.
173
174  If the function definition changes after a result has been cached it will be
175  detected by examining the functions bytecode (co_code, co_consts,
176  func_defaults, co_argcount) and it will be recomputed.
177
178  LIMITATIONS:
179    1 Caching uses the apply function and will work with anything that can be
180      pickled, so any limitation in apply or pickle extends to caching.
181    2 A function to be cached should not depend on global variables
182      as wrong results may occur if globals are changed after a result has
183      been cached.
184
185  -----------------------------------------------------------------------------
186  Additional functionality:
187
188  Keyword args
189    Keyword arguments (kwargs) can be added as a dictionary of keyword: value
190    pairs, following the syntax of the built-in function apply().
191    A Python function call of the form
192   
193      result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)   
194
195    is then cached as follows
196
197      from caching import cache
198      result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
199   
200    The default value of kwargs is {} 
201
202  Explicit dependencies:
203    The call
204      cache(func,(arg1,...,argn), dependencies = <list of filenames>)
205    Checks the size, creation time and modification time of each listed file.
206    If any file has changed the function is recomputed and the results stored
207    again.
208
209  Specify caching directory:
210    The call
211      cache(func,(arg1,...,argn), cachedir = <cachedir>)
212    designates <cachedir> where cached data are stored. Use ~ to indicate users
213    home directory - not $HOME. The default is ~/.python_cache on a UNIX
214    platform and c:/.python_cache on a Win platform.
215
216  Silent operation:
217    The call
218      cache(func,(arg1,...,argn), verbose=False)
219    suppresses messages to standard output.
220
221  Compression:
222    The call
223      cache(func,(arg1,...,argn), compression=False)
224    disables compression. (Default: compression=True). If the requested compressed
225    or uncompressed file is not there, it'll try the other version.
226
227  Forced evaluation:
228    The call
229      cache(func,(arg1,...,argn), evaluate=True)
230    forces the function to evaluate even though cached data may exist.
231
232  Testing for presence of cached result:
233    The call
234      cache(func,(arg1,...,argn), test=True)
235    retrieves cached result if it exists, otherwise None. The function will not
236    be evaluated. If both evaluate and test are switched on, evaluate takes
237    precedence.
238    ??NOTE: In case of hash collisions, this may return the wrong result as
239    ??it only checks if *a* cached result is present.
240    # I think this was due to the bytecode option being False for some reason. (23/1/2009).
241   
242  Obtain cache filenames:
243    The call   
244      cache(func,(arg1,...,argn), return_filename=True)
245    returns the hashed base filename under which this function and its
246    arguments would be cached
247
248  Clearing cached results:
249    The call
250      cache(func,'clear')
251    clears all cached data for 'func' and
252      cache('clear')
253    clears all cached data.
254 
255    NOTE: The string 'clear' can be passed an *argument* to func using
256      cache(func,('clear',)) or cache(func,tuple(['clear'])).
257
258    New form of clear:
259      cache(func,(arg1,...,argn), clear=True)
260    clears cached data for particular combination func and args
261     
262  """
263
264  # Imports and input checks
265  #
266  import types, time, string
267
268  if not cachedir:
269    cachedir = options['cachedir']
270
271  if verbose == None:  # Do NOT write 'if not verbose:', it could be zero.
272    verbose = options['verbose']
273
274  if compression == None:  # Do NOT write 'if not compression:',
275                           # it could be zero.
276    compression = options['compression']
277
278  # Create cache directory if needed
279  CD = checkdir(cachedir,verbose)
280
281  # Handle the case cache('clear')
282  if type(func) == types.StringType:
283    if string.lower(func) == 'clear':
284      clear_cache(CD,verbose=verbose)
285      return
286
287  # Handle the case cache(func, 'clear')
288  if type(args) == types.StringType:
289    if string.lower(args) == 'clear':
290      clear_cache(CD,func,verbose=verbose)
291      return
292
293  # Force singleton arg into a tuple
294  if type(args) != types.TupleType:
295    args = tuple([args])
296 
297  # Check that kwargs is a dictionary
298  if type(kwargs) != types.DictType:
299    raise TypeError   
300   
301  # Hash arguments (and keyword args) to integer
302  arghash = myhash((args, kwargs))
303 
304  # Get sizes and timestamps for files listed in dependencies.
305  # Force singletons into a tuple.
306  if dependencies and type(dependencies) != types.TupleType \
307                  and type(dependencies) != types.ListType:
308    dependencies = tuple([dependencies])
309  deps = get_depstats(dependencies)
310
311  # Extract function name from func object
312  funcname = get_funcname(func)
313
314  # Create cache filename
315  FN = funcname+'['+`arghash`+']'  # The symbol '(' does not work under unix
316
317  if return_filename:
318    return(FN)
319
320  if clear:
321    for file_type in file_types:
322      file_name = CD+FN+'_'+file_type
323      for fn in [file_name, file_name + '.z']:
324        if os.access(fn, os.F_OK):             
325          if unix:
326            os.remove(fn)
327          else:
328            # FIXME: os.remove doesn't work under windows       
329            os.system('del '+fn)
330          if verbose is True:
331            print 'MESSAGE (caching): File %s deleted' %fn
332        ##else:
333        ##  print '%s was not accessed' %fn
334    return None
335
336
337  #-------------------------------------------------------------------       
338 
339  # Check if previous computation has been cached
340  if evaluate is True:
341    Retrieved = None  # Force evaluation of func regardless of caching status.
342    reason = 5
343  else:
344    T, FN, Retrieved, reason, comptime, loadtime, compressed = \
345        CacheLookup(CD, FN, func, 
346                    args, kwargs, 
347                    deps, 
348                    verbose, 
349                    compression,
350                    dependencies)
351
352  if not Retrieved:
353    if test:  # Do not attempt to evaluate function
354      T = None
355    else:  # Evaluate function and save to cache
356      if verbose is True:
357       
358        msg1(funcname, args, kwargs,reason)
359
360      # Remove expired files automatically
361      if options['expire']:
362        DeleteOldFiles(CD,verbose)
363       
364      # Save args before function is evaluated in case
365      # they are modified by function
366      save_args_to_cache(CD,FN,args,kwargs,compression)
367
368      # Execute and time function with supplied arguments
369      t0 = time.time()
370      T = apply(func,args,kwargs)
371      #comptime = round(time.time()-t0)
372      comptime = time.time()-t0
373
374      if verbose is True:
375        msg2(funcname,args,kwargs,comptime,reason)
376
377      # Save results and estimated loading time to cache
378      loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
379                                       funcname, dependencies, compression)
380      if verbose is True:
381        msg3(loadtime, CD, FN, deps, compression)
382      compressed = compression
383
384  if options['savestat'] and (not test or Retrieved):
385  ##if options['savestat']:
386    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
387
388  return(T)  # Return results in all cases
389
390# -----------------------------------------------------------------------------
391
392def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
393  """Generate statistics of caching efficiency.
394
395  USAGE:
396    cachestat(sortidx, period, showuser, cachedir)
397
398  ARGUMENTS:
399    sortidx --  Index of field by which lists are (default: 4)
400                Legal values are
401                 0: 'Name'
402                 1: 'Hits'
403                 2: 'CPU'
404                 3: 'Time Saved'
405                 4: 'Gain(%)'
406                 5: 'Size'
407    period --   If set to -1 all available caching history is used.
408                If set 0 only the current month is used (default -1).
409    showuser -- Flag for additional table showing user statistics
410                (default: None).
411    cachedir -- Directory for cache files (default: options['cachedir']).
412
413  DESCRIPTION:
414    Logged caching statistics is converted into summaries of the form
415    --------------------------------------------------------------------------
416    Function Name   Hits   Exec(s)  Cache(s)  Saved(s)   Gain(%)      Size
417    --------------------------------------------------------------------------
418  """
419
420  __cachestat(sortidx, period, showuser, cachedir)
421  return
422
423# -----------------------------------------------------------------------------
424
425# Has mostly been moved to proper unit test.
426# What remains here includes example of the
427# cache statistics form.
428def test(cachedir=None, verbose=False, compression=None):
429  """Test the functionality of caching.
430
431  USAGE:
432    test(verbose)
433
434  ARGUMENTS:
435    verbose --     Flag whether caching will output its statistics (default=False)
436    cachedir --    Directory for cache files (Default: options['cachedir'])
437    compression -- Flag zlib compression (Default: options['compression'])
438  """
439   
440  import string, time
441
442  # Initialise
443  #
444  #import caching
445  #reload(caching)
446
447  if not cachedir:
448    cachedir = options['cachedir']
449
450  if verbose is None:  # Do NOT write 'if not verbose:', it could be zero.
451    verbose = options['verbose']
452 
453  if compression == None:  # Do NOT write 'if not compression:',
454                           # it could be zero.
455    compression = options['compression']
456  else:
457    try:
458      set_option('compression', compression)
459    except:
460      test_error('Set option failed')     
461
462  try:
463    import zlib
464  except:
465    print
466    print '*** Could not find zlib, default to no-compression      ***'
467    print '*** Installing zlib will improve performance of caching ***'
468    print
469    compression = 0       
470    set_option('compression', compression)   
471 
472  print 
473  print_header_box('Testing caching module - please stand by')
474  print   
475
476  # Define a test function to be cached
477  #
478  def f(a,b,c,N,x=0,y='abcdefg'):
479    """f(a,b,c,N)
480       Do something time consuming and produce a complex result.
481    """
482
483    import string
484
485    B = []
486    for n in range(N):
487      s = str(n+2.0/(n + 4.0))+'.a'*10
488      B.append((a,b,c,s,n,x,y))
489    return(B)
490   
491  # Check that default cachedir is OK
492  #     
493  CD = checkdir(cachedir,verbose)   
494   
495   
496  # Make a dependency file
497  #   
498  try:
499    DepFN = CD + 'testfile.tmp'
500    DepFN_wildcard = CD + 'test*.tmp'
501    Depfile = open(DepFN,'w')
502    Depfile.write('We are the knights who say NI!')
503    Depfile.close()
504    test_OK('Wrote file %s' %DepFN)
505  except:
506    test_error('Could not open file %s for writing - check your environment' \
507               % DepFN)
508
509  # Check set_option (and switch stats off
510  #   
511  try:
512    set_option('savestat',0)
513    assert(options['savestat'] == 0)
514    test_OK('Set option')
515  except:
516    test_error('Set option failed')   
517   
518  # Make some test input arguments
519  #
520  N = 5000  #Make N fairly small here
521
522  a = [1,2]
523  b = ('Thou shalt count the number three',4)
524  c = {'Five is right out': 6, (7,8): 9}
525  x = 3
526  y = 'holy hand granate'
527
528  # Test caching
529  #
530  if compression:
531    comprange = 2
532  else:
533    comprange = 1
534
535  for comp in range(comprange):
536 
537    # Evaluate and store
538    #
539    try:
540      T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
541                   verbose=verbose, compression=comp)
542      if comp:                   
543        test_OK('Caching evaluation with compression')
544      else:     
545        test_OK('Caching evaluation without compression')     
546    except:
547      if comp:
548        test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
549      else:
550        test_error('Caching evaluation failed - try caching.test(verbose=1)')
551
552    # Retrieve
553    #                           
554    try:                         
555      T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
556                   compression=comp) 
557
558      if comp:                   
559        test_OK('Caching retrieval with compression')
560      else:     
561        test_OK('Caching retrieval without compression')     
562    except:
563      if comp:
564        test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
565      else:                                     
566        test_error('Caching retrieval failed - try caching.test(verbose=1)')
567
568    # Reference result
569    #   
570    T3 = f(a,b,c,N,x=x,y=y)  # Compute without caching
571   
572    if T1 == T2 and T2 == T3:
573      if comp:
574        test_OK('Basic caching functionality (with compression)')
575      else:
576        test_OK('Basic caching functionality (without compression)')
577    else:
578      test_error('Cached result does not match computed result')
579
580
581  # Test return_filename
582  #   
583  try:
584    FN = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
585                 return_filename=1)   
586    assert(FN[:2] == 'f[')
587    test_OK('Return of cache filename')
588  except:
589    test_error('Return of cache filename failed')
590
591  # Test existence of cachefiles
592  # 
593  try:
594    (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
595    (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
596    (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
597    test_OK('Presence of cache files')
598    datafile.close()
599    argsfile.close()
600    admfile.close()
601  except:
602    test_error('Expected cache files did not exist') 
603             
604  # Test 'test' function when cache is present
605  #     
606  try:
607    #T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
608    #                   evaluate=1) 
609    T4 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
610    assert(T1 == T4)
611
612    test_OK("Option 'test' when cache file present")
613  except:
614    test_error("Option 'test' when cache file present failed")     
615
616  # Test that 'clear' works
617  #
618  #try:
619  #  cache(f,'clear',verbose=verbose)
620  #  test_OK('Clearing of cache files')
621  #except:
622  #  test_error('Clear does not work')
623  try:
624    cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)   
625    test_OK('Clearing of cache files')
626  except:
627    test_error('Clear does not work') 
628
629 
630
631  # Test 'test' function when cache is absent
632  #     
633  try:
634    T4 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
635    assert(T4 is None)
636    test_OK("Option 'test' when cache absent")
637  except:
638    test_error("Option 'test' when cache absent failed")     
639         
640  # Test dependencies
641  #
642  T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
643               dependencies=DepFN) 
644  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
645               dependencies=DepFN)                     
646                       
647  if T1 == T2:
648    test_OK('Basic dependencies functionality')
649  else:
650    test_error('Dependencies do not work')
651
652  # Test basic wildcard dependency
653  #
654  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
655               dependencies=DepFN_wildcard)                     
656   
657  if T1 == T3:
658    test_OK('Basic dependencies with wildcard functionality')
659  else:
660    test_error('Dependencies with wildcards do not work')
661
662
663  # Test that changed timestamp in dependencies triggers recomputation
664 
665  # Modify dependency file
666  Depfile = open(DepFN,'a')
667  Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
668  Depfile.close()
669 
670  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
671               dependencies=DepFN, test = 1)                     
672 
673  if T3 is None:
674    test_OK('Changed dependencies recognised')
675  else:
676    test_error('Changed dependencies not recognised')   
677 
678  # Test recomputation when dependencies have changed
679  #
680  T3 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
681               dependencies=DepFN)                       
682  if T1 == T3:
683    test_OK('Recomputed value with changed dependencies')
684  else:
685    test_error('Recomputed value with changed dependencies failed')
686
687  # Performance test (with statistics)
688  # Don't really rely on this as it will depend on specific computer.
689  #
690
691  set_option('savestat',1)
692
693  N = 20*N   #Should be large on fast computers...
694  tt = time.time()
695  T1 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
696  t1 = time.time() - tt
697 
698  tt = time.time()
699  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
700  t2 = time.time() - tt
701 
702  if T1 == T2:
703    if t1 > t2:
704      test_OK('Performance test: relative time saved = %s pct' \
705              %str(round((t1-t2)*100/t1,2)))
706    #else:
707    #  print 'WARNING: Performance a bit low - this could be specific to current platform'
708  else:       
709    test_error('Basic caching failed for new problem')
710           
711  # Test presence of statistics file
712  #
713  try: 
714    DIRLIST = os.listdir(CD)
715    SF = []
716    for FN in DIRLIST:
717      if string.find(FN,statsfile) >= 0:
718        fid = open(CD+FN,'r')
719        fid.close()
720    test_OK('Statistics files present') 
721  except:
722    test_OK('Statistics files cannot be opened')         
723     
724  print_header_box('Show sample output of the caching function:')
725 
726  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
727  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
728  T2 = cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
729 
730  print_header_box('Show sample output of cachestat():')
731  if unix:
732    cachestat()   
733  else:
734    try:
735      import time
736      t = time.strptime('2030','%Y')
737      cachestat()
738    except: 
739      print 'cachestat() does not work here, because it'
740      print 'relies on time.strptime() which is unavailable in Windows'
741     
742  print
743  test_OK('Caching self test completed')   
744     
745           
746  # Test setoption (not yet implemented)
747  #
748
749 
750#==============================================================================
751# Auxiliary functions
752#==============================================================================
753
754# Import pickler
755# cPickle is used by functions mysave, myload, and compare
756#
757import cPickle  # 10 to 100 times faster than pickle
758pickler = cPickle
759
760# Local immutable constants
761#
762comp_level = 1              # Compression level for zlib.
763                            # comp_level = 1 works well.
764textwidth1 = 16             # Text width of key fields in report forms.
765#textwidth2 = 132            # Maximal width of textual representation of
766textwidth2 = 300            # Maximal width of textual representation of
767                            # arguments.
768textwidth3 = 16             # Initial width of separation lines. Is modified.
769textwidth4 = 50             # Text width in test_OK()
770statsfile  = '.cache_stat'  # Basefilename for cached statistics.
771                            # It will reside in the chosen cache directory.
772
773file_types = ['Result',     # File name extension for cached function results.
774              'Args',       # File name extension for stored function args.
775              'Admin']      # File name extension for administrative info.
776
777Reason_msg = ['OK',         # Verbose reasons for recomputation
778              'No cached result', 
779              'Dependencies have changed', 
780              'Arguments have changed',
781              'Bytecode has changed',
782              'Recomputation was requested by caller',
783              'Cached file was unreadable']             
784             
785# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
786
787def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, 
788                dependencies):
789  """Determine whether cached result exists and return info.
790
791  USAGE:
792    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \ 
793    CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
794                dependencies)
795
796  INPUT ARGUMENTS:
797    CD --            Cache Directory
798    FN --            Suggested cache file name
799    func --          Function object
800    args --          Tuple of arguments
801    kwargs --        Dictionary of keyword arguments   
802    deps --          Dependencies time stamps
803    verbose --       Flag text output
804    compression --   Flag zlib compression
805    dependencies --  Given list of dependencies
806   
807  OUTPUT ARGUMENTS:
808    T --             Cached result if present otherwise None
809    FN --            File name under which new results must be saved
810    Retrieved --     True if a valid cached result was found
811    reason --        0: OK (if Retrieved),
812                     1: No cached result,
813                     2: Dependencies have changed,
814                     3: Arguments have changed
815                     4: Bytecode has changed
816                     5: Recomputation was forced
817                     6: Unreadable file
818    comptime --      Number of seconds it took to computed cachged result
819    loadtime --      Number of seconds it took to load cached result
820    compressed --    Flag (0,1) if cached results were compressed or not
821
822  DESCRIPTION:
823    Determine if cached result exists as follows:
824    Load in saved arguments and bytecode stored under hashed filename.
825    If they are identical to current arguments and bytecode and if dependencies
826    have not changed their time stamp, then return cached result.
827
828    Otherwise return filename under which new results should be cached.
829    Hash collisions are handled recursively by calling CacheLookup again with a
830    modified filename.
831  """
832
833  import time, string, types
834
835  # Assess whether cached result exists - compressed or not.
836  #
837  if verbose:
838    print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
839           %(CD+FN, file_types[0], file_types[1], file_types[2])
840  (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
841  (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
842  (admfile,compressed2) =  myopen(CD+FN+'_'+file_types[2],"rb",compression)
843
844  if verbose is True and deps is not None:
845    print 'Caching: Dependencies are', deps.keys()
846
847  if not (argsfile and datafile and admfile) or \
848     not (compressed0 == compressed1 and compressed0 == compressed2):
849    # Cached result does not exist or files were compressed differently
850    #
851    # This will ensure that evaluation will take place unless all files are
852    # present.
853
854    reason = 1
855    return(None,FN,None,reason,None,None,None) #Recompute using same filename
856
857  compressed = compressed0  # Remember if compressed files were actually used
858  datafile.close()
859
860  # Retrieve arguments and adm. info
861  #
862  R, reason = myload(argsfile, compressed)  # The original arguments
863  argsfile.close()
864   
865  if reason > 0:
866      # Recompute using same filename   
867      return(None, FN, None, reason, None, None, None)
868  else:   
869      (argsref, kwargsref) = R
870
871  R, reason = myload(admfile, compressed)
872  admfile.close() 
873
874  if reason > 0:
875    return(None,FN,None,reason,None,None,None) #Recompute using same filename
876
877 
878  depsref  = R[0]  # Dependency statistics
879  comptime = R[1]  # The computation time
880  coderef  = R[2]  # The byte code
881  funcname = R[3]  # The function name
882
883  # Check if dependencies have changed
884  #
885  if dependencies and not compare(depsref, deps):
886    if verbose:
887      print 'MESSAGE (caching.py): Dependencies', dependencies, \
888            'have changed - recomputing'
889    # Don't use cached file - recompute
890    reason = 2
891    return(None, FN, None, reason, None, None, None)
892
893  # Get bytecode from func
894  #
895  bytecode = get_bytecode(func)
896
897  #print compare(argsref,args),   
898  #print compare(kwargsref,kwargs),
899  #print compare(bytecode,coderef)
900
901  # Check if arguments or bytecode have changed
902  if compare(argsref, args) and compare(kwargsref, kwargs) and \
903     (not options['bytecode'] or compare(bytecode, coderef)):
904
905    # Arguments and dependencies match. Get cached results
906    T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
907    if reason > 0:
908        # Recompute using same FN     
909        return(None, FN, None, reason, None, None, None)
910
911    Retrieved = 1
912    reason = 0
913
914    if verbose:
915      msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
916
917      if loadtime >= comptime:
918        print 'WARNING (caching.py): Caching did not yield any gain.'
919        print '                      Consider executing function ',
920        print '('+funcname+') without caching.'
921  else:
922
923    # Non matching arguments or bytecodes signify a hash-collision.
924    # This is resolved by recursive search of cache filenames
925    # until either a matching or an unused filename is found.
926    #
927    (T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
928        CacheLookup(CD, FN+'x', func, args, kwargs, deps, 
929                    verbose, compression, dependencies)
930
931    # DEBUGGING
932    # if not Retrieved:
933    #   print 'Arguments did not match'
934    # else:
935    #   print 'Match found !'
936   
937    # The real reason is that args or bytecodes have changed.
938    # Not that the recursive seach has found an unused filename
939    if not Retrieved:
940      if not compare(bytecode, coderef):
941        reason = 4 # Bytecode has changed
942      else:   
943        reason = 3 # Arguments have changed
944       
945   
946  return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
947
948# -----------------------------------------------------------------------------
949
950def clear_cache(CD,func=None, verbose=None):
951  """Clear cache for func.
952
953  USAGE:
954     clear(CD, func, verbose)
955
956  ARGUMENTS:
957     CD --       Caching directory (required)
958     func --     Function object (default: None)
959     verbose --  Flag verbose output (default: None)
960
961  DESCRIPTION:
962
963    If func == None, clear everything,
964    otherwise clear only files pertaining to func.
965  """
966
967  import os, re
968   
969  if CD[-1] != os.sep:
970    CD = CD+os.sep
971 
972  if verbose == None:
973    verbose = options['verbose']
974
975  # FIXME: Windows version needs to be tested
976
977  if func:
978    funcname = get_funcname(func)
979    if verbose:
980      print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
981
982    file_names = os.listdir(CD)
983    for file_name in file_names:
984      #RE = re.search('^' + funcname,file_name)  #Inefficient
985      #if RE:
986      if file_name[:len(funcname)] == funcname:
987        if unix:
988          os.remove(CD+file_name)
989        else:
990          os.system('del '+CD+file_name)
991          # FIXME: os.remove doesn't work under windows
992  else:
993    file_names = os.listdir(CD)
994    if len(file_names) > 0:
995      if verbose:
996        print 'MESSAGE (caching.py): Remove the following files:'
997        for file_name in file_names:
998            print file_name
999
1000        A = raw_input('Delete (Y/N)[N] ?')
1001      else:
1002        A = 'Y' 
1003       
1004      if A == 'Y' or A == 'y':
1005        for file_name in file_names:
1006          if unix:
1007            os.remove(CD+file_name)
1008          else:
1009            os.system('del '+CD+file_name)
1010            # FIXME: os.remove doesn't work under windows
1011          #exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
1012
1013# -----------------------------------------------------------------------------
1014
1015def DeleteOldFiles(CD,verbose=None):
1016  """Remove expired files
1017
1018  USAGE:
1019    DeleteOldFiles(CD,verbose=None)
1020  """
1021
1022  if verbose == None:
1023    verbose = options['verbose']
1024
1025  maxfiles = options['maxfiles']
1026
1027  # FIXME: Windows version
1028
1029  import os
1030  block = 1000  # How many files to delete per invokation
1031  Files = os.listdir(CD)
1032  numfiles = len(Files)
1033  if not unix: return  # FIXME: Windows case ?
1034
1035  if numfiles > maxfiles:
1036    delfiles = numfiles-maxfiles+block
1037    if verbose:
1038      print 'Deleting '+`delfiles`+' expired files:'
1039      os.system('ls -lur '+CD+'* | head -' + `delfiles`)            # List them
1040    os.system('ls -ur '+CD+'* | head -' + `delfiles` + ' | xargs /bin/rm')
1041                                                                  # Delete them
1042    # FIXME: Replace this with os.listdir and os.remove
1043
1044# -----------------------------------------------------------------------------
1045
1046def save_args_to_cache(CD, FN, args, kwargs, compression):
1047  """Save arguments to cache
1048
1049  USAGE:
1050    save_args_to_cache(CD,FN,args,kwargs,compression)
1051  """
1052
1053  import time, os, sys, types
1054
1055  (argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1056
1057  if argsfile is None:
1058    msg = 'ERROR (caching): Could not open argsfile for writing: %s' %FN
1059    raise IOError, msg
1060
1061  mysave((args,kwargs),argsfile,compression)  # Save args and kwargs to cache
1062  argsfile.close()
1063
1064  # Change access rights if possible
1065  #
1066  #if unix:
1067  #  try:
1068  #    exitcode=os.system('chmod 666 '+argsfile.name)
1069  #  except:
1070  #    pass
1071  #else:
1072  #  pass  # FIXME: Take care of access rights under Windows
1073
1074  return
1075
1076# -----------------------------------------------------------------------------
1077
1078def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1079                          dependencies, compression):
1080  """Save computed results T and admin info to cache
1081
1082  USAGE:
1083    save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1084                          dependencies, compression)
1085  """
1086
1087  import time, os, sys, types
1088
1089  (datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1090  (admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1091
1092  if not datafile:
1093    if verbose:
1094      print 'ERROR (caching): Could not open %s' %datafile.name
1095    raise IOError
1096
1097  if not admfile:
1098    if verbose:
1099      print 'ERROR (caching): Could not open %s' %admfile.name
1100    raise IOError
1101
1102  t0 = time.time()
1103
1104  mysave(T,datafile,compression)  # Save data to cache
1105  datafile.close()
1106  #savetime = round(time.time()-t0,2)
1107  savetime = time.time()-t0 
1108
1109  bytecode = get_bytecode(func)  # Get bytecode from function object
1110  admtup = (deps, comptime, bytecode, funcname)  # Gather admin info
1111
1112  mysave(admtup,admfile,compression)  # Save admin info to cache
1113  admfile.close()
1114
1115  # Change access rights if possible
1116  #
1117  #if unix:
1118  #  try:
1119  #    exitcode=os.system('chmod 666 '+datafile.name)
1120  #    exitcode=os.system('chmod 666 '+admfile.name)
1121  #  except:
1122  #    pass
1123  #else:
1124  #  pass  # FIXME: Take care of access rights under Windows
1125
1126  return(savetime)
1127
1128# -----------------------------------------------------------------------------
1129
1130def load_from_cache(CD, FN, compression):
1131  """Load previously cached data from file FN
1132
1133  USAGE:
1134    load_from_cache(CD,FN,compression)
1135  """
1136
1137  import time
1138
1139  (datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1140  t0 = time.time()
1141  T, reason = myload(datafile,compressed)
1142
1143  loadtime = time.time()-t0
1144  datafile.close() 
1145
1146  return T, loadtime, compressed, reason
1147
1148# -----------------------------------------------------------------------------
1149
1150def myopen(FN, mode, compression=True):
1151  """Open file FN using given mode
1152
1153  USAGE:
1154    myopen(FN, mode, compression=True)
1155
1156  ARGUMENTS:
1157    FN --           File name to be opened
1158    mode --         Open mode (as in open)
1159    compression --  Flag zlib compression
1160
1161  DESCRIPTION:
1162     if compression
1163       Attempt first to open FN + '.z'
1164       If this fails try to open FN
1165     else do the opposite
1166     Return file handle plus info about whether it was compressed or not.
1167  """
1168
1169  import string
1170
1171  # Determine if file exists already (if writing was requested)
1172  # This info is only used to determine if access modes should be set
1173  #
1174  if 'w' in mode or 'a' in mode:
1175    try:
1176      file = open(FN+'.z','r')
1177      file.close()
1178      new_file = 0
1179    except:
1180      try:
1181        file = open(FN,'r') 
1182        file.close()
1183        new_file = 0
1184      except:
1185        new_file = 1
1186  else:
1187    new_file = 0 #Assume it exists if mode was not 'w'
1188 
1189
1190  compressed = 0
1191  if compression:
1192    try:
1193      file = open(FN+'.z',mode)
1194      compressed = 1
1195    except:
1196      try:
1197        file = open(FN,mode)
1198      except:
1199        file = None
1200  else:
1201    try:
1202      file = open(FN,mode)
1203    except:
1204      try:
1205        file = open(FN+'.z',mode)
1206        compressed = 1
1207      except:
1208        file = None
1209
1210  # Now set access rights if it is a new file
1211  #
1212  if file and new_file:
1213    if unix:
1214      exitcode=os.system('chmod 666 '+file.name)
1215    else:
1216      pass  # FIXME: Take care of access rights under Windows
1217
1218  return(file,compressed)
1219
1220# -----------------------------------------------------------------------------
1221
1222def myload(file, compressed):
1223  """Load data from file
1224
1225  USAGE:
1226    myload(file, compressed)
1227  """
1228
1229  reason = 0
1230  try:
1231    if compressed:
1232      import zlib
1233
1234      RsC = file.read()
1235      try:
1236        Rs  = zlib.decompress(RsC)
1237      except:
1238        #  File "./caching.py", line 1032, in load_from_cache
1239        #  T = myload(datafile,compressed)
1240        #  File "./caching.py", line 1124, in myload
1241        #  Rs  = zlib.decompress(RsC)
1242        #  zlib.error: Error -5 while decompressing data
1243        #print 'ERROR (caching): Could not decompress ', file.name
1244        #raise Exception
1245        reason = 6  # Unreadable file
1246        return None, reason 
1247     
1248     
1249      del RsC  # Free up some space
1250      R = pickler.loads(Rs)
1251    else:
1252      try:
1253        R = pickler.load(file)
1254      #except EOFError, e:
1255      except:
1256        #Catch e.g., file with 0 length or corrupted
1257        reason = 6  # Unreadable file
1258        return None, reason
1259     
1260  except MemoryError:
1261    import sys
1262    if options['verbose']:
1263      print 'ERROR (caching): Out of memory while loading %s, aborting' \
1264            %(file.name)
1265
1266    # Raise the error again for now
1267    #
1268    raise MemoryError
1269
1270  return R, reason
1271
1272# -----------------------------------------------------------------------------
1273
1274def mysave(T, file, compression):
1275  """Save data T to file
1276
1277  USAGE:
1278    mysave(T, file, compression)
1279
1280  """
1281
1282  bin = options['bin']
1283
1284  if compression:
1285    try:
1286      import zlib
1287    except:
1288      print
1289      print '*** Could not find zlib ***'
1290      print '*** Try to run caching with compression off ***'
1291      print "*** caching.set_option('compression', 0) ***"
1292      raise Exception
1293     
1294
1295    try:
1296      Ts  = pickler.dumps(T, bin)
1297    except MemoryError:
1298      msg = '****WARNING (caching.py): Could not pickle data for compression.'
1299      msg += ' Try using compression = False'
1300      raise MemoryError, msg
1301    else: 
1302      # Compressed pickling     
1303      TsC = zlib.compress(Ts, comp_level)
1304      file.write(TsC)
1305  else:
1306      # Uncompressed pickling
1307      pickler.dump(T, file, bin)
1308
1309      # FIXME: This may not work on Windoze network drives.
1310      # The error msg is IOError: [Errno 22] Invalid argument
1311      # Testing with small files was OK, though.
1312      # I think this is an OS problem.
1313
1314      # Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1315     
1316# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1317#
1318#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1319#
1320#The following extract is taken from the site http://www.python.org:
1321#
1322#---------------------------------------------------------------------------------------------
1323#exception IOError
1324#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1325#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1326#---------------------------------------------------------------------------------------------
1327#
1328#The error code(s) that accompany exceptions are described at:
1329#http://www.python.org/dev/doc/devel//lib/module-errno.html
1330#
1331#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1332#
1333#exceptions.IOError invalid argument Errno 22
1334       
1335      #try:
1336      #  pickler.dump(T,file,bin)
1337      #except IOError, e:
1338      #  print e
1339      #  msg = 'Could not store to %s, bin=%s' %(file, bin)
1340      #  raise msg
1341     
1342
1343# -----------------------------------------------------------------------------
1344
1345   
1346def myhash(T, ids=None):
1347  """Compute hashed integer from a range of inputs.
1348  If T is not hashable being e.g. a tuple T, myhash will recursively
1349  hash the values individually
1350
1351  USAGE:
1352    myhash(T)
1353
1354  ARGUMENTS:
1355    T -- Anything
1356  """
1357
1358  from types import TupleType, ListType, DictType, InstanceType 
1359   
1360  if type(T) in [TupleType, ListType, DictType, InstanceType]: 
1361      # Keep track of unique id's to protect against infinite recursion
1362      if ids is None: ids = []
1363
1364      # Check if T has already been encountered
1365      i = id(T) 
1366 
1367      if i in ids:
1368          return 0 # T has been hashed already     
1369      else:
1370          ids.append(i)
1371   
1372
1373   
1374  # Start hashing 
1375 
1376  # On some architectures None, False and True gets different hash values
1377  if T is None:
1378      return(-1)
1379  if T is False:
1380      return(0)
1381  if T is True:
1382      return(1)
1383
1384  # Get hash values for hashable entries
1385  if type(T) in [TupleType, ListType]:
1386      hvals = []
1387      for t in T:
1388          h = myhash(t, ids)
1389          hvals.append(h)
1390      val = hash(tuple(hvals))
1391  elif type(T) == DictType:
1392      # Make dictionary ordering unique 
1393      I = T.items()
1394      I.sort()   
1395      val = myhash(I, ids)
1396  elif isinstance(T, num.ndarray):
1397      T = num.array(T) # Ensure array is contiguous
1398
1399      # Use mean value for efficiency
1400      val = hash(num.average(T.flat))
1401  elif type(T) == InstanceType:
1402      # Use the attribute values
1403      val = myhash(T.__dict__, ids)
1404  else:
1405      try:
1406          val = hash(T)
1407      except:
1408          val = 1
1409
1410  return(val)
1411
1412
1413
1414def compare(A, B, ids=None):
1415    """Safe comparison of general objects
1416
1417    USAGE:
1418      compare(A,B)
1419
1420    DESCRIPTION:
1421      Return 1 if A and B they are identical, 0 otherwise
1422    """
1423
1424    from types import TupleType, ListType, DictType, InstanceType
1425   
1426    # Keep track of unique id's to protect against infinite recursion
1427    if ids is None: ids = {}
1428
1429    # Check if T has already been encountered
1430    iA = id(A) 
1431    iB = id(B)     
1432   
1433    if ids.has_key((iA, iB)):
1434        # A and B have been compared already
1435        #print 'Found', (iA, iB), A, B
1436        return ids[(iA, iB)]
1437    else:
1438        ids[(iA, iB)] = True
1439   
1440   
1441    # Check if arguments are of same type
1442    if type(A) != type(B):
1443        return False
1444       
1445 
1446    # Compare recursively based on argument type
1447    if type(A) in [TupleType, ListType]:
1448        N = len(A)
1449        if len(B) != N: 
1450            identical = False
1451        else:
1452            identical = True
1453            for i in range(N):
1454                if not compare(A[i], B[i], ids): 
1455                    identical = False; break
1456                   
1457    elif type(A) == DictType:
1458        if len(A) != len(B):
1459            identical = False
1460        else:                       
1461            # Make dictionary ordering unique
1462            a = A.items(); a.sort()   
1463            b = B.items(); b.sort()
1464           
1465            identical = compare(a, b, ids)
1466           
1467    elif isinstance(A, num.ndarray):
1468        # Use element by element comparison
1469        identical = num.alltrue(A==B)
1470
1471    elif type(A) == InstanceType:
1472        # Take care of special case where elements are instances           
1473        # Base comparison on attributes     
1474        identical = compare(A.__dict__, 
1475                            B.__dict__, 
1476                            ids)
1477    else:       
1478        # Fall back to general code
1479        try:
1480            identical = (A == B)
1481        except:
1482            import pickle
1483            # Use pickle to compare data
1484            # The native pickler must be used
1485            # since the faster cPickle does not
1486            # guarantee a unique translation           
1487            try:
1488                identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
1489            except:
1490                identical = False
1491
1492    # Record result of comparison and return           
1493    ids[(iA, iB)] = identical
1494   
1495    return(identical)
1496
1497   
1498# -----------------------------------------------------------------------------
1499
1500def nospace(s):
1501  """Replace spaces in string s with underscores
1502
1503  USAGE:
1504    nospace(s)
1505
1506  ARGUMENTS:
1507    s -- string
1508  """
1509
1510  import string
1511
1512  newstr = ''
1513  for i in range(len(s)):
1514    if s[i] == ' ':
1515      newstr = newstr+'_'
1516    else:
1517      newstr = newstr+s[i]
1518
1519  return(newstr)
1520
1521# -----------------------------------------------------------------------------
1522
1523def get_funcname(func):
1524  """Retrieve name of function object func (depending on its type)
1525
1526  USAGE:
1527    get_funcname(func)
1528  """
1529
1530  import types, string
1531
1532  if type(func) == types.FunctionType:
1533    funcname = func.func_name
1534  elif type(func) == types.BuiltinFunctionType:
1535    funcname = func.__name__
1536  else:
1537    tab = string.maketrans("<>'","   ")
1538    tmp = string.translate(`func`,tab)
1539    tmp = string.split(tmp)
1540    funcname = string.join(tmp)
1541
1542    # Truncate memory address as in
1543    # class __main__.Dummy at 0x00A915D0
1544    index = funcname.find('at 0x')
1545    if index >= 0:
1546      funcname = funcname[:index+5] # Keep info that there is an address
1547
1548  funcname = nospace(funcname)
1549  return(funcname)
1550
1551# -----------------------------------------------------------------------------
1552
1553def get_bytecode(func):
1554  """ Get bytecode from function object.
1555
1556  USAGE:
1557    get_bytecode(func)
1558  """
1559
1560  import types
1561
1562  if type(func) == types.FunctionType:
1563    return get_func_code_details(func)
1564  elif type(func) == types.MethodType:
1565    return get_func_code_details(func.im_func)
1566  elif type(func) == types.InstanceType:   
1567    if callable(func):
1568      # Get bytecode from __call__ method
1569      bytecode = get_func_code_details(func.__call__.im_func)
1570     
1571      # Add hash value of object to detect attribute changes
1572      return bytecode + (myhash(func),) 
1573    else:
1574      msg = 'Instance %s was passed into caching in the role of a function ' % str(func)
1575      msg = ' but it was not callable.'
1576      raise Exception, msg
1577  elif type(func) in [types.BuiltinFunctionType, types.BuiltinMethodType]:     
1578    # Built-in functions are assumed not to change 
1579    return None, 0, 0, 0
1580  elif type(func) == types.ClassType:
1581      # Get bytecode from __init__ method
1582      bytecode = get_func_code_details(func.__init__.im_func)   
1583      return bytecode     
1584  else:
1585    msg = 'Unknown function type: %s' % type(func)
1586    raise Exception, msg
1587
1588
1589 
1590 
1591def get_func_code_details(func):
1592  """Extract co_code, co_consts, co_argcount, func_defaults
1593  """
1594 
1595  bytecode = func.func_code.co_code
1596  consts = func.func_code.co_consts
1597  argcount = func.func_code.co_argcount   
1598  defaults = func.func_defaults       
1599 
1600  return bytecode, consts, argcount, defaults 
1601
1602# -----------------------------------------------------------------------------
1603
1604def get_depstats(dependencies):
1605  """ Build dictionary of dependency files and their size, mod. time and ctime.
1606
1607  USAGE:
1608    get_depstats(dependencies):
1609  """
1610
1611  import types
1612
1613  #print 'Caching DEBUG: Dependencies are', dependencies
1614  d = {}
1615  if dependencies:
1616
1617    #Expand any wildcards
1618    import glob
1619    expanded_dependencies = []
1620    for FN in dependencies:
1621      expanded_FN = glob.glob(FN)
1622
1623      if expanded_FN == []:
1624        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1625        raise Exception, errmsg     
1626
1627      expanded_dependencies += expanded_FN
1628
1629   
1630    for FN in expanded_dependencies:
1631      if not type(FN) == types.StringType:
1632        errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1633        errmsg += '                   Dependency given: %s' %FN
1634        raise Exception, errmsg     
1635      if not os.access(FN,os.F_OK):
1636        errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1637        raise Exception, errmsg
1638      (size,atime,mtime,ctime) = filestat(FN)
1639
1640      # We don't use atime because that would cause recomputation every time.
1641      # We don't use ctime because that is irrelevant and confusing for users.
1642      d.update({FN : (size,mtime)})
1643
1644  return(d)
1645
1646# -----------------------------------------------------------------------------
1647
1648def filestat(FN):
1649  """A safe wrapper using os.stat to get basic file statistics
1650     The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1651
1652  USAGE:
1653    filestat(FN)
1654
1655  DESCRIPTION:
1656     Must compile Python with
1657     CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1658              configure
1659     as given in section 8.1.1 Large File Support in the Libray Reference
1660  """
1661
1662  import os, time
1663
1664  try:
1665    stats = os.stat(FN)
1666    size  = stats[6]
1667    atime = stats[7]
1668    mtime = stats[8]
1669    ctime = stats[9]
1670  except:
1671
1672    # Hack to get the results anyway (works only on Unix at the moment)
1673    #
1674    print 'Hack to get os.stat when files are too large'
1675
1676    if unix:
1677      tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1678      # Unique filename, FIXME: Use random number
1679
1680      # Get size and access time (atime)
1681      #
1682      exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1683      (size,atime) = get_lsline(tmp)
1684
1685      # Get size and modification time (mtime)
1686      #
1687      exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1688      (size,mtime) = get_lsline(tmp)
1689
1690      # Get size and ctime
1691      #
1692      exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1693      (size,ctime) = get_lsline(tmp)
1694
1695      try:
1696        exitcode=os.system('rm '+tmp)
1697        # FIXME: Gives error if file doesn't exist
1698      except:
1699        pass
1700    else:
1701      pass
1702      raise Exception  # FIXME: Windows case
1703
1704  return(long(size),atime,mtime,ctime)
1705
1706# -----------------------------------------------------------------------------
1707
1708def get_lsline(FN):
1709  """get size and time for filename
1710
1711  USAGE:
1712    get_lsline(file_name)
1713
1714  DESCRIPTION:
1715    Read in one line 'ls -la' item from file (generated by filestat) and
1716    convert time to seconds since epoch. Return file size and time.
1717  """
1718
1719  import string, time
1720
1721  f = open(FN,'r')
1722  info = f.read()
1723  info = string.split(info)
1724
1725  size = info[4]
1726  week = info[5]
1727  mon  = info[6]
1728  day  = info[7]
1729  hour = info[8]
1730  year = info[9]
1731
1732  str = week+' '+mon+' '+day+' '+hour+' '+year
1733  timetup = time.strptime(str)
1734  t = time.mktime(timetup)
1735  return(size, t)
1736
1737# -----------------------------------------------------------------------------
1738
1739def checkdir(CD, verbose=None, warn=False):
1740  """Check or create caching directory
1741
1742  USAGE:
1743    checkdir(CD,verbose):
1744
1745  ARGUMENTS:
1746    CD -- Directory
1747    verbose -- Flag verbose output (default: None)
1748
1749  DESCRIPTION:
1750    If CD does not exist it will be created if possible
1751  """
1752
1753  import os
1754  import os.path
1755
1756  if CD[-1] != os.sep: 
1757    CD = CD + os.sep  # Add separator for directories
1758
1759  CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1760  if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1761    try:
1762      exitcode=os.mkdir(CD)
1763
1764      # Change access rights if possible
1765      #
1766      if unix:
1767        exitcode=os.system('chmod 777 '+CD)
1768      else:
1769        pass  # FIXME: What about acces rights under Windows?
1770      if verbose: print 'MESSAGE: Directory', CD, 'created.'
1771    except:
1772      if warn is True:
1773        print 'WARNING: Directory', CD, 'could not be created.'
1774      if unix:
1775        CD = '/tmp/'
1776      else:
1777        CD = 'C:' 
1778      if warn is True:
1779        print 'Using directory %s instead' %CD
1780
1781  return(CD)
1782
1783checkdir(cachedir, warn=True)
1784
1785#==============================================================================
1786# Statistics
1787#==============================================================================
1788
1789def addstatsline(CD, funcname, FN, Retrieved, reason, comptime, loadtime,
1790                 compression):
1791  """Add stats entry
1792
1793  USAGE:
1794    addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1795
1796  DESCRIPTION:
1797    Make one entry in the stats file about one cache hit recording time saved
1798    and other statistics. The data are used by the function cachestat.
1799  """
1800
1801  import os, time
1802
1803  try:
1804    TimeTuple = time.localtime(time.time())
1805    extension = time.strftime('%b%Y',TimeTuple)
1806    SFN = CD+statsfile+'.'+extension
1807    #statfile = open(SFN,'a')
1808    (statfile, dummy) = myopen(SFN,'a',compression=0)
1809
1810    # Change access rights if possible
1811    #
1812    #if unix:
1813    #  try:
1814    #    exitcode=os.system('chmod 666 '+SFN)
1815    #  except:
1816    #    pass
1817  except:
1818    print 'Warning: Stat file could not be opened'
1819
1820  try:
1821    if os.environ.has_key('USER'):
1822      user = os.environ['USER']
1823    else:
1824      user = 'Nobody'
1825
1826    date = time.asctime(TimeTuple)
1827
1828    if Retrieved:
1829      hit = '1'
1830    else:
1831      hit = '0'
1832
1833    # Get size of result file
1834    #   
1835    if compression:
1836      stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1837    else:
1838      stats = os.stat(CD+FN+'_'+file_types[0])
1839 
1840    if stats: 
1841      size = stats[6]
1842    else:
1843      size = -1  # Error condition, but don't crash. This is just statistics 
1844
1845    # Build entry
1846    # 
1847    entry = date             + ',' +\
1848            user             + ',' +\
1849            FN               + ',' +\
1850            str(int(size))   + ',' +\
1851            str(compression) + ',' +\
1852            hit              + ',' +\
1853            str(reason)      + ',' +\
1854            str(round(comptime,4)) + ',' +\
1855            str(round(loadtime,4)) +\
1856            CR
1857           
1858    statfile.write(entry)
1859    statfile.close()
1860  except:
1861    print 'Warning: Writing of stat file failed'
1862
1863# -----------------------------------------------------------------------------
1864
1865# FIXME: should take cachedir as an optional arg
1866#
1867def __cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
1868  """  List caching statistics.
1869
1870  USAGE:
1871    __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1872
1873      Generate statistics of caching efficiency.
1874      The parameter sortidx determines by what field lists are sorted.
1875      If the optional keyword period is set to -1,
1876      all available caching history is used.
1877      If it is 0 only the current month is used.
1878      Future versions will include more than one month....
1879      OMN 20/8/2000
1880  """
1881
1882  import os
1883  import os.path
1884  from string import split, rstrip, find
1885  from time import strptime, localtime, strftime, mktime, ctime
1886
1887  # sortidx = 4    # Index into Fields[1:]. What to sort by.
1888
1889  Fields = ['Name', 'Hits', 'Exec(s)', \
1890            'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1891  Widths = [25,7,9,9,9,9,13]
1892  #Types = ['s','d','d','d','d','.2f','d']
1893  Types = ['s','d','.2f','.2f','.2f','.2f','d'] 
1894
1895  Dictnames = ['Function', 'User']
1896
1897  if not cachedir:
1898    cachedir = checkdir(options['cachedir'])
1899
1900  SD = os.path.expanduser(cachedir)  # Expand ~ or ~user in pathname
1901
1902  if period == -1:  # Take all available stats
1903    SFILENAME = statsfile
1904  else:  # Only stats from current month 
1905       # MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1906    TimeTuple = localtime(time())
1907    extension = strftime('%b%Y',TimeTuple)
1908    SFILENAME = statsfile+'.'+extension
1909
1910  DIRLIST = os.listdir(SD)
1911  SF = []
1912  for FN in DIRLIST:
1913    if find(FN,SFILENAME) >= 0:
1914      SF.append(FN)
1915
1916  blocksize = 15000000
1917  total_read = 0
1918  total_hits = 0
1919  total_discarded = 0
1920  firstday = mktime(strptime('2030','%Y'))
1921             # FIXME: strptime don't exist in WINDOWS ?
1922  lastday = 0
1923
1924  FuncDict = {}
1925  UserDict = {}
1926  for FN in SF:
1927    input = open(SD+FN,'r')
1928    print 'Reading file ', SD+FN
1929
1930    while True:
1931      A = input.readlines(blocksize)
1932      if len(A) == 0: break
1933      total_read = total_read + len(A)
1934      for record in A:
1935        record = tuple(split(rstrip(record),','))
1936        #print record, len(record)
1937
1938        if len(record) == 9:
1939          timestamp = record[0]
1940       
1941          try:
1942            t = mktime(strptime(timestamp))
1943          except:
1944            total_discarded = total_discarded + 1         
1945            continue   
1946             
1947          if t > lastday:
1948            lastday = t
1949          if t < firstday:
1950            firstday = t
1951
1952          user     = record[1]
1953          func     = record[2]
1954
1955          # Strip hash-stamp off
1956          #
1957          i = find(func,'[')
1958          func = func[:i]
1959
1960          size        = float(record[3])
1961
1962          # Compression kepword can be Boolean
1963          if record[4] in ['True', '1']:
1964            compression = 1
1965          elif record[4] in ['False', '0']: 
1966            compression = 0
1967          else:
1968            print 'Unknown value of compression', record[4]
1969            print record
1970            total_discarded = total_discarded + 1           
1971            continue
1972
1973          #compression = int(record[4]) # Can be Boolean
1974          hit         = int(record[5])
1975          reason      = int(record[6])   # Not used here   
1976          cputime     = float(record[7])
1977          loadtime    = float(record[8])
1978
1979          if hit:
1980            total_hits = total_hits + 1
1981            saving = cputime-loadtime
1982
1983            if cputime != 0:
1984              rel_saving = round(100.0*saving/cputime,2)
1985            else:
1986              #rel_saving = round(1.0*saving,2)
1987              rel_saving = 100.0 - round(1.0*saving,2)  # A bit of a hack
1988
1989            info = [1,cputime,loadtime,saving,rel_saving,size]
1990
1991            UpdateDict(UserDict,user,info)
1992            UpdateDict(FuncDict,func,info)
1993          else:
1994            pass #Stats on recomputations and their reasons could go in here
1995             
1996        else:
1997          #print 'Record discarded'
1998          #print record
1999          total_discarded = total_discarded + 1
2000
2001    input.close()
2002
2003  # Compute averages of all sums and write list
2004  #
2005
2006  if total_read == 0:
2007    printline(Widths,'=')
2008    print 'CACHING STATISTICS: No valid records read'
2009    printline(Widths,'=')
2010    return
2011
2012  print
2013  printline(Widths,'=')
2014  print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
2015  printline(Widths,'=')
2016  #print '  Period:', ctime(firstday), 'to', ctime(lastday)
2017  print '  Total number of valid records', total_read
2018  print '  Total number of discarded records', total_discarded
2019  print '  Total number of hits', total_hits
2020  print
2021
2022  print '  Fields', Fields[2:], 'are averaged over number of hits'
2023  print '  Time is measured in seconds and size in bytes'
2024  print '  Tables are sorted by', Fields[1:][sortidx]
2025
2026  # printline(Widths,'-')
2027
2028  if showuser:
2029    Dictionaries = [FuncDict, UserDict]
2030  else:
2031    Dictionaries = [FuncDict]
2032
2033  i = 0
2034  for Dict in Dictionaries:
2035    for key in Dict.keys():
2036      rec = Dict[key]
2037      for n in range(len(rec)):
2038        if n > 0:
2039          rec[n] = round(1.0*rec[n]/rec[0],2)
2040      Dict[key] = rec
2041
2042    # Sort and output
2043    #
2044    keylist = SortDict(Dict,sortidx)
2045
2046    # Write Header
2047    #
2048    print
2049    #print Dictnames[i], 'statistics:'; i=i+1
2050    printline(Widths,'-')
2051    n = 0
2052    for s in Fields:
2053      if s == Fields[0]:  # Left justify
2054        s = Dictnames[i] + ' ' + s; i=i+1
2055        exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
2056      else:
2057        exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
2058    print
2059    printline(Widths,'-')
2060
2061    # Output Values
2062    #
2063    for key in keylist:
2064      rec = Dict[key]
2065      n = 0
2066      if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
2067      exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
2068      for val in rec:
2069        exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
2070      print
2071    print
2072
2073#==============================================================================
2074# Auxiliary stats functions
2075#==============================================================================
2076
2077def UpdateDict(Dict, key, info):
2078  """Update dictionary by adding new values to existing.
2079
2080  USAGE:
2081    UpdateDict(Dict,key,info)
2082  """
2083
2084  if Dict.has_key(key):
2085    dinfo = Dict[key]
2086    for n in range(len(dinfo)):
2087      dinfo[n] = info[n] + dinfo[n]
2088  else:
2089    dinfo = info[:]  # Make a copy of info list
2090
2091  Dict[key] = dinfo
2092  return Dict
2093
2094# -----------------------------------------------------------------------------
2095
2096def SortDict(Dict, sortidx=0):
2097  """Sort dictionary
2098
2099  USAGE:
2100    SortDict(Dict,sortidx):
2101
2102  DESCRIPTION:
2103    Sort dictionary of lists according field number 'sortidx'
2104  """
2105
2106  import types
2107
2108  sortlist  = []
2109  keylist = Dict.keys()
2110  for key in keylist:
2111    rec = Dict[key]
2112    if not type(rec) in [types.ListType, types.TupleType]:
2113      rec = [rec]
2114
2115    if sortidx > len(rec)-1:
2116      if options['verbose']:
2117        print 'ERROR: Sorting index to large, sortidx = ', sortidx
2118      raise IndexError
2119
2120    val = rec[sortidx]
2121    sortlist.append(val)
2122
2123  A = map(None,sortlist,keylist)
2124  A.sort()
2125  keylist = map(lambda x: x[1], A)  # keylist sorted by sortidx
2126
2127  return(keylist)
2128
2129# -----------------------------------------------------------------------------
2130
2131def printline(Widths,char):
2132  """Print textline in fixed field.
2133
2134  USAGE:
2135    printline(Widths,char)
2136  """
2137
2138  s = ''
2139  for n in range(len(Widths)):
2140    s = s+Widths[n]*char
2141    if n > 0:
2142      s = s+char
2143
2144  print s
2145
2146#==============================================================================
2147# Messages
2148#==============================================================================
2149
2150def msg1(funcname, args, kwargs, reason):
2151  """Message 1
2152
2153  USAGE:
2154    msg1(funcname, args, kwargs, reason):
2155  """
2156
2157  import string
2158  #print 'MESSAGE (caching.py): Evaluating function', funcname,
2159
2160  print_header_box('Evaluating function %s' %funcname)
2161 
2162  msg7(args, kwargs)
2163  msg8(reason) 
2164 
2165  print_footer()
2166 
2167  #
2168  # Old message
2169  #
2170  #args_present = 0
2171  #if args:
2172  #  if len(args) == 1:
2173  #    print 'with argument', mkargstr(args[0], textwidth2),
2174  #  else:
2175  #    print 'with arguments', mkargstr(args, textwidth2),
2176  #  args_present = 1     
2177  #   
2178  #if kwargs:
2179  #  if args_present:
2180  #    word = 'and'
2181  #  else:
2182  #    word = 'with'
2183  #     
2184  #  if len(kwargs) == 1:
2185  #    print word + ' keyword argument', mkargstr(kwargs, textwidth2)
2186  #  else:
2187  #    print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
2188  #  args_present = 1           
2189  #else:
2190  #  print    # Newline when no keyword args present
2191  #       
2192  #if not args_present:   
2193  #  print '',  # Default if no args or kwargs present
2194   
2195   
2196
2197# -----------------------------------------------------------------------------
2198
2199def msg2(funcname, args, kwargs, comptime, reason):
2200  """Message 2
2201
2202  USAGE:
2203    msg2(funcname, args, kwargs, comptime, reason)
2204  """
2205
2206  import string
2207
2208  #try:
2209  #  R = Reason_msg[reason]
2210  #except:
2211  #  R = 'Unknown reason' 
2212 
2213  #print_header_box('Caching statistics (storing) - %s' %R)
2214  print_header_box('Caching statistics (storing)') 
2215 
2216  msg6(funcname,args,kwargs)
2217  msg8(reason)
2218
2219  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2220
2221# -----------------------------------------------------------------------------
2222
2223def msg3(savetime, CD, FN, deps, compression):
2224  """Message 3
2225
2226  USAGE:
2227    msg3(savetime, CD, FN, deps, compression)
2228  """
2229
2230  import string
2231  print string.ljust('| Loading time:', textwidth1) + str(round(savetime,2)) + \
2232                     ' seconds (estimated)'
2233  msg5(CD,FN,deps,compression)
2234
2235# -----------------------------------------------------------------------------
2236
2237def msg4(funcname, args, kwargs, deps, comptime, loadtime, CD, FN, compression):
2238  """Message 4
2239
2240  USAGE:
2241    msg4(funcname, args, kwargs, deps, comptime, loadtime, CD, FN, compression)
2242  """
2243
2244  import string
2245
2246  print_header_box('Caching statistics (retrieving)')
2247 
2248  msg6(funcname,args,kwargs)
2249  print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2250  print string.ljust('| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
2251  print string.ljust('| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
2252        ' seconds'
2253  msg5(CD,FN,deps,compression)
2254
2255# -----------------------------------------------------------------------------
2256
2257def msg5(CD, FN, deps, compression):
2258  """Message 5
2259
2260  USAGE:
2261    msg5(CD, FN, deps, compression)
2262
2263  DESCRIPTION:
2264   Print dependency stats. Used by msg3 and msg4
2265  """
2266
2267  import os, time, string
2268
2269  print '|'
2270  print string.ljust('| Caching dir: ', textwidth1) + CD
2271
2272  if compression:
2273    suffix = '.z'
2274    bytetext = 'bytes, compressed'
2275  else:
2276    suffix = ''
2277    bytetext = 'bytes'
2278
2279  for file_type in file_types:
2280    file_name = FN + '_' + file_type + suffix
2281    print string.ljust('| ' + file_type + ' file: ', textwidth1) + file_name,
2282    stats = os.stat(CD+file_name)
2283    print '('+ str(stats[6]) + ' ' + bytetext + ')'
2284
2285  print '|'
2286  if len(deps) > 0:
2287    print '| Dependencies:  '
2288    dependencies  = deps.keys()
2289    dlist = []; maxd = 0
2290    tlist = []; maxt = 0
2291    slist = []; maxs = 0
2292    for d in dependencies:
2293      stats = deps[d]
2294      t = time.ctime(stats[1])
2295      s = str(stats[0])
2296      #if s[-1] == 'L':
2297      #  s = s[:-1]  # Strip rightmost 'long integer' L off.
2298      #              # FIXME: Unnecessary in versions later than 1.5.2
2299
2300      if len(d) > maxd: maxd = len(d)
2301      if len(t) > maxt: maxt = len(t)
2302      if len(s) > maxs: maxs = len(s)
2303      dlist.append(d)
2304      tlist.append(t)
2305      slist.append(s)
2306
2307    for n in range(len(dlist)):
2308      d = string.ljust(dlist[n]+':', maxd+1)
2309      t = string.ljust(tlist[n], maxt)
2310      s = string.rjust(slist[n], maxs)
2311
2312      print '| ', d, t, ' ', s, 'bytes'
2313  else:
2314    print '| No dependencies'
2315  print_footer()
2316
2317# -----------------------------------------------------------------------------
2318
2319def msg6(funcname, args, kwargs):
2320  """Message 6
2321
2322  USAGE:
2323    msg6(funcname, args, kwargs)
2324  """
2325
2326  import string
2327  print string.ljust('| Function:', textwidth1) + funcname
2328
2329  msg7(args, kwargs)
2330 
2331# -----------------------------------------------------------------------------   
2332
2333def msg7(args, kwargs):
2334  """Message 7
2335 
2336  USAGE:
2337    msg7(args, kwargs):
2338  """
2339 
2340  import string
2341 
2342  args_present = 0 
2343  if args:
2344    if len(args) == 1:
2345      print string.ljust('| Argument:', textwidth1) + mkargstr(args[0], \
2346                         textwidth2)
2347    else:
2348      print string.ljust('| Arguments:', textwidth1) + \
2349            mkargstr(args, textwidth2)
2350    args_present = 1
2351           
2352  if kwargs:
2353    if len(kwargs) == 1:
2354      print string.ljust('| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
2355                         textwidth2)
2356    else:
2357      print string.ljust('| Keyword Args:', textwidth1) + \
2358            mkargstr(kwargs, textwidth2)
2359    args_present = 1
2360
2361  if not args_present:               
2362    print '| No arguments' # Default if no args or kwargs present
2363
2364# -----------------------------------------------------------------------------
2365
2366def msg8(reason):
2367  """Message 8
2368 
2369  USAGE:
2370    msg8(reason):
2371  """
2372 
2373  import string
2374   
2375  try:
2376    R = Reason_msg[reason]
2377  except:
2378    R = 'Unknown' 
2379 
2380  print string.ljust('| Reason:', textwidth1) + R
2381   
2382# -----------------------------------------------------------------------------
2383
2384def print_header_box(line):
2385  """Print line in a nice box.
2386 
2387  USAGE:
2388    print_header_box(line)
2389
2390  """
2391  global textwidth3
2392
2393  import time
2394
2395  time_stamp = time.ctime(time.time())
2396  line = time_stamp + '. ' + line
2397   
2398  N = len(line) + 1
2399  s = '+' + '-'*N + CR
2400
2401  print s + '| ' + line + CR + s,
2402
2403  textwidth3 = N
2404
2405# -----------------------------------------------------------------------------
2406   
2407def print_footer():
2408  """Print line same width as that of print_header_box.
2409  """
2410 
2411  N = textwidth3
2412  s = '+' + '-'*N + CR   
2413     
2414  print s     
2415     
2416# -----------------------------------------------------------------------------
2417
2418def mkargstr(args, textwidth, argstr = '', level=0):
2419  """ Generate a string containing first textwidth characters of arguments.
2420
2421  USAGE:
2422    mkargstr(args, textwidth, argstr = '', level=0)
2423
2424  DESCRIPTION:
2425    Exactly the same as str(args) possibly followed by truncation,
2426    but faster if args is huge.
2427  """
2428
2429  import types
2430
2431  if level > 10:
2432      # Protect against circular structures
2433      return '...'
2434 
2435  WasTruncated = 0
2436
2437  if not type(args) in [types.TupleType, types.ListType, types.DictType]:
2438    if type(args) == types.StringType:
2439      argstr = argstr + "'"+str(args)+"'"
2440    else:
2441      # Truncate large numeric arrays before using str()
2442      if isinstance(args, num.ndarray):
2443#        if len(args.flat) > textwidth: 
2444#        Changed by Duncan and Nick 21/2/07 .flat has problems with
2445#        non-contigous arrays and ravel is equal to .flat except it
2446#        can work with non-contiguous  arrays
2447        if len(num.ravel(args)) > textwidth:
2448          args = 'Array: ' + str(args.shape)
2449
2450      argstr = argstr + str(args)
2451  else:
2452    if type(args) == types.DictType:
2453      argstr = argstr + "{"
2454      for key in args.keys():
2455        argstr = argstr + mkargstr(key, textwidth, level=level+1) + ": " + \
2456                 mkargstr(args[key], textwidth, level=level+1) + ", "
2457        if len(argstr) > textwidth:
2458          WasTruncated = 1
2459          break
2460      argstr = argstr[:-2]  # Strip off trailing comma     
2461      argstr = argstr + "}"
2462
2463    else:
2464      if type(args) == types.TupleType:
2465        lc = '('
2466        rc = ')'
2467      else:
2468        lc = '['
2469        rc = ']'
2470      argstr = argstr + lc
2471      for arg in args:
2472        argstr = argstr + mkargstr(arg, textwidth, level=level+1) + ', '
2473        if len(argstr) > textwidth:
2474          WasTruncated = 1
2475          break
2476
2477      # Strip off trailing comma and space unless singleton tuple
2478      #
2479      if type(args) == types.TupleType and len(args) == 1:
2480        argstr = argstr[:-1]   
2481      else:
2482        argstr = argstr[:-2]
2483      argstr = argstr + rc
2484
2485  if len(argstr) > textwidth:
2486    WasTruncated = 1
2487
2488  if WasTruncated:
2489    argstr = argstr[:textwidth]+'...'
2490  return(argstr)
2491
2492# -----------------------------------------------------------------------------
2493
2494def test_OK(msg):
2495  """Print OK msg if test is OK.
2496 
2497  USAGE
2498    test_OK(message)
2499  """
2500
2501  import string
2502   
2503  print string.ljust(msg, textwidth4) + ' - OK' 
2504 
2505  #raise StandardError
2506 
2507# -----------------------------------------------------------------------------
2508
2509def test_error(msg):
2510  """Print error if test fails.
2511 
2512  USAGE
2513    test_error(message)
2514  """
2515 
2516  print 'ERROR (caching.test): %s' %msg
2517  print 'Please send this code example and output to '
2518  print 'Ole.Nielsen@anu.edu.au'
2519  print
2520  print
2521 
2522  raise StandardError
2523
2524#-------------------------------------------------------------
2525if __name__ == "__main__":
2526  pass
Note: See TracBrowser for help on using the repository browser.