Context Navigation

source: anuga_core/source/anuga/caching/caching.py @ 5571

Last change on this file since 5571 was 5359, checked in by duncan, 16 years ago
changing where caching goes
File size: 69.7 KB

Line
1	# =============================================================================
2	# caching.py - Supervised caching of function results.
3	# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4	# Australian National University (1999-2003)
5	# Geoscience Australia (2003-present)
6	#
7	# This program is free software; you can redistribute it and/or modify
8	# it under the terms of the GNU General Public License as published by
9	# the Free Software Foundation; either version 2 of the License, or
10	# (at your option) any later version.
11	#
12	# This program is distributed in the hope that it will be useful,
13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	# GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16	# for more details.
17	#
18	# You should have received a copy of the GNU General Public License
19	# along with this program; if not, write to the Free Software
20	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
21	#
22	#
23	# Contact address: Ole.Nielsen@ga.gov.au
24	#
25	# Version 1.5.6 February 2002
26	# =============================================================================
27
28	"""Module caching.py - Supervised caching of function results.
29
30	Public functions:
31
32	cache(func,args) -- Cache values returned from func given args.
33	cachestat() -- Reports statistics about cache hits and time saved.
34	test() -- Conducts a basic test of the caching functionality.
35
36	See doc strings of individual functions for detailed documentation.
37	"""
38
39	# -----------------------------------------------------------------------------
40	# Initialisation code
41
42	# Determine platform
43	#
44	from os import getenv
45
46	import os
47	if os.name in ['nt', 'dos', 'win32', 'what else?']:
48	unix = 0
49	else:
50	unix = 1
51
52	cache_dir = '.python_cache'
53	# Make default caching directory name
54	#
55	if unix:
56	homedir = getenv('INUNDATIONHOME')
57	if homedir is None:
58	homedir = '~'
59	else:
60	homedir = homedir + os.sep + '.cache'
61	# Since homedir will be a group area, individually label the caches
62	user = getenv('LOGNAME')
63	if user is not None:
64	cache_dir += '_' + user
65
66	CR = '\n'
67	else:
68	homedir = 'c:'
69	CR = '\r\n' #FIXME: Not tested under windows
70
71	cachedir = homedir + os.sep + cache_dir + os.sep
72
73	# -----------------------------------------------------------------------------
74	# Options directory with default values - to be set by user
75	#
76
77	options = {
78	'cachedir': cachedir, # Default cache directory
79	'maxfiles': 1000000, # Maximum number of cached files
80	'savestat': 1, # Log caching info to stats file
81	'verbose': 1, # Write messages to standard output
82	'bin': 1, # Use binary format (more efficient)
83	'compression': 1, # Use zlib compression
84	'bytecode': 0, # Recompute if bytecode has changed
85	'expire': 0 # Automatically remove files that have been accessed
86	# least recently
87	}
88
89	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
90
91	def set_option(key, value):
92	"""Function to set values in the options directory.
93
94	USAGE:
95	set_option(key, value)
96
97	ARGUMENTS:
98	key -- Key in options dictionary. (Required)
99	value -- New value for key. (Required)
100
101	DESCRIPTION:
102	Function to set values in the options directory.
103	Raises an exception if key is not in options.
104	"""
105
106	if options.has_key(key):
107	options[key] = value
108	else:
109	raise KeyError(key) # Key not found, raise an exception
110
111	# -----------------------------------------------------------------------------
112	# Function cache - the main routine
113
114	def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
115	verbose=None, compression=None, evaluate=0, test=0, clear=0,
116	return_filename=0):
117	"""Supervised caching of function results.
118
119	USAGE:
120	result = cache(func, args, kwargs, dependencies, cachedir, verbose,
121	compression, evaluate, test, return_filename)
122
123	ARGUMENTS:
124	func -- Function object (Required)
125	args -- Arguments to func (Default: ())
126	kwargs -- Keyword arguments to func (Default: {})
127	dependencies -- Filenames that func depends on (Default: None)
128	cachedir -- Directory for cache files (Default: options['cachedir'])
129	verbose -- Flag verbose output to stdout
130	(Default: options['verbose'])
131	compression -- Flag zlib compression (Default: options['compression'])
132	evaluate -- Flag forced evaluation of func (Default: 0)
133	test -- Flag test for cached results (Default: 0)
134	clear -- Flag delete cached results (Default: 0)
135	return_filename -- Flag return of cache filename (Default: 0)
136
137	DESCRIPTION:
138	A Python function call of the form
139
140	result = func(arg1,...,argn)
141
142	can be replaced by
143
144	from caching import cache
145	result = cache(func,(arg1,...,argn))
146
147	The latter form returns the same output as the former but reuses cached
148	results if the function has been computed previously in the same context.
149	'result' and the arguments can be simple types, tuples, list, dictionaries or
150	objects, but not unhashable types such as functions or open file objects.
151	The function 'func' may be a member function of an object or a module.
152
153	This type of caching is particularly useful for computationally intensive
154	functions with few frequently used combinations of input arguments. Note that
155	if the inputs or output are very large caching might not save time because
156	disc access may dominate the execution time.
157
158	If the function definition changes after a result has been cached it will be
159	detected by examining the functions bytecode (co_code, co_consts,
160	func_defualts, co_argcount) and it will be recomputed.
161
162	LIMITATIONS:
163	1 Caching uses the apply function and will work with anything that can be
164	pickled, so any limitation in apply or pickle extends to caching.
165	2 A function to be cached should not depend on global variables
166	as wrong results may occur if globals are changed after a result has
167	been cached.
168
169	-----------------------------------------------------------------------------
170	Additional functionality:
171
172	Keyword args
173	Keyword arguments (kwargs) can be added as a dictionary of keyword: value
174	pairs, following the syntax of the built-in function apply().
175	A Python function call of the form
176
177	result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)
178
179	is then cached as follows
180
181	from caching import cache
182	result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
183
184	The default value of kwargs is {}
185
186	Explicit dependencies:
187	The call
188	cache(func,(arg1,...,argn),dependencies = <list of filenames>)
189	Checks the size, creation time and modification time of each listed file.
190	If any file has changed the function is recomputed and the results stored
191	again.
192
193	Specify caching directory:
194	The call
195	cache(func,(arg1,...,argn), cachedir = <cachedir>)
196	designates <cachedir> where cached data are stored. Use ~ to indicate users
197	home directory - not $HOME. The default is ~/.python_cache on a UNIX
198	platform and c:/.python_cache on a Win platform.
199
200	Silent operation:
201	The call
202	cache(func,(arg1,...,argn),verbose=0)
203	suppresses messages to standard output.
204
205	Compression:
206	The call
207	cache(func,(arg1,...,argn),compression=0)
208	disables compression. (Default: compression=1). If the requested compressed
209	or uncompressed file is not there, it'll try the other version.
210
211	Forced evaluation:
212	The call
213	cache(func,(arg1,...,argn),evaluate=1)
214	forces the function to evaluate even though cached data may exist.
215
216	Testing for presence of cached result:
217	The call
218	cache(func,(arg1,...,argn),test=1)
219	retrieves cached result if it exists, otherwise None. The function will not
220	be evaluated. If both evaluate and test are switched on, evaluate takes
221	precedence.
222
223	Obtain cache filenames:
224	The call
225	cache(func,(arg1,...,argn),return_filename=1)
226	returns the hashed base filename under which this function and its
227	arguments would be cached
228
229	Clearing cached results:
230	The call
231	cache(func,'clear')
232	clears all cached data for 'func' and
233	cache('clear')
234	clears all cached data.
235
236	NOTE: The string 'clear' can be passed an argument to func using
237	cache(func,('clear',)) or cache(func,tuple(['clear'])).
238
239	New form of clear:
240	cache(func,(arg1,...,argn),clear=1)
241	clears cached data for particular combination func and args
242
243	"""
244
245	# Imports and input checks
246	#
247	import types, time, string
248
249	if not cachedir:
250	cachedir = options['cachedir']
251
252	if verbose == None: # Do NOT write 'if not verbose:', it could be zero.
253	verbose = options['verbose']
254
255	if compression == None: # Do NOT write 'if not compression:',
256	# it could be zero.
257	compression = options['compression']
258
259	# Create cache directory if needed
260	#
261	CD = checkdir(cachedir,verbose)
262
263	# Handle the case cache('clear')
264	#
265	if type(func) == types.StringType:
266	if string.lower(func) == 'clear':
267	clear_cache(CD,verbose=verbose)
268	return
269
270	# Handle the case cache(func, 'clear')
271	#
272	if type(args) == types.StringType:
273	if string.lower(args) == 'clear':
274	clear_cache(CD,func,verbose=verbose)
275	return
276
277	# Force singleton arg into a tuple
278	#
279	if type(args) != types.TupleType:
280	args = tuple([args])
281
282	# Check that kwargs is a dictionary
283	#
284	if type(kwargs) != types.DictType:
285	raise TypeError
286
287	#print 'hashing' #FIXME: make faster hashing function
288
289	# Hash arguments (and keyword args) to integer
290	#
291	arghash = myhash((args,kwargs))
292
293	# Get sizes and timestamps for files listed in dependencies.
294	# Force singletons into a tuple.
295	#
296
297	if dependencies and type(dependencies) != types.TupleType \
298	and type(dependencies) != types.ListType:
299	dependencies = tuple([dependencies])
300	deps = get_depstats(dependencies)
301
302	# Extract function name from func object
303	#
304	funcname = get_funcname(func)
305
306	# Create cache filename
307	#
308	FN = funcname+'['+`arghash`+']' # The symbol '(' does not work under unix
309
310	if return_filename:
311	return(FN)
312
313	if clear:
314	for file_type in file_types:
315	file_name = CD+FN+'_'+file_type
316	for fn in [file_name, file_name + '.z']:
317	if os.access(fn, os.F_OK):
318	if unix:
319	os.remove(fn)
320	else:
321	# FIXME: os.remove doesn't work under windows
322	os.system('del '+fn)
323	if verbose is True:
324	print 'MESSAGE (caching): File %s deleted' %fn
325	##else:
326	## print '%s was not accessed' %fn
327	return None
328
329
330	#-------------------------------------------------------------------
331
332	# Check if previous computation has been cached
333	#
334	if evaluate:
335	Retrieved = None # Force evaluation of func regardless of caching status.
336	reason = 4
337	else:
338	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
339	CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
340	dependencies)
341
342	if not Retrieved:
343	if test: # Do not attempt to evaluate function
344	T = None
345	else: # Evaluate function and save to cache
346	if verbose:
347
348	msg1(funcname, args, kwargs,reason)
349
350	# Remove expired files automatically
351	#
352	if options['expire']:
353	DeleteOldFiles(CD,verbose)
354
355	# Save args before function is evaluated in case
356	# they are modified by function
357	#
358	save_args_to_cache(CD,FN,args,kwargs,compression)
359
360	# Execute and time function with supplied arguments
361	#
362	t0 = time.time()
363	T = apply(func,args,kwargs)
364	#comptime = round(time.time()-t0)
365	comptime = time.time()-t0
366
367	if verbose:
368	msg2(funcname,args,kwargs,comptime,reason)
369
370	# Save results and estimated loading time to cache
371	#
372	loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
373	funcname, dependencies, compression)
374	if verbose:
375	msg3(loadtime, CD, FN, deps, compression)
376	compressed = compression
377
378	if options['savestat'] and (not test or Retrieved):
379	##if options['savestat']:
380	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
381
382	return(T) # Return results in all cases
383
384	# -----------------------------------------------------------------------------
385
386	def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
387	"""Generate statistics of caching efficiency.
388
389	USAGE:
390	cachestat(sortidx, period, showuser, cachedir)
391
392	ARGUMENTS:
393	sortidx -- Index of field by which lists are (default: 4)
394	Legal values are
395	0: 'Name'
396	1: 'Hits'
397	2: 'CPU'
398	3: 'Time Saved'
399	4: 'Gain(%)'
400	5: 'Size'
401	period -- If set to -1 all available caching history is used.
402	If set 0 only the current month is used (default -1).
403	showuser -- Flag for additional table showing user statistics
404	(default: None).
405	cachedir -- Directory for cache files (default: options['cachedir']).
406
407	DESCRIPTION:
408	Logged caching statistics is converted into summaries of the form
409	--------------------------------------------------------------------------
410	Function Name Hits Exec(s) Cache(s) Saved(s) Gain(%) Size
411	--------------------------------------------------------------------------
412	"""
413
414	__cachestat(sortidx, period, showuser, cachedir)
415	return
416
417	# -----------------------------------------------------------------------------
418
419	#Has mostly been moved to proper unit test
420	def test(cachedir=None,verbose=0,compression=None):
421	"""Test the functionality of caching.
422
423	USAGE:
424	test(verbose)
425
426	ARGUMENTS:
427	verbose -- Flag whether caching will output its statistics (default=0)
428	cachedir -- Directory for cache files (Default: options['cachedir'])
429	compression -- Flag zlib compression (Default: options['compression'])
430	"""
431
432	import string, time
433
434	# Initialise
435	#
436	import caching
437	reload(caching)
438
439	if not cachedir:
440	cachedir = options['cachedir']
441
442	if verbose is None: # Do NOT write 'if not verbose:', it could be zero.
443	verbose = options['verbose']
444
445	if compression == None: # Do NOT write 'if not compression:',
446	# it could be zero.
447	compression = options['compression']
448	else:
449	try:
450	set_option('compression', compression)
451	except:
452	test_error('Set option failed')
453
454	try:
455	import zlib
456	except:
457	print
458	print '* Could not find zlib, default to no-compression *'
459	print '* Installing zlib will improve performance of caching *'
460	print
461	compression = 0
462	set_option('compression', compression)
463
464	print
465	print_header_box('Testing caching module - please stand by')
466	print
467
468	# Define a test function to be cached
469	#
470	def f(a,b,c,N,x=0,y='abcdefg'):
471	"""f(a,b,c,N)
472	Do something time consuming and produce a complex result.
473	"""
474
475	import string
476
477	B = []
478	for n in range(N):
479	s = str(n+2.0/(n + 4.0))+'.a'*10
480	B.append((a,b,c,s,n,x,y))
481	return(B)
482
483	# Check that default cachedir is OK
484	#
485	CD = checkdir(cachedir,verbose)
486
487
488	# Make a dependency file
489	#
490	try:
491	DepFN = CD + 'testfile.tmp'
492	DepFN_wildcard = CD + 'test*.tmp'
493	Depfile = open(DepFN,'w')
494	Depfile.write('We are the knights who say NI!')
495	Depfile.close()
496	test_OK('Wrote file %s' %DepFN)
497	except:
498	test_error('Could not open file %s for writing - check your environment' \
499	% DepFN)
500
501	# Check set_option (and switch stats off
502	#
503	try:
504	set_option('savestat',0)
505	assert(options['savestat'] == 0)
506	test_OK('Set option')
507	except:
508	test_error('Set option failed')
509
510	# Make some test input arguments
511	#
512	N = 5000 #Make N fairly small here
513
514	a = [1,2]
515	b = ('Thou shalt count the number three',4)
516	c = {'Five is right out': 6, (7,8): 9}
517	x = 3
518	y = 'holy hand granate'
519
520	# Test caching
521	#
522	if compression:
523	comprange = 2
524	else:
525	comprange = 1
526
527	for comp in range(comprange):
528
529	# Evaluate and store
530	#
531	try:
532	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
533	verbose=verbose, compression=comp)
534	if comp:
535	test_OK('Caching evaluation with compression')
536	else:
537	test_OK('Caching evaluation without compression')
538	except:
539	if comp:
540	test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
541	else:
542	test_error('Caching evaluation failed - try caching.test(verbose=1)')
543
544	# Retrieve
545	#
546	try:
547	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
548	compression=comp)
549
550	if comp:
551	test_OK('Caching retrieval with compression')
552	else:
553	test_OK('Caching retrieval without compression')
554	except:
555	if comp:
556	test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
557	else:
558	test_error('Caching retrieval failed - try caching.test(verbose=1)')
559
560	# Reference result
561	#
562	T3 = f(a,b,c,N,x=x,y=y) # Compute without caching
563
564	if T1 == T2 and T2 == T3:
565	if comp:
566	test_OK('Basic caching functionality (with compression)')
567	else:
568	test_OK('Basic caching functionality (without compression)')
569	else:
570	test_error('Cached result does not match computed result')
571
572
573	# Test return_filename
574	#
575	try:
576	FN = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
577	return_filename=1)
578	assert(FN[:2] == 'f[')
579	test_OK('Return of cache filename')
580	except:
581	test_error('Return of cache filename failed')
582
583	# Test existence of cachefiles
584	#
585	try:
586	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
587	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
588	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
589	test_OK('Presence of cache files')
590	datafile.close()
591	argsfile.close()
592	admfile.close()
593	except:
594	test_error('Expected cache files did not exist')
595
596	# Test 'test' function when cache is present
597	#
598	try:
599	#T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
600	# evaluate=1)
601	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
602	assert(T1 == T4)
603
604	test_OK("Option 'test' when cache file present")
605	except:
606	test_error("Option 'test' when cache file present failed")
607
608	# Test that 'clear' works
609	#
610	#try:
611	# caching.cache(f,'clear',verbose=verbose)
612	# test_OK('Clearing of cache files')
613	#except:
614	# test_error('Clear does not work')
615	try:
616	caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)
617	test_OK('Clearing of cache files')
618	except:
619	test_error('Clear does not work')
620
621
622
623	# Test 'test' function when cache is absent
624	#
625	try:
626	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
627	assert(T4 is None)
628	test_OK("Option 'test' when cache absent")
629	except:
630	test_error("Option 'test' when cache absent failed")
631
632	# Test dependencies
633	#
634	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
635	dependencies=DepFN)
636	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
637	dependencies=DepFN)
638
639	if T1 == T2:
640	test_OK('Basic dependencies functionality')
641	else:
642	test_error('Dependencies do not work')
643
644	# Test basic wildcard dependency
645	#
646	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
647	dependencies=DepFN_wildcard)
648
649	if T1 == T3:
650	test_OK('Basic dependencies with wildcard functionality')
651	else:
652	test_error('Dependencies with wildcards do not work')
653
654
655	# Test that changed timestamp in dependencies triggers recomputation
656
657	# Modify dependency file
658	Depfile = open(DepFN,'a')
659	Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
660	Depfile.close()
661
662	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
663	dependencies=DepFN, test = 1)
664
665	if T3 is None:
666	test_OK('Changed dependencies recognised')
667	else:
668	test_error('Changed dependencies not recognised')
669
670	# Test recomputation when dependencies have changed
671	#
672	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
673	dependencies=DepFN)
674	if T1 == T3:
675	test_OK('Recomputed value with changed dependencies')
676	else:
677	test_error('Recomputed value with changed dependencies failed')
678
679	# Performance test (with statistics)
680	# Don't really rely on this as it will depend on specific computer.
681	#
682
683	set_option('savestat',1)
684
685	N = 20*N #Should be large on fast computers...
686	tt = time.time()
687	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
688	t1 = time.time() - tt
689
690	tt = time.time()
691	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
692	t2 = time.time() - tt
693
694	if T1 == T2:
695	if t1 > t2:
696	test_OK('Performance test: relative time saved = %s pct' \
697	%str(round((t1-t2)*100/t1,2)))
698	#else:
699	# print 'WARNING: Performance a bit low - this could be specific to current platform'
700	else:
701	test_error('Basic caching failed for new problem')
702
703	# Test presence of statistics file
704	#
705	try:
706	DIRLIST = os.listdir(CD)
707	SF = []
708	for FN in DIRLIST:
709	if string.find(FN,statsfile) >= 0:
710	fid = open(CD+FN,'r')
711	fid.close()
712	test_OK('Statistics files present')
713	except:
714	test_OK('Statistics files cannot be opened')
715
716	print_header_box('Show sample output of the caching function:')
717
718	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
719	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
720	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
721
722	print_header_box('Show sample output of cachestat():')
723	if unix:
724	cachestat()
725	else:
726	try:
727	import time
728	t = time.strptime('2030','%Y')
729	cachestat()
730	except:
731	print 'caching.cachestat() does not work here, because it'
732	print 'relies on time.strptime() which is unavailable in Windows'
733
734	print
735	test_OK('Caching self test completed')
736
737
738	# Test setoption (not yet implemented)
739	#
740
741
742	#==============================================================================
743	# Auxiliary functions
744	#==============================================================================
745
746	# Import pickler
747	# cPickle is used by functions mysave, myload, and compare
748	#
749	import cPickle # 10 to 100 times faster than pickle
750	pickler = cPickle
751
752	# Local immutable constants
753	#
754	comp_level = 1 # Compression level for zlib.
755	# comp_level = 1 works well.
756	textwidth1 = 16 # Text width of key fields in report forms.
757	#textwidth2 = 132 # Maximal width of textual representation of
758	textwidth2 = 300 # Maximal width of textual representation of
759	# arguments.
760	textwidth3 = 16 # Initial width of separation lines. Is modified.
761	textwidth4 = 50 # Text width in test_OK()
762	statsfile = '.cache_stat' # Basefilename for cached statistics.
763	# It will reside in the chosen cache directory.
764
765	file_types = ['Result', # File name extension for cached function results.
766	'Args', # File name extension for stored function args.
767	'Admin'] # File name extension for administrative info.
768
769	Reason_msg = ['OK', # Verbose reasons for recomputation
770	'No cached result',
771	'Dependencies have changed',
772	'Byte code or arguments have changed',
773	'Recomputation was requested by caller',
774	'Cached file was unreadable']
775
776	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
777
778	def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression,
779	dependencies):
780	"""Determine whether cached result exists and return info.
781
782	USAGE:
783	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
784	CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
785	dependencies)
786
787	INPUT ARGUMENTS:
788	CD -- Cache Directory
789	FN -- Suggested cache file name
790	func -- Function object
791	args -- Tuple of arguments
792	kwargs -- Dictionary of keyword arguments
793	deps -- Dependencies time stamps
794	verbose -- Flag text output
795	compression -- Flag zlib compression
796	dependencies -- Given list of dependencies
797
798	OUTPUT ARGUMENTS:
799	T -- Cached result if present otherwise None
800	FN -- File name under which new results must be saved
801	Retrieved -- True if a valid cached result was found
802	reason -- 0: OK (if Retrieved),
803	1: No cached result,
804	2: Dependencies have changed,
805	3: Arguments or Bytecode have changed
806	4: Recomputation was forced
807	comptime -- Number of seconds it took to computed cachged result
808	loadtime -- Number of seconds it took to load cached result
809	compressed -- Flag (0,1) if cached results were compressed or not
810
811	DESCRIPTION:
812	Determine if cached result exists as follows:
813	Load in saved arguments and bytecode stored under hashed filename.
814	If they are identical to current arguments and bytecode and if dependencies
815	have not changed their time stamp, then return cached result.
816
817	Otherwise return filename under which new results should be cached.
818	Hash collisions are handled recursively by calling CacheLookup again with a
819	modified filename.
820	"""
821
822	import time, string, types
823
824	# Assess whether cached result exists - compressed or not.
825	#
826	if verbose:
827	print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
828	%(CD+FN, file_types[0], file_types[1], file_types[2])
829	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
830	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
831	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
832
833	if verbose is True and deps is not None:
834	print 'Caching: Dependencies are', deps.keys()
835
836	if not (argsfile and datafile and admfile) or \
837	not (compressed0 == compressed1 and compressed0 == compressed2):
838	# Cached result does not exist or files were compressed differently
839	#
840	# This will ensure that evaluation will take place unless all files are
841	# present.
842
843	reason = 1
844	return(None,FN,None,reason,None,None,None) #Recompute using same filename
845
846	compressed = compressed0 # Remember if compressed files were actually used
847	datafile.close()
848
849	# Retrieve arguments and adm. info
850	#
851	R, reason = myload(argsfile,compressed) # The original arguments
852	argsfile.close()
853
854	##if R == None and reason > 0:
855	if reason > 0:
856	return(None,FN,None,reason,None,None,None) #Recompute using same filename
857	else:
858	(argsref, kwargsref) = R
859
860	R, reason = myload(admfile,compressed)
861	admfile.close()
862	##if R == None and reason > 0:
863	if reason > 0:
864	return(None,FN,None,reason,None,None,None) #Recompute using same filename
865
866
867	depsref = R[0] # Dependency statistics
868	comptime = R[1] # The computation time
869	coderef = R[2] # The byte code
870	funcname = R[3] # The function name
871
872	# Check if dependencies have changed
873	#
874	if dependencies and not compare(depsref,deps):
875	if verbose:
876	print 'MESSAGE (caching.py): Dependencies', dependencies, \
877	'have changed - recomputing'
878	# Don't use cached file - recompute
879	reason = 2
880	return(None,FN,None,reason,None,None,None)
881
882	# Get bytecode from func
883	#
884	bytecode = get_bytecode(func)
885
886	#print compare(argsref,args),
887	#print compare(kwargsref,kwargs),
888	#print compare(bytecode,coderef)
889
890	# Check if arguments or bytecode have changed
891	#
892	if compare(argsref,args) and compare(kwargsref,kwargs) and \
893	(not options['bytecode'] or compare(bytecode,coderef)):
894
895	# Arguments and dependencies match. Get cached results
896	#
897	T, loadtime, compressed, reason = load_from_cache(CD,FN,compressed)
898	###if T == None and reason > 0: #This doesn't work if T is a numeric array
899	if reason > 0:
900	return(None,FN,None,reason,None,None,None) #Recompute using same FN
901
902	Retrieved = 1
903	reason = 0
904
905	if verbose:
906	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
907
908	if loadtime >= comptime:
909	print 'WARNING (caching.py): Caching did not yield any gain.'
910	print ' Consider executing function ',
911	print '('+funcname+') without caching.'
912	else:
913
914	# Non matching arguments or bytecodes signify a hash-collision.
915	# This is resolved by recursive search of cache filenames
916	# until either a matching or an unused filename is found.
917	#
918	(T,FN,Retrieved,reason,comptime,loadtime,compressed) = \
919	CacheLookup(CD,FN+'x',func,args,kwargs,deps,verbose,compression, \
920	dependencies)
921
922	# DEBUGGING
923	# if not Retrieved:
924	# print 'Arguments did not match'
925	# else:
926	# print 'Match found !'
927	if not Retrieved:
928	reason = 3 #The real reason is that args or bytecodes have changed.
929	#Not that the recursive seach has found an unused filename
930
931	return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
932
933	# -----------------------------------------------------------------------------
934
935	def clear_cache(CD,func=None, verbose=None):
936	"""Clear cache for func.
937
938	USAGE:
939	clear(CD, func, verbose)
940
941	ARGUMENTS:
942	CD -- Caching directory (required)
943	func -- Function object (default: None)
944	verbose -- Flag verbose output (default: None)
945
946	DESCRIPTION:
947
948	If func == None, clear everything,
949	otherwise clear only files pertaining to func.
950	"""
951
952	import os, re
953
954	if CD[-1] != os.sep:
955	CD = CD+os.sep
956
957	if verbose == None:
958	verbose = options['verbose']
959
960	# FIXME: Windows version needs to be tested
961
962	if func:
963	funcname = get_funcname(func)
964	if verbose:
965	print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
966
967	file_names = os.listdir(CD)
968	for file_name in file_names:
969	#RE = re.search('^' + funcname,file_name) #Inefficient
970	#if RE:
971	if file_name[:len(funcname)] == funcname:
972	if unix:
973	os.remove(CD+file_name)
974	else:
975	os.system('del '+CD+file_name)
976	# FIXME: os.remove doesn't work under windows
977	else:
978	file_names = os.listdir(CD)
979	if len(file_names) > 0:
980	if verbose:
981	print 'MESSAGE (caching.py): Remove the following files:'
982	for file_name in file_names:
983	print file_name
984
985	A = raw_input('Delete (Y/N)[N] ?')
986	else:
987	A = 'Y'
988
989	if A == 'Y' or A == 'y':
990	for file_name in file_names:
991	if unix:
992	os.remove(CD+file_name)
993	else:
994	os.system('del '+CD+file_name)
995	# FIXME: os.remove doesn't work under windows
996	#exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
997
998	# -----------------------------------------------------------------------------
999
1000	def DeleteOldFiles(CD,verbose=None):
1001	"""Remove expired files
1002
1003	USAGE:
1004	DeleteOldFiles(CD,verbose=None)
1005	"""
1006
1007	if verbose == None:
1008	verbose = options['verbose']
1009
1010	maxfiles = options['maxfiles']
1011
1012	# FIXME: Windows version
1013
1014	import os
1015	block = 1000 # How many files to delete per invokation
1016	Files = os.listdir(CD)
1017	numfiles = len(Files)
1018	if not unix: return # FIXME: Windows case ?
1019
1020	if numfiles > maxfiles:
1021	delfiles = numfiles-maxfiles+block
1022	if verbose:
1023	print 'Deleting '+`delfiles`+' expired files:'
1024	os.system('ls -lur '+CD+'* \| head -' + `delfiles`) # List them
1025	os.system('ls -ur '+CD+'* \| head -' + `delfiles` + ' \| xargs /bin/rm')
1026	# Delete them
1027	# FIXME: Replace this with os.listdir and os.remove
1028
1029	# -----------------------------------------------------------------------------
1030
1031	def save_args_to_cache(CD,FN,args,kwargs,compression):
1032	"""Save arguments to cache
1033
1034	USAGE:
1035	save_args_to_cache(CD,FN,args,kwargs,compression)
1036	"""
1037
1038	import time, os, sys, types
1039
1040	(argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1041
1042	if not argsfile:
1043	if verbose:
1044	print 'ERROR (caching): Could not open %s' %argsfile.name
1045	raise IOError
1046
1047	mysave((args,kwargs),argsfile,compression) # Save args and kwargs to cache
1048	argsfile.close()
1049
1050	# Change access rights if possible
1051	#
1052	#if unix:
1053	# try:
1054	# exitcode=os.system('chmod 666 '+argsfile.name)
1055	# except:
1056	# pass
1057	#else:
1058	# pass # FIXME: Take care of access rights under Windows
1059
1060	return
1061
1062	# -----------------------------------------------------------------------------
1063
1064	def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1065	dependencies, compression):
1066	"""Save computed results T and admin info to cache
1067
1068	USAGE:
1069	save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1070	dependencies, compression)
1071	"""
1072
1073	import time, os, sys, types
1074
1075	(datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1076	(admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1077
1078	if not datafile:
1079	if verbose:
1080	print 'ERROR (caching): Could not open %s' %datafile.name
1081	raise IOError
1082
1083	if not admfile:
1084	if verbose:
1085	print 'ERROR (caching): Could not open %s' %admfile.name
1086	raise IOError
1087
1088	t0 = time.time()
1089
1090	mysave(T,datafile,compression) # Save data to cache
1091	datafile.close()
1092	#savetime = round(time.time()-t0,2)
1093	savetime = time.time()-t0
1094
1095	bytecode = get_bytecode(func) # Get bytecode from function object
1096	admtup = (deps, comptime, bytecode, funcname) # Gather admin info
1097
1098	mysave(admtup,admfile,compression) # Save admin info to cache
1099	admfile.close()
1100
1101	# Change access rights if possible
1102	#
1103	#if unix:
1104	# try:
1105	# exitcode=os.system('chmod 666 '+datafile.name)
1106	# exitcode=os.system('chmod 666 '+admfile.name)
1107	# except:
1108	# pass
1109	#else:
1110	# pass # FIXME: Take care of access rights under Windows
1111
1112	return(savetime)
1113
1114	# -----------------------------------------------------------------------------
1115
1116	def load_from_cache(CD,FN,compression):
1117	"""Load previously cached data from file FN
1118
1119	USAGE:
1120	load_from_cache(CD,FN,compression)
1121	"""
1122
1123	import time
1124
1125	(datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1126	t0 = time.time()
1127	T, reason = myload(datafile,compressed)
1128	#loadtime = round(time.time()-t0,2)
1129	loadtime = time.time()-t0
1130	datafile.close()
1131
1132	return T, loadtime, compressed, reason
1133
1134	# -----------------------------------------------------------------------------
1135
1136	def myopen(FN,mode,compression=1):
1137	"""Open file FN using given mode
1138
1139	USAGE:
1140	myopen(FN,mode,compression=1)
1141
1142	ARGUMENTS:
1143	FN -- File name to be opened
1144	mode -- Open mode (as in open)
1145	compression -- Flag zlib compression
1146
1147	DESCRIPTION:
1148	if compression
1149	Attempt first to open FN + '.z'
1150	If this fails try to open FN
1151	else do the opposite
1152	Return file handle plus info about whether it was compressed or not.
1153	"""
1154
1155	import string
1156
1157	# Determine if file exists already (if writing was requested)
1158	# This info is only used to determine if access modes should be set
1159	#
1160	if 'w' in mode or 'a' in mode:
1161	try:
1162	file = open(FN+'.z','r')
1163	file.close()
1164	new_file = 0
1165	except:
1166	try:
1167	file = open(FN,'r')
1168	file.close()
1169	new_file = 0
1170	except:
1171	new_file = 1
1172	else:
1173	new_file = 0 #Assume it exists if mode was not 'w'
1174
1175
1176	compressed = 0
1177	if compression:
1178	try:
1179	file = open(FN+'.z',mode)
1180	compressed = 1
1181	except:
1182	try:
1183	file = open(FN,mode)
1184	except:
1185	file = None
1186	else:
1187	try:
1188	file = open(FN,mode)
1189	except:
1190	try:
1191	file = open(FN+'.z',mode)
1192	compressed = 1
1193	except:
1194	file = None
1195
1196	# Now set access rights if it is a new file
1197	#
1198	if file and new_file:
1199	if unix:
1200	exitcode=os.system('chmod 666 '+file.name)
1201	else:
1202	pass # FIXME: Take care of access rights under Windows
1203
1204	return(file,compressed)
1205
1206	# -----------------------------------------------------------------------------
1207
1208	def myload(file, compressed):
1209	"""Load data from file
1210
1211	USAGE:
1212	myload(file, compressed)
1213	"""
1214
1215	reason = 0
1216	try:
1217	if compressed:
1218	import zlib
1219
1220	RsC = file.read()
1221	try:
1222	Rs = zlib.decompress(RsC)
1223	except:
1224	# File "./caching.py", line 1032, in load_from_cache
1225	# T = myload(datafile,compressed)
1226	# File "./caching.py", line 1124, in myload
1227	# Rs = zlib.decompress(RsC)
1228	# zlib.error: Error -5 while decompressing data
1229	#print 'ERROR (caching): Could not decompress ', file.name
1230	#raise Exception
1231	reason = 5 #(Unreadable file)
1232	return None, reason
1233
1234
1235	del RsC # Free up some space
1236	R = pickler.loads(Rs)
1237	else:
1238	try:
1239	R = pickler.load(file)
1240	#except EOFError, e:
1241	except:
1242	#Catch e.g., file with 0 length or corrupted
1243	reason = 5 #(Unreadable file)
1244	return None, reason
1245
1246	except MemoryError:
1247	import sys
1248	if options['verbose']:
1249	print 'ERROR (caching): Out of memory while loading %s, aborting' \
1250	%(file.name)
1251
1252	# Raise the error again for now
1253	#
1254	raise MemoryError
1255
1256	return R, reason
1257
1258	# -----------------------------------------------------------------------------
1259
1260	def mysave(T,file,compression):
1261	"""Save data T to file
1262
1263	USAGE:
1264	mysave(T,file,compression)
1265
1266	"""
1267
1268	bin = options['bin']
1269
1270	if compression:
1271	try:
1272	import zlib
1273	except:
1274	print
1275	print '* Could not find zlib *'
1276	print '* Try to run caching with compression off *'
1277	print "* caching.set_option('compression', 0) *"
1278	raise Exception
1279
1280
1281	try:
1282	Ts = pickler.dumps(T, bin)
1283	except MemoryError:
1284	msg = '****WARNING (caching.py): Could not pickle data for compression.'
1285	msg += ' Try using compression = False'
1286	raise MemoryError, msg
1287	else:
1288	#Compressed pickling
1289	TsC = zlib.compress(Ts, comp_level)
1290	file.write(TsC)
1291	else:
1292	#Uncompressed pickling
1293	pickler.dump(T, file, bin)
1294
1295	# FIXME: This may not work on Windoze network drives.
1296	# The error msg is IOError: [Errno 22] Invalid argument
1297	# Testing with small files was OK, though.
1298	# I think this is an OS problem.
1299
1300	# Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1301
1302	# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1303	#
1304	#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1305	#
1306	#The following extract is taken from the site http://www.python.org:
1307	#
1308	#---------------------------------------------------------------------------------------------
1309	#exception IOError
1310	#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1311	#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1312	#---------------------------------------------------------------------------------------------
1313	#
1314	#The error code(s) that accompany exceptions are described at:
1315	#http://www.python.org/dev/doc/devel//lib/module-errno.html
1316	#
1317	#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1318	#
1319	#exceptions.IOError invalid argument Errno 22
1320
1321	#try:
1322	# pickler.dump(T,file,bin)
1323	#except IOError, e:
1324	# print e
1325	# msg = 'Could not store to %s, bin=%s' %(file, bin)
1326	# raise msg
1327
1328
1329	# -----------------------------------------------------------------------------
1330
1331	def myhash(T):
1332	"""Compute hashed integer from hashable values of tuple T
1333
1334	USAGE:
1335	myhash(T)
1336
1337	ARGUMENTS:
1338	T -- Tuple
1339	"""
1340
1341	import types
1342
1343	# On some architectures None, False and True gets different hash values
1344	if T is None:
1345	return(-1)
1346	if T is False:
1347	return(0)
1348	if T is True:
1349	return(1)
1350
1351	# Get hash vals for hashable entries
1352	#
1353	if type(T) == types.TupleType or type(T) == types.ListType:
1354	hvals = []
1355	for k in range(len(T)):
1356	h = myhash(T[k])
1357	hvals.append(h)
1358	val = hash(tuple(hvals))
1359	elif type(T) == types.DictType:
1360	val = dicthash(T)
1361	else:
1362	try:
1363	val = hash(T)
1364	except:
1365	val = 1
1366	try:
1367	import Numeric
1368	if type(T) == Numeric.ArrayType:
1369	hvals = []
1370	for e in T:
1371	h = myhash(e)
1372	hvals.append(h)
1373	val = hash(tuple(hvals))
1374	else:
1375	val = 1 #Could implement other Numeric types here
1376	except:
1377	pass
1378
1379	return(val)
1380
1381	# -----------------------------------------------------------------------------
1382
1383	def dicthash(D):
1384	"""Compute hashed integer from hashable values of dictionary D
1385
1386	USAGE:
1387	dicthash(D)
1388	"""
1389
1390	keys = D.keys()
1391
1392	# Get hash values for hashable entries
1393	#
1394	hvals = []
1395	for k in range(len(keys)):
1396	try:
1397	h = myhash(D[keys[k]])
1398	hvals.append(h)
1399	except:
1400	pass
1401
1402	# Hash obtained values into one value
1403	#
1404	return(hash(tuple(hvals)))
1405
1406	# -----------------------------------------------------------------------------
1407
1408	def compare(A,B):
1409	"""Safe comparison of general objects
1410
1411	USAGE:
1412	compare(A,B)
1413
1414	DESCRIPTION:
1415	Return 1 if A and B they are identical, 0 otherwise
1416	"""
1417
1418	try:
1419	identical = (A == B)
1420	except:
1421	try:
1422	identical = (pickler.dumps(A) == pickler.dumps(B))
1423	except:
1424	identical = 0
1425
1426	return(identical)
1427
1428	# -----------------------------------------------------------------------------
1429
1430	def nospace(s):
1431	"""Replace spaces in string s with underscores
1432
1433	USAGE:
1434	nospace(s)
1435
1436	ARGUMENTS:
1437	s -- string
1438	"""
1439
1440	import string
1441
1442	newstr = ''
1443	for i in range(len(s)):
1444	if s[i] == ' ':
1445	newstr = newstr+'_'
1446	else:
1447	newstr = newstr+s[i]
1448
1449	return(newstr)
1450
1451	# -----------------------------------------------------------------------------
1452
1453	def get_funcname(func):
1454	"""Retrieve name of function object func (depending on its type)
1455
1456	USAGE:
1457	get_funcname(func)
1458	"""
1459
1460	import types, string
1461
1462	if type(func) == types.FunctionType:
1463	funcname = func.func_name
1464	elif type(func) == types.BuiltinFunctionType:
1465	funcname = func.__name__
1466	else:
1467	tab = string.maketrans("<>'"," ")
1468	tmp = string.translate(`func`,tab)
1469	tmp = string.split(tmp)
1470	funcname = string.join(tmp)
1471
1472	# Truncate memory address as in
1473	# class __main__.Dummy at 0x00A915D0
1474	index = funcname.find('at 0x')
1475	if index >= 0:
1476	funcname = funcname[:index+5] # Keep info that there is an address
1477
1478	funcname = nospace(funcname)
1479	return(funcname)
1480
1481	# -----------------------------------------------------------------------------
1482
1483	def get_bytecode(func):
1484	""" Get bytecode from function object.
1485
1486	USAGE:
1487	get_bytecode(func)
1488	"""
1489
1490	import types
1491
1492	if type(func) == types.FunctionType:
1493	bytecode = func.func_code.co_code
1494	consts = func.func_code.co_consts
1495	argcount = func.func_code.co_argcount
1496	defaults = func.func_defaults
1497	elif type(func) == types.MethodType:
1498	bytecode = func.im_func.func_code.co_code
1499	consts = func.im_func.func_code.co_consts
1500	argcount = func.im_func.func_code.co_argcount
1501	defaults = func.im_func.func_defaults
1502	else:
1503	#raise Exception #Test only
1504	bytecode = None #Built-in functions are assumed not to change
1505	consts = 0
1506	argcount = 0
1507	defaults = 0
1508
1509	return (bytecode, consts, argcount, defaults)
1510
1511	# -----------------------------------------------------------------------------
1512
1513	def get_depstats(dependencies):
1514	""" Build dictionary of dependency files and their size, mod. time and ctime.
1515
1516	USAGE:
1517	get_depstats(dependencies):
1518	"""
1519
1520	import types
1521
1522	#print 'Caching DEBUG: Dependencies are', dependencies
1523	d = {}
1524	if dependencies:
1525
1526	#Expand any wildcards
1527	import glob
1528	expanded_dependencies = []
1529	for FN in dependencies:
1530	expanded_FN = glob.glob(FN)
1531
1532	if expanded_FN == []:
1533	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1534	raise Exception, errmsg
1535
1536	expanded_dependencies += expanded_FN
1537
1538
1539	for FN in expanded_dependencies:
1540	if not type(FN) == types.StringType:
1541	errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1542	errmsg += ' Dependency given: %s' %FN
1543	raise Exception, errmsg
1544	if not os.access(FN,os.F_OK):
1545	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1546	raise Exception, errmsg
1547	(size,atime,mtime,ctime) = filestat(FN)
1548
1549	# We don't use atime because that would cause recomputation every time.
1550	# We don't use ctime because that is irrelevant and confusing for users.
1551	d.update({FN : (size,mtime)})
1552
1553	return(d)
1554
1555	# -----------------------------------------------------------------------------
1556
1557	def filestat(FN):
1558	"""A safe wrapper using os.stat to get basic file statistics
1559	The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1560
1561	USAGE:
1562	filestat(FN)
1563
1564	DESCRIPTION:
1565	Must compile Python with
1566	CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1567	configure
1568	as given in section 8.1.1 Large File Support in the Libray Reference
1569	"""
1570
1571	import os, time
1572
1573	try:
1574	stats = os.stat(FN)
1575	size = stats[6]
1576	atime = stats[7]
1577	mtime = stats[8]
1578	ctime = stats[9]
1579	except:
1580
1581	# Hack to get the results anyway (works only on Unix at the moment)
1582	#
1583	print 'Hack to get os.stat when files are too large'
1584
1585	if unix:
1586	tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1587	# Unique filename, FIXME: Use random number
1588
1589	# Get size and access time (atime)
1590	#
1591	exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1592	(size,atime) = get_lsline(tmp)
1593
1594	# Get size and modification time (mtime)
1595	#
1596	exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1597	(size,mtime) = get_lsline(tmp)
1598
1599	# Get size and ctime
1600	#
1601	exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1602	(size,ctime) = get_lsline(tmp)
1603
1604	try:
1605	exitcode=os.system('rm '+tmp)
1606	# FIXME: Gives error if file doesn't exist
1607	except:
1608	pass
1609	else:
1610	pass
1611	raise Exception # FIXME: Windows case
1612
1613	return(long(size),atime,mtime,ctime)
1614
1615	# -----------------------------------------------------------------------------
1616
1617	def get_lsline(FN):
1618	"""get size and time for filename
1619
1620	USAGE:
1621	get_lsline(file_name)
1622
1623	DESCRIPTION:
1624	Read in one line 'ls -la' item from file (generated by filestat) and
1625	convert time to seconds since epoch. Return file size and time.
1626	"""
1627
1628	import string, time
1629
1630	f = open(FN,'r')
1631	info = f.read()
1632	info = string.split(info)
1633
1634	size = info[4]
1635	week = info[5]
1636	mon = info[6]
1637	day = info[7]
1638	hour = info[8]
1639	year = info[9]
1640
1641	str = week+' '+mon+' '+day+' '+hour+' '+year
1642	timetup = time.strptime(str)
1643	t = time.mktime(timetup)
1644	return(size, t)
1645
1646	# -----------------------------------------------------------------------------
1647
1648	def checkdir(CD,verbose=None):
1649	"""Check or create caching directory
1650
1651	USAGE:
1652	checkdir(CD,verbose):
1653
1654	ARGUMENTS:
1655	CD -- Directory
1656	verbose -- Flag verbose output (default: None)
1657
1658	DESCRIPTION:
1659	If CD does not exist it will be created if possible
1660	"""
1661
1662	import os
1663	import os.path
1664
1665	if CD[-1] != os.sep:
1666	CD = CD + os.sep # Add separator for directories
1667
1668	CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1669	if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1670	try:
1671	exitcode=os.mkdir(CD)
1672
1673	# Change access rights if possible
1674	#
1675	if unix:
1676	exitcode=os.system('chmod 777 '+CD)
1677	else:
1678	pass # FIXME: What about acces rights under Windows?
1679	if verbose: print 'MESSAGE: Directory', CD, 'created.'
1680	except:
1681	print 'WARNING: Directory', CD, 'could not be created.'
1682	if unix:
1683	CD = '/tmp/'
1684	else:
1685	CD = 'C:'
1686	print 'Using directory %s instead' %CD
1687
1688	return(CD)
1689
1690	#==============================================================================
1691	# Statistics
1692	#==============================================================================
1693
1694	def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
1695	compression):
1696	"""Add stats entry
1697
1698	USAGE:
1699	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1700
1701	DESCRIPTION:
1702	Make one entry in the stats file about one cache hit recording time saved
1703	and other statistics. The data are used by the function cachestat.
1704	"""
1705
1706	import os, time
1707
1708	try:
1709	TimeTuple = time.localtime(time.time())
1710	extension = time.strftime('%b%Y',TimeTuple)
1711	SFN = CD+statsfile+'.'+extension
1712	#statfile = open(SFN,'a')
1713	(statfile, dummy) = myopen(SFN,'a',compression=0)
1714
1715	# Change access rights if possible
1716	#
1717	#if unix:
1718	# try:
1719	# exitcode=os.system('chmod 666 '+SFN)
1720	# except:
1721	# pass
1722	except:
1723	print 'Warning: Stat file could not be opened'
1724
1725	try:
1726	if os.environ.has_key('USER'):
1727	user = os.environ['USER']
1728	else:
1729	user = 'Nobody'
1730
1731	date = time.asctime(TimeTuple)
1732
1733	if Retrieved:
1734	hit = '1'
1735	else:
1736	hit = '0'
1737
1738	# Get size of result file
1739	#
1740	if compression:
1741	stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1742	else:
1743	stats = os.stat(CD+FN+'_'+file_types[0])
1744
1745	if stats:
1746	size = stats[6]
1747	else:
1748	size = -1 # Error condition, but don't crash. This is just statistics
1749
1750	# Build entry
1751	#
1752	entry = date + ',' +\
1753	user + ',' +\
1754	FN + ',' +\
1755	str(int(size)) + ',' +\
1756	str(compression) + ',' +\
1757	hit + ',' +\
1758	str(reason) + ',' +\
1759	str(round(comptime,4)) + ',' +\
1760	str(round(loadtime,4)) +\
1761	CR
1762
1763	statfile.write(entry)
1764	statfile.close()
1765	except:
1766	print 'Warning: Writing of stat file failed'
1767
1768	# -----------------------------------------------------------------------------
1769
1770	# FIXME: should take cachedir as an optional arg
1771	#
1772	def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1773	""" List caching statistics.
1774
1775	USAGE:
1776	__cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1777
1778	Generate statistics of caching efficiency.
1779	The parameter sortidx determines by what field lists are sorted.
1780	If the optional keyword period is set to -1,
1781	all available caching history is used.
1782	If it is 0 only the current month is used.
1783	Future versions will include more than one month....
1784	OMN 20/8/2000
1785	"""
1786
1787	import os
1788	import os.path
1789	from string import split, rstrip, find
1790	from time import strptime, localtime, strftime, mktime, ctime
1791
1792	# sortidx = 4 # Index into Fields[1:]. What to sort by.
1793
1794	Fields = ['Name', 'Hits', 'Exec(s)', \
1795	'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1796	Widths = [25,7,9,9,9,9,13]
1797	#Types = ['s','d','d','d','d','.2f','d']
1798	Types = ['s','d','.2f','.2f','.2f','.2f','d']
1799
1800	Dictnames = ['Function', 'User']
1801
1802	if not cachedir:
1803	cachedir = checkdir(options['cachedir'])
1804
1805	SD = os.path.expanduser(cachedir) # Expand ~ or ~user in pathname
1806
1807	if period == -1: # Take all available stats
1808	SFILENAME = statsfile
1809	else: # Only stats from current month
1810	# MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1811	TimeTuple = localtime(time())
1812	extension = strftime('%b%Y',TimeTuple)
1813	SFILENAME = statsfile+'.'+extension
1814
1815	DIRLIST = os.listdir(SD)
1816	SF = []
1817	for FN in DIRLIST:
1818	if find(FN,SFILENAME) >= 0:
1819	SF.append(FN)
1820
1821	blocksize = 15000000
1822	total_read = 0
1823	total_hits = 0
1824	total_discarded = 0
1825	firstday = mktime(strptime('2030','%Y'))
1826	# FIXME: strptime don't exist in WINDOWS ?
1827	lastday = 0
1828
1829	FuncDict = {}
1830	UserDict = {}
1831	for FN in SF:
1832	input = open(SD+FN,'r')
1833	print 'Reading file ', SD+FN
1834
1835	while 1:
1836	A = input.readlines(blocksize)
1837	if len(A) == 0: break
1838	total_read = total_read + len(A)
1839	for record in A:
1840	record = tuple(split(rstrip(record),','))
1841	#print record, len(record)
1842
1843	if len(record) == 9:
1844	timestamp = record[0]
1845
1846	try:
1847	t = mktime(strptime(timestamp))
1848	except:
1849	total_discarded = total_discarded + 1
1850	continue
1851
1852	if t > lastday:
1853	lastday = t
1854	if t < firstday:
1855	firstday = t
1856
1857	user = record[1]
1858	func = record[2]
1859
1860	# Strip hash-stamp off
1861	#
1862	i = find(func,'[')
1863	func = func[:i]
1864
1865	size = float(record[3])
1866
1867	# Compression kepword can be Boolean
1868	if record[4] in ['True', '1']:
1869	compression = 1
1870	elif record[4] in ['False', '0']:
1871	compression = 0
1872	else:
1873	print 'Unknown value of compression', record[4]
1874	print record
1875	total_discarded = total_discarded + 1
1876	continue
1877
1878	#compression = int(record[4]) # Can be Boolean
1879	hit = int(record[5])
1880	reason = int(record[6]) # Not used here
1881	cputime = float(record[7])
1882	loadtime = float(record[8])
1883
1884	if hit:
1885	total_hits = total_hits + 1
1886	saving = cputime-loadtime
1887
1888	if cputime != 0:
1889	rel_saving = round(100.0*saving/cputime,2)
1890	else:
1891	#rel_saving = round(1.0*saving,2)
1892	rel_saving = 100.0 - round(1.0*saving,2) # A bit of a hack
1893
1894	info = [1,cputime,loadtime,saving,rel_saving,size]
1895
1896	UpdateDict(UserDict,user,info)
1897	UpdateDict(FuncDict,func,info)
1898	else:
1899	pass #Stats on recomputations and their reasons could go in here
1900
1901	else:
1902	#print 'Record discarded'
1903	#print record
1904	total_discarded = total_discarded + 1
1905
1906	input.close()
1907
1908	# Compute averages of all sums and write list
1909	#
1910
1911	if total_read == 0:
1912	printline(Widths,'=')
1913	print 'CACHING STATISTICS: No valid records read'
1914	printline(Widths,'=')
1915	return
1916
1917	print
1918	printline(Widths,'=')
1919	print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
1920	printline(Widths,'=')
1921	#print ' Period:', ctime(firstday), 'to', ctime(lastday)
1922	print ' Total number of valid records', total_read
1923	print ' Total number of discarded records', total_discarded
1924	print ' Total number of hits', total_hits
1925	print
1926
1927	print ' Fields', Fields[2:], 'are averaged over number of hits'
1928	print ' Time is measured in seconds and size in bytes'
1929	print ' Tables are sorted by', Fields[1:][sortidx]
1930
1931	# printline(Widths,'-')
1932
1933	if showuser:
1934	Dictionaries = [FuncDict, UserDict]
1935	else:
1936	Dictionaries = [FuncDict]
1937
1938	i = 0
1939	for Dict in Dictionaries:
1940	for key in Dict.keys():
1941	rec = Dict[key]
1942	for n in range(len(rec)):
1943	if n > 0:
1944	rec[n] = round(1.0*rec[n]/rec[0],2)
1945	Dict[key] = rec
1946
1947	# Sort and output
1948	#
1949	keylist = SortDict(Dict,sortidx)
1950
1951	# Write Header
1952	#
1953	print
1954	#print Dictnames[i], 'statistics:'; i=i+1
1955	printline(Widths,'-')
1956	n = 0
1957	for s in Fields:
1958	if s == Fields[0]: # Left justify
1959	s = Dictnames[i] + ' ' + s; i=i+1
1960	exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
1961	else:
1962	exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
1963	print
1964	printline(Widths,'-')
1965
1966	# Output Values
1967	#
1968	for key in keylist:
1969	rec = Dict[key]
1970	n = 0
1971	if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
1972	exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
1973	for val in rec:
1974	exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
1975	print
1976	print
1977
1978	#==============================================================================
1979	# Auxiliary stats functions
1980	#==============================================================================
1981
1982	def UpdateDict(Dict,key,info):
1983	"""Update dictionary by adding new values to existing.
1984
1985	USAGE:
1986	UpdateDict(Dict,key,info)
1987	"""
1988
1989	if Dict.has_key(key):
1990	dinfo = Dict[key]
1991	for n in range(len(dinfo)):
1992	dinfo[n] = info[n] + dinfo[n]
1993	else:
1994	dinfo = info[:] # Make a copy of info list
1995
1996	Dict[key] = dinfo
1997	return Dict
1998
1999	# -----------------------------------------------------------------------------
2000
2001	def SortDict(Dict,sortidx=0):
2002	"""Sort dictionary
2003
2004	USAGE:
2005	SortDict(Dict,sortidx):
2006
2007	DESCRIPTION:
2008	Sort dictionary of lists according field number 'sortidx'
2009	"""
2010
2011	import types
2012
2013	sortlist = []
2014	keylist = Dict.keys()
2015	for key in keylist:
2016	rec = Dict[key]
2017	if not type(rec) in [types.ListType, types.TupleType]:
2018	rec = [rec]
2019
2020	if sortidx > len(rec)-1:
2021	if options['verbose']:
2022	print 'ERROR: Sorting index to large, sortidx = ', sortidx
2023	raise IndexError
2024
2025	val = rec[sortidx]
2026	sortlist.append(val)
2027
2028	A = map(None,sortlist,keylist)
2029	A.sort()
2030	keylist = map(lambda x: x[1], A) # keylist sorted by sortidx
2031
2032	return(keylist)
2033
2034	# -----------------------------------------------------------------------------
2035
2036	def printline(Widths,char):
2037	"""Print textline in fixed field.
2038
2039	USAGE:
2040	printline(Widths,char)
2041	"""
2042
2043	s = ''
2044	for n in range(len(Widths)):
2045	s = s+Widths[n]*char
2046	if n > 0:
2047	s = s+char
2048
2049	print s
2050
2051	#==============================================================================
2052	# Messages
2053	#==============================================================================
2054
2055	def msg1(funcname,args,kwargs,reason):
2056	"""Message 1
2057
2058	USAGE:
2059	msg1(funcname,args,kwargs,reason):
2060	"""
2061
2062	import string
2063	#print 'MESSAGE (caching.py): Evaluating function', funcname,
2064
2065	print_header_box('Evaluating function %s' %funcname)
2066
2067	msg7(args,kwargs)
2068	msg8(reason)
2069
2070	print_footer()
2071
2072	#
2073	# Old message
2074	#
2075	#args_present = 0
2076	#if args:
2077	# if len(args) == 1:
2078	# print 'with argument', mkargstr(args[0], textwidth2),
2079	# else:
2080	# print 'with arguments', mkargstr(args, textwidth2),
2081	# args_present = 1
2082	#
2083	#if kwargs:
2084	# if args_present:
2085	# word = 'and'
2086	# else:
2087	# word = 'with'
2088	#
2089	# if len(kwargs) == 1:
2090	# print word + ' keyword argument', mkargstr(kwargs, textwidth2)
2091	# else:
2092	# print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
2093	# args_present = 1
2094	#else:
2095	# print # Newline when no keyword args present
2096	#
2097	#if not args_present:
2098	# print '', # Default if no args or kwargs present
2099
2100
2101
2102	# -----------------------------------------------------------------------------
2103
2104	def msg2(funcname,args,kwargs,comptime,reason):
2105	"""Message 2
2106
2107	USAGE:
2108	msg2(funcname,args,kwargs,comptime,reason)
2109	"""
2110
2111	import string
2112
2113	#try:
2114	# R = Reason_msg[reason]
2115	#except:
2116	# R = 'Unknown reason'
2117
2118	#print_header_box('Caching statistics (storing) - %s' %R)
2119	print_header_box('Caching statistics (storing)')
2120
2121	msg6(funcname,args,kwargs)
2122	msg8(reason)
2123
2124	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2125
2126	# -----------------------------------------------------------------------------
2127
2128	def msg3(savetime, CD, FN, deps,compression):
2129	"""Message 3
2130
2131	USAGE:
2132	msg3(savetime, CD, FN, deps,compression)
2133	"""
2134
2135	import string
2136	print string.ljust('\| Loading time:', textwidth1) + str(round(savetime,2)) + \
2137	' seconds (estimated)'
2138	msg5(CD,FN,deps,compression)
2139
2140	# -----------------------------------------------------------------------------
2141
2142	def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
2143	"""Message 4
2144
2145	USAGE:
2146	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
2147	"""
2148
2149	import string
2150
2151	print_header_box('Caching statistics (retrieving)')
2152
2153	msg6(funcname,args,kwargs)
2154	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2155	print string.ljust('\| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
2156	print string.ljust('\| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
2157	' seconds'
2158	msg5(CD,FN,deps,compression)
2159
2160	# -----------------------------------------------------------------------------
2161
2162	def msg5(CD,FN,deps,compression):
2163	"""Message 5
2164
2165	USAGE:
2166	msg5(CD,FN,deps,compression)
2167
2168	DESCRIPTION:
2169	Print dependency stats. Used by msg3 and msg4
2170	"""
2171
2172	import os, time, string
2173
2174	print '\|'
2175	print string.ljust('\| Caching dir: ', textwidth1) + CD
2176
2177	if compression:
2178	suffix = '.z'
2179	bytetext = 'bytes, compressed'
2180	else:
2181	suffix = ''
2182	bytetext = 'bytes'
2183
2184	for file_type in file_types:
2185	file_name = FN + '_' + file_type + suffix
2186	print string.ljust('\| ' + file_type + ' file: ', textwidth1) + file_name,
2187	stats = os.stat(CD+file_name)
2188	print '('+ str(stats[6]) + ' ' + bytetext + ')'
2189
2190	print '\|'
2191	if len(deps) > 0:
2192	print '\| Dependencies: '
2193	dependencies = deps.keys()
2194	dlist = []; maxd = 0
2195	tlist = []; maxt = 0
2196	slist = []; maxs = 0
2197	for d in dependencies:
2198	stats = deps[d]
2199	t = time.ctime(stats[1])
2200	s = str(stats[0])
2201	#if s[-1] == 'L':
2202	# s = s[:-1] # Strip rightmost 'long integer' L off.
2203	# # FIXME: Unnecessary in versions later than 1.5.2
2204
2205	if len(d) > maxd: maxd = len(d)
2206	if len(t) > maxt: maxt = len(t)
2207	if len(s) > maxs: maxs = len(s)
2208	dlist.append(d)
2209	tlist.append(t)
2210	slist.append(s)
2211
2212	for n in range(len(dlist)):
2213	d = string.ljust(dlist[n]+':', maxd+1)
2214	t = string.ljust(tlist[n], maxt)
2215	s = string.rjust(slist[n], maxs)
2216
2217	print '\| ', d, t, ' ', s, 'bytes'
2218	else:
2219	print '\| No dependencies'
2220	print_footer()
2221
2222	# -----------------------------------------------------------------------------
2223
2224	def msg6(funcname,args,kwargs):
2225	"""Message 6
2226
2227	USAGE:
2228	msg6(funcname,args,kwargs)
2229	"""
2230
2231	import string
2232	print string.ljust('\| Function:', textwidth1) + funcname
2233
2234	msg7(args,kwargs)
2235
2236	# -----------------------------------------------------------------------------
2237
2238	def msg7(args,kwargs):
2239	"""Message 7
2240
2241	USAGE:
2242	msg7(args,kwargs):
2243	"""
2244
2245	import string
2246
2247	args_present = 0
2248	if args:
2249	if len(args) == 1:
2250	print string.ljust('\| Argument:', textwidth1) + mkargstr(args[0], \
2251	textwidth2)
2252	else:
2253	print string.ljust('\| Arguments:', textwidth1) + \
2254	mkargstr(args, textwidth2)
2255	args_present = 1
2256
2257	if kwargs:
2258	if len(kwargs) == 1:
2259	print string.ljust('\| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
2260	textwidth2)
2261	else:
2262	print string.ljust('\| Keyword Args:', textwidth1) + \
2263	mkargstr(kwargs, textwidth2)
2264	args_present = 1
2265
2266	if not args_present:
2267	print '\| No arguments' # Default if no args or kwargs present
2268
2269	# -----------------------------------------------------------------------------
2270
2271	def msg8(reason):
2272	"""Message 8
2273
2274	USAGE:
2275	msg8(reason):
2276	"""
2277
2278	import string
2279
2280	try:
2281	R = Reason_msg[reason]
2282	except:
2283	R = 'Unknown'
2284
2285	print string.ljust('\| Reason:', textwidth1) + R
2286
2287	# -----------------------------------------------------------------------------
2288
2289	def print_header_box(line):
2290	"""Print line in a nice box.
2291
2292	USAGE:
2293	print_header_box(line)
2294
2295	"""
2296	global textwidth3
2297
2298	import time
2299
2300	time_stamp = time.ctime(time.time())
2301	line = time_stamp + '. ' + line
2302
2303	N = len(line) + 1
2304	s = '+' + '-'*N + CR
2305
2306	print s + '\| ' + line + CR + s,
2307
2308	textwidth3 = N
2309
2310	# -----------------------------------------------------------------------------
2311
2312	def print_footer():
2313	"""Print line same width as that of print_header_box.
2314	"""
2315
2316	N = textwidth3
2317	s = '+' + '-'*N + CR
2318
2319	print s
2320
2321	# -----------------------------------------------------------------------------
2322
2323	def mkargstr(args, textwidth, argstr = ''):
2324	""" Generate a string containing first textwidth characters of arguments.
2325
2326	USAGE:
2327	mkargstr(args, textwidth, argstr = '')
2328
2329	DESCRIPTION:
2330	Exactly the same as str(args) possibly followed by truncation,
2331	but faster if args is huge.
2332	"""
2333
2334	import types
2335
2336	WasTruncated = 0
2337
2338	if not type(args) in [types.TupleType, types.ListType, types.DictType]:
2339	if type(args) == types.StringType:
2340	argstr = argstr + "'"+str(args)+"'"
2341	else:
2342	#Truncate large Numeric arrays before using str()
2343	import Numeric
2344	if type(args) == Numeric.ArrayType:
2345	# if len(args.flat) > textwidth:
2346	# Changed by Duncan and Nick 21/2/07 .flat has problems with
2347	# non-contigous arrays and ravel is equal to .flat except it
2348	# can work with non-contiguous arrays
2349	if len(Numeric.ravel(args)) > textwidth:
2350	args = 'Array: ' + str(args.shape)
2351
2352	argstr = argstr + str(args)
2353	else:
2354	if type(args) == types.DictType:
2355	argstr = argstr + "{"
2356	for key in args.keys():
2357	argstr = argstr + mkargstr(key, textwidth) + ": " + \
2358	mkargstr(args[key], textwidth) + ", "
2359	if len(argstr) > textwidth:
2360	WasTruncated = 1
2361	break
2362	argstr = argstr[:-2] # Strip off trailing comma
2363	argstr = argstr + "}"
2364
2365	else:
2366	if type(args) == types.TupleType:
2367	lc = '('
2368	rc = ')'
2369	else:
2370	lc = '['
2371	rc = ']'
2372	argstr = argstr + lc
2373	for arg in args:
2374	argstr = argstr + mkargstr(arg, textwidth) + ', '
2375	if len(argstr) > textwidth:
2376	WasTruncated = 1
2377	break
2378
2379	# Strip off trailing comma and space unless singleton tuple
2380	#
2381	if type(args) == types.TupleType and len(args) == 1:
2382	argstr = argstr[:-1]
2383	else:
2384	argstr = argstr[:-2]
2385	argstr = argstr + rc
2386
2387	if len(argstr) > textwidth:
2388	WasTruncated = 1
2389
2390	if WasTruncated:
2391	argstr = argstr[:textwidth]+'...'
2392	return(argstr)
2393
2394	# -----------------------------------------------------------------------------
2395
2396	def test_OK(msg):
2397	"""Print OK msg if test is OK.
2398
2399	USAGE
2400	test_OK(message)
2401	"""
2402
2403	import string
2404
2405	print string.ljust(msg, textwidth4) + ' - OK'
2406
2407	#raise StandardError
2408
2409	# -----------------------------------------------------------------------------
2410
2411	def test_error(msg):
2412	"""Print error if test fails.
2413
2414	USAGE
2415	test_error(message)
2416	"""
2417
2418	print 'ERROR (caching.test): %s' %msg
2419	print 'Please send this code example and output to '
2420	print 'Ole.Nielsen@anu.edu.au'
2421	print
2422	print
2423
2424	#import sys
2425	#sys.exit()
2426	raise StandardError
2427
2428	#-------------------------------------------------------------
2429	if __name__ == "__main__":
2430	pass

Note: See TracBrowser for help on using the repository browser.

Download in other formats: