Context Navigation

source: anuga_core/source/anuga/caching/caching.py @ 5860

Last change on this file since 5860 was 5860, checked in by ole, 15 years ago
Thought of testing hash-collision and found issue with comparison This is now fixed and tests verified.
File size: 72.1 KB

Line
1	# =============================================================================
2	# caching.py - Supervised caching of function results.
3	# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4	# Australian National University (1999-2003)
5	# Geoscience Australia (2003-present)
6	#
7	# This program is free software; you can redistribute it and/or modify
8	# it under the terms of the GNU General Public License as published by
9	# the Free Software Foundation; either version 2 of the License, or
10	# (at your option) any later version.
11	#
12	# This program is distributed in the hope that it will be useful,
13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	# GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16	# for more details.
17	#
18	# You should have received a copy of the GNU General Public License
19	# along with this program; if not, write to the Free Software
20	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
21	#
22	#
23	# Contact address: Ole.Nielsen@ga.gov.au
24	#
25	# Version 1.5.6 February 2002
26	# =============================================================================
27
28	"""Module caching.py - Supervised caching of function results.
29
30	Public functions:
31
32	cache(func,args) -- Cache values returned from func given args.
33	cachestat() -- Reports statistics about cache hits and time saved.
34	test() -- Conducts a basic test of the caching functionality.
35
36	See doc strings of individual functions for detailed documentation.
37	"""
38
39	# -----------------------------------------------------------------------------
40	# Initialisation code
41
42	# Determine platform
43	#
44	from os import getenv
45	import types
46
47	import os
48	if os.name in ['nt', 'dos', 'win32', 'what else?']:
49	unix = 0
50	else:
51	unix = 1
52
53	cache_dir = '.python_cache'
54	# Make default caching directory name
55	#
56	if unix:
57	homedir = getenv('INUNDATIONHOME')
58	if homedir is None:
59	homedir = '~'
60	else:
61	homedir = homedir + os.sep + '.cache'
62	# Since homedir will be a group area, individually label the caches
63	user = getenv('LOGNAME')
64	if user is not None:
65	cache_dir += '_' + user
66
67	CR = '\n'
68	else:
69	homedir = 'c:'
70	CR = '\r\n' #FIXME: Not tested under windows
71
72	cachedir = homedir + os.sep + cache_dir + os.sep
73
74	# -----------------------------------------------------------------------------
75	# Options directory with default values - to be set by user
76	#
77
78	options = {
79	'cachedir': cachedir, # Default cache directory
80	'maxfiles': 1000000, # Maximum number of cached files
81	'savestat': 1, # Log caching info to stats file
82	'verbose': 1, # Write messages to standard output
83	'bin': 1, # Use binary format (more efficient)
84	'compression': 1, # Use zlib compression
85	'bytecode': 0, # Recompute if bytecode has changed
86	'expire': 0 # Automatically remove files that have been accessed
87	# least recently
88	}
89
90	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
91
92	def set_option(key, value):
93	"""Function to set values in the options directory.
94
95	USAGE:
96	set_option(key, value)
97
98	ARGUMENTS:
99	key -- Key in options dictionary. (Required)
100	value -- New value for key. (Required)
101
102	DESCRIPTION:
103	Function to set values in the options directory.
104	Raises an exception if key is not in options.
105	"""
106
107	if options.has_key(key):
108	options[key] = value
109	else:
110	raise KeyError(key) # Key not found, raise an exception
111
112	# -----------------------------------------------------------------------------
113	# Function cache - the main routine
114
115	def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
116	verbose=None, compression=None, evaluate=0, test=0, clear=0,
117	return_filename=0):
118	"""Supervised caching of function results.
119
120	USAGE:
121	result = cache(func, args, kwargs, dependencies, cachedir, verbose,
122	compression, evaluate, test, return_filename)
123
124	ARGUMENTS:
125	func -- Function object (Required)
126	args -- Arguments to func (Default: ())
127	kwargs -- Keyword arguments to func (Default: {})
128	dependencies -- Filenames that func depends on (Default: None)
129	cachedir -- Directory for cache files (Default: options['cachedir'])
130	verbose -- Flag verbose output to stdout
131	(Default: options['verbose'])
132	compression -- Flag zlib compression (Default: options['compression'])
133	evaluate -- Flag forced evaluation of func (Default: 0)
134	test -- Flag test for cached results (Default: 0)
135	clear -- Flag delete cached results (Default: 0)
136	return_filename -- Flag return of cache filename (Default: 0)
137
138	DESCRIPTION:
139	A Python function call of the form
140
141	result = func(arg1,...,argn)
142
143	can be replaced by
144
145	from caching import cache
146	result = cache(func,(arg1,...,argn))
147
148	The latter form returns the same output as the former but reuses cached
149	results if the function has been computed previously in the same context.
150	'result' and the arguments can be simple types, tuples, list, dictionaries or
151	objects, but not unhashable types such as functions or open file objects.
152	The function 'func' may be a member function of an object or a module.
153
154	This type of caching is particularly useful for computationally intensive
155	functions with few frequently used combinations of input arguments. Note that
156	if the inputs or output are very large caching might not save time because
157	disc access may dominate the execution time.
158
159	If the function definition changes after a result has been cached it will be
160	detected by examining the functions bytecode (co_code, co_consts,
161	func_defualts, co_argcount) and it will be recomputed.
162
163	LIMITATIONS:
164	1 Caching uses the apply function and will work with anything that can be
165	pickled, so any limitation in apply or pickle extends to caching.
166	2 A function to be cached should not depend on global variables
167	as wrong results may occur if globals are changed after a result has
168	been cached.
169
170	-----------------------------------------------------------------------------
171	Additional functionality:
172
173	Keyword args
174	Keyword arguments (kwargs) can be added as a dictionary of keyword: value
175	pairs, following the syntax of the built-in function apply().
176	A Python function call of the form
177
178	result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)
179
180	is then cached as follows
181
182	from caching import cache
183	result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
184
185	The default value of kwargs is {}
186
187	Explicit dependencies:
188	The call
189	cache(func,(arg1,...,argn),dependencies = <list of filenames>)
190	Checks the size, creation time and modification time of each listed file.
191	If any file has changed the function is recomputed and the results stored
192	again.
193
194	Specify caching directory:
195	The call
196	cache(func,(arg1,...,argn), cachedir = <cachedir>)
197	designates <cachedir> where cached data are stored. Use ~ to indicate users
198	home directory - not $HOME. The default is ~/.python_cache on a UNIX
199	platform and c:/.python_cache on a Win platform.
200
201	Silent operation:
202	The call
203	cache(func,(arg1,...,argn),verbose=0)
204	suppresses messages to standard output.
205
206	Compression:
207	The call
208	cache(func,(arg1,...,argn),compression=0)
209	disables compression. (Default: compression=1). If the requested compressed
210	or uncompressed file is not there, it'll try the other version.
211
212	Forced evaluation:
213	The call
214	cache(func,(arg1,...,argn),evaluate=1)
215	forces the function to evaluate even though cached data may exist.
216
217	Testing for presence of cached result:
218	The call
219	cache(func,(arg1,...,argn),test=1)
220	retrieves cached result if it exists, otherwise None. The function will not
221	be evaluated. If both evaluate and test are switched on, evaluate takes
222	precedence.
223	??NOTE: In case of hash collisions, this may return the wrong result as
224	??it only checks if a cached result is present.
225
226	Obtain cache filenames:
227	The call
228	cache(func,(arg1,...,argn),return_filename=1)
229	returns the hashed base filename under which this function and its
230	arguments would be cached
231
232	Clearing cached results:
233	The call
234	cache(func,'clear')
235	clears all cached data for 'func' and
236	cache('clear')
237	clears all cached data.
238
239	NOTE: The string 'clear' can be passed an argument to func using
240	cache(func,('clear',)) or cache(func,tuple(['clear'])).
241
242	New form of clear:
243	cache(func,(arg1,...,argn),clear=1)
244	clears cached data for particular combination func and args
245
246	"""
247
248	# Imports and input checks
249	#
250	import types, time, string
251
252	if not cachedir:
253	cachedir = options['cachedir']
254
255	if verbose == None: # Do NOT write 'if not verbose:', it could be zero.
256	verbose = options['verbose']
257
258	if compression == None: # Do NOT write 'if not compression:',
259	# it could be zero.
260	compression = options['compression']
261
262	# Create cache directory if needed
263	CD = checkdir(cachedir,verbose)
264
265	# Handle the case cache('clear')
266	if type(func) == types.StringType:
267	if string.lower(func) == 'clear':
268	clear_cache(CD,verbose=verbose)
269	return
270
271	# Handle the case cache(func, 'clear')
272	if type(args) == types.StringType:
273	if string.lower(args) == 'clear':
274	clear_cache(CD,func,verbose=verbose)
275	return
276
277	# Force singleton arg into a tuple
278	if type(args) != types.TupleType:
279	args = tuple([args])
280
281	# Check that kwargs is a dictionary
282	if type(kwargs) != types.DictType:
283	raise TypeError
284
285	# Hash arguments (and keyword args) to integer
286	arghash = myhash((args, kwargs))
287
288	# Get sizes and timestamps for files listed in dependencies.
289	# Force singletons into a tuple.
290	if dependencies and type(dependencies) != types.TupleType \
291	and type(dependencies) != types.ListType:
292	dependencies = tuple([dependencies])
293	deps = get_depstats(dependencies)
294
295	# Extract function name from func object
296	funcname = get_funcname(func)
297
298	# Create cache filename
299	FN = funcname+'['+`arghash`+']' # The symbol '(' does not work under unix
300
301	if return_filename:
302	return(FN)
303
304	if clear:
305	for file_type in file_types:
306	file_name = CD+FN+'_'+file_type
307	for fn in [file_name, file_name + '.z']:
308	if os.access(fn, os.F_OK):
309	if unix:
310	os.remove(fn)
311	else:
312	# FIXME: os.remove doesn't work under windows
313	os.system('del '+fn)
314	if verbose is True:
315	print 'MESSAGE (caching): File %s deleted' %fn
316	##else:
317	## print '%s was not accessed' %fn
318	return None
319
320
321	#-------------------------------------------------------------------
322
323	# Check if previous computation has been cached
324	if evaluate:
325	Retrieved = None # Force evaluation of func regardless of caching status.
326	reason = 5
327	else:
328	T, FN, Retrieved, reason, comptime, loadtime, compressed = \
329	CacheLookup(CD, FN, func,
330	args, kwargs,
331	deps,
332	verbose,
333	compression,
334	dependencies)
335
336	if not Retrieved:
337	if test: # Do not attempt to evaluate function
338	T = None
339	else: # Evaluate function and save to cache
340	if verbose:
341
342	msg1(funcname, args, kwargs,reason)
343
344	# Remove expired files automatically
345	if options['expire']:
346	DeleteOldFiles(CD,verbose)
347
348	# Save args before function is evaluated in case
349	# they are modified by function
350	save_args_to_cache(CD,FN,args,kwargs,compression)
351
352	# Execute and time function with supplied arguments
353	t0 = time.time()
354	T = apply(func,args,kwargs)
355	#comptime = round(time.time()-t0)
356	comptime = time.time()-t0
357
358	if verbose:
359	msg2(funcname,args,kwargs,comptime,reason)
360
361	# Save results and estimated loading time to cache
362	loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
363	funcname, dependencies, compression)
364	if verbose:
365	msg3(loadtime, CD, FN, deps, compression)
366	compressed = compression
367
368	if options['savestat'] and (not test or Retrieved):
369	##if options['savestat']:
370	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
371
372	return(T) # Return results in all cases
373
374	# -----------------------------------------------------------------------------
375
376	def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
377	"""Generate statistics of caching efficiency.
378
379	USAGE:
380	cachestat(sortidx, period, showuser, cachedir)
381
382	ARGUMENTS:
383	sortidx -- Index of field by which lists are (default: 4)
384	Legal values are
385	0: 'Name'
386	1: 'Hits'
387	2: 'CPU'
388	3: 'Time Saved'
389	4: 'Gain(%)'
390	5: 'Size'
391	period -- If set to -1 all available caching history is used.
392	If set 0 only the current month is used (default -1).
393	showuser -- Flag for additional table showing user statistics
394	(default: None).
395	cachedir -- Directory for cache files (default: options['cachedir']).
396
397	DESCRIPTION:
398	Logged caching statistics is converted into summaries of the form
399	--------------------------------------------------------------------------
400	Function Name Hits Exec(s) Cache(s) Saved(s) Gain(%) Size
401	--------------------------------------------------------------------------
402	"""
403
404	__cachestat(sortidx, period, showuser, cachedir)
405	return
406
407	# -----------------------------------------------------------------------------
408
409	#Has mostly been moved to proper unit test
410	def test(cachedir=None,verbose=0,compression=None):
411	"""Test the functionality of caching.
412
413	USAGE:
414	test(verbose)
415
416	ARGUMENTS:
417	verbose -- Flag whether caching will output its statistics (default=0)
418	cachedir -- Directory for cache files (Default: options['cachedir'])
419	compression -- Flag zlib compression (Default: options['compression'])
420	"""
421
422	import string, time
423
424	# Initialise
425	#
426	import caching
427	reload(caching)
428
429	if not cachedir:
430	cachedir = options['cachedir']
431
432	if verbose is None: # Do NOT write 'if not verbose:', it could be zero.
433	verbose = options['verbose']
434
435	if compression == None: # Do NOT write 'if not compression:',
436	# it could be zero.
437	compression = options['compression']
438	else:
439	try:
440	set_option('compression', compression)
441	except:
442	test_error('Set option failed')
443
444	try:
445	import zlib
446	except:
447	print
448	print '* Could not find zlib, default to no-compression *'
449	print '* Installing zlib will improve performance of caching *'
450	print
451	compression = 0
452	set_option('compression', compression)
453
454	print
455	print_header_box('Testing caching module - please stand by')
456	print
457
458	# Define a test function to be cached
459	#
460	def f(a,b,c,N,x=0,y='abcdefg'):
461	"""f(a,b,c,N)
462	Do something time consuming and produce a complex result.
463	"""
464
465	import string
466
467	B = []
468	for n in range(N):
469	s = str(n+2.0/(n + 4.0))+'.a'*10
470	B.append((a,b,c,s,n,x,y))
471	return(B)
472
473	# Check that default cachedir is OK
474	#
475	CD = checkdir(cachedir,verbose)
476
477
478	# Make a dependency file
479	#
480	try:
481	DepFN = CD + 'testfile.tmp'
482	DepFN_wildcard = CD + 'test*.tmp'
483	Depfile = open(DepFN,'w')
484	Depfile.write('We are the knights who say NI!')
485	Depfile.close()
486	test_OK('Wrote file %s' %DepFN)
487	except:
488	test_error('Could not open file %s for writing - check your environment' \
489	% DepFN)
490
491	# Check set_option (and switch stats off
492	#
493	try:
494	set_option('savestat',0)
495	assert(options['savestat'] == 0)
496	test_OK('Set option')
497	except:
498	test_error('Set option failed')
499
500	# Make some test input arguments
501	#
502	N = 5000 #Make N fairly small here
503
504	a = [1,2]
505	b = ('Thou shalt count the number three',4)
506	c = {'Five is right out': 6, (7,8): 9}
507	x = 3
508	y = 'holy hand granate'
509
510	# Test caching
511	#
512	if compression:
513	comprange = 2
514	else:
515	comprange = 1
516
517	for comp in range(comprange):
518
519	# Evaluate and store
520	#
521	try:
522	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
523	verbose=verbose, compression=comp)
524	if comp:
525	test_OK('Caching evaluation with compression')
526	else:
527	test_OK('Caching evaluation without compression')
528	except:
529	if comp:
530	test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
531	else:
532	test_error('Caching evaluation failed - try caching.test(verbose=1)')
533
534	# Retrieve
535	#
536	try:
537	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
538	compression=comp)
539
540	if comp:
541	test_OK('Caching retrieval with compression')
542	else:
543	test_OK('Caching retrieval without compression')
544	except:
545	if comp:
546	test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
547	else:
548	test_error('Caching retrieval failed - try caching.test(verbose=1)')
549
550	# Reference result
551	#
552	T3 = f(a,b,c,N,x=x,y=y) # Compute without caching
553
554	if T1 == T2 and T2 == T3:
555	if comp:
556	test_OK('Basic caching functionality (with compression)')
557	else:
558	test_OK('Basic caching functionality (without compression)')
559	else:
560	test_error('Cached result does not match computed result')
561
562
563	# Test return_filename
564	#
565	try:
566	FN = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
567	return_filename=1)
568	assert(FN[:2] == 'f[')
569	test_OK('Return of cache filename')
570	except:
571	test_error('Return of cache filename failed')
572
573	# Test existence of cachefiles
574	#
575	try:
576	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
577	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
578	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
579	test_OK('Presence of cache files')
580	datafile.close()
581	argsfile.close()
582	admfile.close()
583	except:
584	test_error('Expected cache files did not exist')
585
586	# Test 'test' function when cache is present
587	#
588	try:
589	#T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
590	# evaluate=1)
591	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
592	assert(T1 == T4)
593
594	test_OK("Option 'test' when cache file present")
595	except:
596	test_error("Option 'test' when cache file present failed")
597
598	# Test that 'clear' works
599	#
600	#try:
601	# caching.cache(f,'clear',verbose=verbose)
602	# test_OK('Clearing of cache files')
603	#except:
604	# test_error('Clear does not work')
605	try:
606	caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)
607	test_OK('Clearing of cache files')
608	except:
609	test_error('Clear does not work')
610
611
612
613	# Test 'test' function when cache is absent
614	#
615	try:
616	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
617	assert(T4 is None)
618	test_OK("Option 'test' when cache absent")
619	except:
620	test_error("Option 'test' when cache absent failed")
621
622	# Test dependencies
623	#
624	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
625	dependencies=DepFN)
626	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
627	dependencies=DepFN)
628
629	if T1 == T2:
630	test_OK('Basic dependencies functionality')
631	else:
632	test_error('Dependencies do not work')
633
634	# Test basic wildcard dependency
635	#
636	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
637	dependencies=DepFN_wildcard)
638
639	if T1 == T3:
640	test_OK('Basic dependencies with wildcard functionality')
641	else:
642	test_error('Dependencies with wildcards do not work')
643
644
645	# Test that changed timestamp in dependencies triggers recomputation
646
647	# Modify dependency file
648	Depfile = open(DepFN,'a')
649	Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
650	Depfile.close()
651
652	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
653	dependencies=DepFN, test = 1)
654
655	if T3 is None:
656	test_OK('Changed dependencies recognised')
657	else:
658	test_error('Changed dependencies not recognised')
659
660	# Test recomputation when dependencies have changed
661	#
662	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
663	dependencies=DepFN)
664	if T1 == T3:
665	test_OK('Recomputed value with changed dependencies')
666	else:
667	test_error('Recomputed value with changed dependencies failed')
668
669	# Performance test (with statistics)
670	# Don't really rely on this as it will depend on specific computer.
671	#
672
673	set_option('savestat',1)
674
675	N = 20*N #Should be large on fast computers...
676	tt = time.time()
677	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
678	t1 = time.time() - tt
679
680	tt = time.time()
681	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
682	t2 = time.time() - tt
683
684	if T1 == T2:
685	if t1 > t2:
686	test_OK('Performance test: relative time saved = %s pct' \
687	%str(round((t1-t2)*100/t1,2)))
688	#else:
689	# print 'WARNING: Performance a bit low - this could be specific to current platform'
690	else:
691	test_error('Basic caching failed for new problem')
692
693	# Test presence of statistics file
694	#
695	try:
696	DIRLIST = os.listdir(CD)
697	SF = []
698	for FN in DIRLIST:
699	if string.find(FN,statsfile) >= 0:
700	fid = open(CD+FN,'r')
701	fid.close()
702	test_OK('Statistics files present')
703	except:
704	test_OK('Statistics files cannot be opened')
705
706	print_header_box('Show sample output of the caching function:')
707
708	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
709	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
710	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
711
712	print_header_box('Show sample output of cachestat():')
713	if unix:
714	cachestat()
715	else:
716	try:
717	import time
718	t = time.strptime('2030','%Y')
719	cachestat()
720	except:
721	print 'caching.cachestat() does not work here, because it'
722	print 'relies on time.strptime() which is unavailable in Windows'
723
724	print
725	test_OK('Caching self test completed')
726
727
728	# Test setoption (not yet implemented)
729	#
730
731
732	#==============================================================================
733	# Auxiliary functions
734	#==============================================================================
735
736	# Import pickler
737	# cPickle is used by functions mysave, myload, and compare
738	#
739	import cPickle # 10 to 100 times faster than pickle
740	pickler = cPickle
741
742	# Local immutable constants
743	#
744	comp_level = 1 # Compression level for zlib.
745	# comp_level = 1 works well.
746	textwidth1 = 16 # Text width of key fields in report forms.
747	#textwidth2 = 132 # Maximal width of textual representation of
748	textwidth2 = 300 # Maximal width of textual representation of
749	# arguments.
750	textwidth3 = 16 # Initial width of separation lines. Is modified.
751	textwidth4 = 50 # Text width in test_OK()
752	statsfile = '.cache_stat' # Basefilename for cached statistics.
753	# It will reside in the chosen cache directory.
754
755	file_types = ['Result', # File name extension for cached function results.
756	'Args', # File name extension for stored function args.
757	'Admin'] # File name extension for administrative info.
758
759	Reason_msg = ['OK', # Verbose reasons for recomputation
760	'No cached result',
761	'Dependencies have changed',
762	'Arguments have changed',
763	'Bytecode has changed',
764	'Recomputation was requested by caller',
765	'Cached file was unreadable']
766
767	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
768
769	def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression,
770	dependencies):
771	"""Determine whether cached result exists and return info.
772
773	USAGE:
774	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
775	CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
776	dependencies)
777
778	INPUT ARGUMENTS:
779	CD -- Cache Directory
780	FN -- Suggested cache file name
781	func -- Function object
782	args -- Tuple of arguments
783	kwargs -- Dictionary of keyword arguments
784	deps -- Dependencies time stamps
785	verbose -- Flag text output
786	compression -- Flag zlib compression
787	dependencies -- Given list of dependencies
788
789	OUTPUT ARGUMENTS:
790	T -- Cached result if present otherwise None
791	FN -- File name under which new results must be saved
792	Retrieved -- True if a valid cached result was found
793	reason -- 0: OK (if Retrieved),
794	1: No cached result,
795	2: Dependencies have changed,
796	3: Arguments have changed
797	4: Bytecode has changed
798	5: Recomputation was forced
799	6: Unreadable file
800	comptime -- Number of seconds it took to computed cachged result
801	loadtime -- Number of seconds it took to load cached result
802	compressed -- Flag (0,1) if cached results were compressed or not
803
804	DESCRIPTION:
805	Determine if cached result exists as follows:
806	Load in saved arguments and bytecode stored under hashed filename.
807	If they are identical to current arguments and bytecode and if dependencies
808	have not changed their time stamp, then return cached result.
809
810	Otherwise return filename under which new results should be cached.
811	Hash collisions are handled recursively by calling CacheLookup again with a
812	modified filename.
813	"""
814
815	import time, string, types
816
817	# Assess whether cached result exists - compressed or not.
818	#
819	if verbose:
820	print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
821	%(CD+FN, file_types[0], file_types[1], file_types[2])
822	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
823	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
824	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
825
826	if verbose is True and deps is not None:
827	print 'Caching: Dependencies are', deps.keys()
828
829	if not (argsfile and datafile and admfile) or \
830	not (compressed0 == compressed1 and compressed0 == compressed2):
831	# Cached result does not exist or files were compressed differently
832	#
833	# This will ensure that evaluation will take place unless all files are
834	# present.
835
836	reason = 1
837	return(None,FN,None,reason,None,None,None) #Recompute using same filename
838
839	compressed = compressed0 # Remember if compressed files were actually used
840	datafile.close()
841
842	# Retrieve arguments and adm. info
843	#
844	R, reason = myload(argsfile, compressed) # The original arguments
845	argsfile.close()
846
847	if reason > 0:
848	# Recompute using same filename
849	return(None, FN, None, reason, None, None, None)
850	else:
851	(argsref, kwargsref) = R
852
853	R, reason = myload(admfile, compressed)
854	admfile.close()
855
856	if reason > 0:
857	return(None,FN,None,reason,None,None,None) #Recompute using same filename
858
859
860	depsref = R[0] # Dependency statistics
861	comptime = R[1] # The computation time
862	coderef = R[2] # The byte code
863	funcname = R[3] # The function name
864
865	# Check if dependencies have changed
866	#
867	if dependencies and not compare(depsref, deps):
868	if verbose:
869	print 'MESSAGE (caching.py): Dependencies', dependencies, \
870	'have changed - recomputing'
871	# Don't use cached file - recompute
872	reason = 2
873	return(None, FN, None, reason, None, None, None)
874
875	# Get bytecode from func
876	#
877	bytecode = get_bytecode(func)
878
879	#print compare(argsref,args),
880	#print compare(kwargsref,kwargs),
881	#print compare(bytecode,coderef)
882
883	# Check if arguments or bytecode have changed
884	if compare(argsref, args) and compare(kwargsref, kwargs) and \
885	(not options['bytecode'] or compare(bytecode, coderef)):
886
887	# Arguments and dependencies match. Get cached results
888	T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
889	if reason > 0:
890	# Recompute using same FN
891	return(None, FN, None, reason, None, None, None)
892
893	Retrieved = 1
894	reason = 0
895
896	if verbose:
897	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
898
899	if loadtime >= comptime:
900	print 'WARNING (caching.py): Caching did not yield any gain.'
901	print ' Consider executing function ',
902	print '('+funcname+') without caching.'
903	else:
904
905	# Non matching arguments or bytecodes signify a hash-collision.
906	# This is resolved by recursive search of cache filenames
907	# until either a matching or an unused filename is found.
908	#
909	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
910	CacheLookup(CD, FN+'x', func, args, kwargs, deps,
911	verbose, compression, dependencies)
912
913	# DEBUGGING
914	# if not Retrieved:
915	# print 'Arguments did not match'
916	# else:
917	# print 'Match found !'
918
919	# The real reason is that args or bytecodes have changed.
920	# Not that the recursive seach has found an unused filename
921	if not Retrieved:
922	if not compare(bytecode, coderef):
923	reason = 4 # Bytecode has changed
924	else:
925	reason = 3 # Arguments have changed
926
927
928	return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
929
930	# -----------------------------------------------------------------------------
931
932	def clear_cache(CD,func=None, verbose=None):
933	"""Clear cache for func.
934
935	USAGE:
936	clear(CD, func, verbose)
937
938	ARGUMENTS:
939	CD -- Caching directory (required)
940	func -- Function object (default: None)
941	verbose -- Flag verbose output (default: None)
942
943	DESCRIPTION:
944
945	If func == None, clear everything,
946	otherwise clear only files pertaining to func.
947	"""
948
949	import os, re
950
951	if CD[-1] != os.sep:
952	CD = CD+os.sep
953
954	if verbose == None:
955	verbose = options['verbose']
956
957	# FIXME: Windows version needs to be tested
958
959	if func:
960	funcname = get_funcname(func)
961	if verbose:
962	print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
963
964	file_names = os.listdir(CD)
965	for file_name in file_names:
966	#RE = re.search('^' + funcname,file_name) #Inefficient
967	#if RE:
968	if file_name[:len(funcname)] == funcname:
969	if unix:
970	os.remove(CD+file_name)
971	else:
972	os.system('del '+CD+file_name)
973	# FIXME: os.remove doesn't work under windows
974	else:
975	file_names = os.listdir(CD)
976	if len(file_names) > 0:
977	if verbose:
978	print 'MESSAGE (caching.py): Remove the following files:'
979	for file_name in file_names:
980	print file_name
981
982	A = raw_input('Delete (Y/N)[N] ?')
983	else:
984	A = 'Y'
985
986	if A == 'Y' or A == 'y':
987	for file_name in file_names:
988	if unix:
989	os.remove(CD+file_name)
990	else:
991	os.system('del '+CD+file_name)
992	# FIXME: os.remove doesn't work under windows
993	#exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
994
995	# -----------------------------------------------------------------------------
996
997	def DeleteOldFiles(CD,verbose=None):
998	"""Remove expired files
999
1000	USAGE:
1001	DeleteOldFiles(CD,verbose=None)
1002	"""
1003
1004	if verbose == None:
1005	verbose = options['verbose']
1006
1007	maxfiles = options['maxfiles']
1008
1009	# FIXME: Windows version
1010
1011	import os
1012	block = 1000 # How many files to delete per invokation
1013	Files = os.listdir(CD)
1014	numfiles = len(Files)
1015	if not unix: return # FIXME: Windows case ?
1016
1017	if numfiles > maxfiles:
1018	delfiles = numfiles-maxfiles+block
1019	if verbose:
1020	print 'Deleting '+`delfiles`+' expired files:'
1021	os.system('ls -lur '+CD+'* \| head -' + `delfiles`) # List them
1022	os.system('ls -ur '+CD+'* \| head -' + `delfiles` + ' \| xargs /bin/rm')
1023	# Delete them
1024	# FIXME: Replace this with os.listdir and os.remove
1025
1026	# -----------------------------------------------------------------------------
1027
1028	def save_args_to_cache(CD, FN, args, kwargs, compression):
1029	"""Save arguments to cache
1030
1031	USAGE:
1032	save_args_to_cache(CD,FN,args,kwargs,compression)
1033	"""
1034
1035	import time, os, sys, types
1036
1037	(argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1038
1039	if argsfile is None:
1040	msg = 'ERROR (caching): Could not open argsfile for writing: %s' %FN
1041	raise IOError, msg
1042
1043	mysave((args,kwargs),argsfile,compression) # Save args and kwargs to cache
1044	argsfile.close()
1045
1046	# Change access rights if possible
1047	#
1048	#if unix:
1049	# try:
1050	# exitcode=os.system('chmod 666 '+argsfile.name)
1051	# except:
1052	# pass
1053	#else:
1054	# pass # FIXME: Take care of access rights under Windows
1055
1056	return
1057
1058	# -----------------------------------------------------------------------------
1059
1060	def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1061	dependencies, compression):
1062	"""Save computed results T and admin info to cache
1063
1064	USAGE:
1065	save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1066	dependencies, compression)
1067	"""
1068
1069	import time, os, sys, types
1070
1071	(datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1072	(admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1073
1074	if not datafile:
1075	if verbose:
1076	print 'ERROR (caching): Could not open %s' %datafile.name
1077	raise IOError
1078
1079	if not admfile:
1080	if verbose:
1081	print 'ERROR (caching): Could not open %s' %admfile.name
1082	raise IOError
1083
1084	t0 = time.time()
1085
1086	mysave(T,datafile,compression) # Save data to cache
1087	datafile.close()
1088	#savetime = round(time.time()-t0,2)
1089	savetime = time.time()-t0
1090
1091	bytecode = get_bytecode(func) # Get bytecode from function object
1092	admtup = (deps, comptime, bytecode, funcname) # Gather admin info
1093
1094	mysave(admtup,admfile,compression) # Save admin info to cache
1095	admfile.close()
1096
1097	# Change access rights if possible
1098	#
1099	#if unix:
1100	# try:
1101	# exitcode=os.system('chmod 666 '+datafile.name)
1102	# exitcode=os.system('chmod 666 '+admfile.name)
1103	# except:
1104	# pass
1105	#else:
1106	# pass # FIXME: Take care of access rights under Windows
1107
1108	return(savetime)
1109
1110	# -----------------------------------------------------------------------------
1111
1112	def load_from_cache(CD,FN,compression):
1113	"""Load previously cached data from file FN
1114
1115	USAGE:
1116	load_from_cache(CD,FN,compression)
1117	"""
1118
1119	import time
1120
1121	(datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1122	t0 = time.time()
1123	T, reason = myload(datafile,compressed)
1124
1125	loadtime = time.time()-t0
1126	datafile.close()
1127
1128	return T, loadtime, compressed, reason
1129
1130	# -----------------------------------------------------------------------------
1131
1132	def myopen(FN,mode,compression=1):
1133	"""Open file FN using given mode
1134
1135	USAGE:
1136	myopen(FN,mode,compression=1)
1137
1138	ARGUMENTS:
1139	FN -- File name to be opened
1140	mode -- Open mode (as in open)
1141	compression -- Flag zlib compression
1142
1143	DESCRIPTION:
1144	if compression
1145	Attempt first to open FN + '.z'
1146	If this fails try to open FN
1147	else do the opposite
1148	Return file handle plus info about whether it was compressed or not.
1149	"""
1150
1151	import string
1152
1153	# Determine if file exists already (if writing was requested)
1154	# This info is only used to determine if access modes should be set
1155	#
1156	if 'w' in mode or 'a' in mode:
1157	try:
1158	file = open(FN+'.z','r')
1159	file.close()
1160	new_file = 0
1161	except:
1162	try:
1163	file = open(FN,'r')
1164	file.close()
1165	new_file = 0
1166	except:
1167	new_file = 1
1168	else:
1169	new_file = 0 #Assume it exists if mode was not 'w'
1170
1171
1172	compressed = 0
1173	if compression:
1174	try:
1175	file = open(FN+'.z',mode)
1176	compressed = 1
1177	except:
1178	try:
1179	file = open(FN,mode)
1180	except:
1181	file = None
1182	else:
1183	try:
1184	file = open(FN,mode)
1185	except:
1186	try:
1187	file = open(FN+'.z',mode)
1188	compressed = 1
1189	except:
1190	file = None
1191
1192	# Now set access rights if it is a new file
1193	#
1194	if file and new_file:
1195	if unix:
1196	exitcode=os.system('chmod 666 '+file.name)
1197	else:
1198	pass # FIXME: Take care of access rights under Windows
1199
1200	return(file,compressed)
1201
1202	# -----------------------------------------------------------------------------
1203
1204	def myload(file, compressed):
1205	"""Load data from file
1206
1207	USAGE:
1208	myload(file, compressed)
1209	"""
1210
1211	reason = 0
1212	try:
1213	if compressed:
1214	import zlib
1215
1216	RsC = file.read()
1217	try:
1218	Rs = zlib.decompress(RsC)
1219	except:
1220	# File "./caching.py", line 1032, in load_from_cache
1221	# T = myload(datafile,compressed)
1222	# File "./caching.py", line 1124, in myload
1223	# Rs = zlib.decompress(RsC)
1224	# zlib.error: Error -5 while decompressing data
1225	#print 'ERROR (caching): Could not decompress ', file.name
1226	#raise Exception
1227	reason = 6 # Unreadable file
1228	return None, reason
1229
1230
1231	del RsC # Free up some space
1232	R = pickler.loads(Rs)
1233	else:
1234	try:
1235	R = pickler.load(file)
1236	#except EOFError, e:
1237	except:
1238	#Catch e.g., file with 0 length or corrupted
1239	reason = 6 # Unreadable file
1240	return None, reason
1241
1242	except MemoryError:
1243	import sys
1244	if options['verbose']:
1245	print 'ERROR (caching): Out of memory while loading %s, aborting' \
1246	%(file.name)
1247
1248	# Raise the error again for now
1249	#
1250	raise MemoryError
1251
1252	return R, reason
1253
1254	# -----------------------------------------------------------------------------
1255
1256	def mysave(T,file,compression):
1257	"""Save data T to file
1258
1259	USAGE:
1260	mysave(T,file,compression)
1261
1262	"""
1263
1264	bin = options['bin']
1265
1266	if compression:
1267	try:
1268	import zlib
1269	except:
1270	print
1271	print '* Could not find zlib *'
1272	print '* Try to run caching with compression off *'
1273	print "* caching.set_option('compression', 0) *"
1274	raise Exception
1275
1276
1277	try:
1278	Ts = pickler.dumps(T, bin)
1279	except MemoryError:
1280	msg = '****WARNING (caching.py): Could not pickle data for compression.'
1281	msg += ' Try using compression = False'
1282	raise MemoryError, msg
1283	else:
1284	# Compressed pickling
1285	TsC = zlib.compress(Ts, comp_level)
1286	file.write(TsC)
1287	else:
1288	# Uncompressed pickling
1289	pickler.dump(T, file, bin)
1290
1291	# FIXME: This may not work on Windoze network drives.
1292	# The error msg is IOError: [Errno 22] Invalid argument
1293	# Testing with small files was OK, though.
1294	# I think this is an OS problem.
1295
1296	# Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1297
1298	# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1299	#
1300	#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1301	#
1302	#The following extract is taken from the site http://www.python.org:
1303	#
1304	#---------------------------------------------------------------------------------------------
1305	#exception IOError
1306	#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1307	#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1308	#---------------------------------------------------------------------------------------------
1309	#
1310	#The error code(s) that accompany exceptions are described at:
1311	#http://www.python.org/dev/doc/devel//lib/module-errno.html
1312	#
1313	#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1314	#
1315	#exceptions.IOError invalid argument Errno 22
1316
1317	#try:
1318	# pickler.dump(T,file,bin)
1319	#except IOError, e:
1320	# print e
1321	# msg = 'Could not store to %s, bin=%s' %(file, bin)
1322	# raise msg
1323
1324
1325	# -----------------------------------------------------------------------------
1326
1327
1328	def myhash(T, ids=None):
1329	"""Compute hashed integer from a range of inputs.
1330	If T is not hashable being e.g. a tuple T, myhash will recursively
1331	hash the values individually
1332
1333	USAGE:
1334	myhash(T)
1335
1336	ARGUMENTS:
1337	T -- Anything
1338	"""
1339
1340	from types import TupleType, ListType, DictType, InstanceType
1341	from Numeric import ArrayType, average
1342
1343
1344	if type(T) in [TupleType, ListType, DictType, InstanceType]:
1345	# Keep track of unique id's to protect against infinite recursion
1346	if ids is None: ids = []
1347
1348	# Check if T has already been encountered
1349	i = id(T)
1350
1351	if i in ids:
1352	return 0 # T has been hashed already
1353	else:
1354	ids.append(i)
1355
1356
1357
1358	# Start hashing
1359
1360	# On some architectures None, False and True gets different hash values
1361	if T is None:
1362	return(-1)
1363	if T is False:
1364	return(0)
1365	if T is True:
1366	return(1)
1367
1368	# Get hash values for hashable entries
1369	if type(T) in [TupleType, ListType]:
1370	hvals = []
1371	for t in T:
1372	h = myhash(t, ids)
1373	hvals.append(h)
1374	val = hash(tuple(hvals))
1375	elif type(T) == DictType:
1376	# Make dictionary ordering unique
1377	I = T.items()
1378	I.sort()
1379	val = myhash(I, ids)
1380	elif type(T) == ArrayType:
1381	# Use mean value for efficiency
1382	val = hash(average(T.flat))
1383	elif type(T) == InstanceType:
1384	val = myhash(T.__dict__, ids)
1385	else:
1386	try:
1387	val = hash(T)
1388	except:
1389	val = 1
1390
1391	return(val)
1392
1393
1394
1395	def compare(A, B, ids=None):
1396	"""Safe comparison of general objects
1397
1398	USAGE:
1399	compare(A,B)
1400
1401	DESCRIPTION:
1402	Return 1 if A and B they are identical, 0 otherwise
1403	"""
1404
1405	from types import TupleType, ListType, DictType, InstanceType
1406	from Numeric import ArrayType, alltrue
1407
1408	# Keep track of unique id's to protect against infinite recursion
1409	if ids is None: ids = {}
1410
1411	# Check if T has already been encountered
1412	iA = id(A)
1413	iB = id(B)
1414
1415	if ids.has_key((iA, iB)):
1416	# A and B have been compared already
1417	#print 'Found', (iA, iB), A, B
1418	return ids[(iA, iB)]
1419	else:
1420	ids[(iA, iB)] = True
1421
1422
1423	# Check if arguments are of same type
1424	if type(A) != type(B):
1425	return False
1426
1427
1428	# Compare recursively based on argument type
1429	if type(A) in [TupleType, ListType]:
1430	N = len(A)
1431	if len(B) != N:
1432	identical = False
1433	else:
1434	identical = True
1435	for i in range(N):
1436	if not compare(A[i], B[i], ids):
1437	identical = False; break
1438
1439	elif type(A) == DictType:
1440	if len(A) != len(B):
1441	identical = False
1442	else:
1443	# Make dictionary ordering unique
1444	a = A.items(); a.sort()
1445	b = B.items(); b.sort()
1446
1447	identical = compare(a, b, ids)
1448
1449	elif type(A) == ArrayType:
1450	# Use element by element comparison
1451	identical = alltrue(A==B)
1452
1453	elif type(A) == InstanceType:
1454	# Take care of special case where elements are instances
1455	# Base comparison on attributes
1456	identical = compare(A.__dict__,
1457	B.__dict__,
1458	ids)
1459	else:
1460	# Fall back to general code
1461	try:
1462	identical = (A == B)
1463	except:
1464	import pickle
1465	# Use pickle to compare data
1466	# The native pickler must be used
1467	# since the faster cPickle does not
1468	# guarantee a unique translation
1469	try:
1470	identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
1471	except:
1472	identical = False
1473
1474	# Record result of comparison and return
1475	ids[(iA, iB)] = identical
1476
1477	return(identical)
1478
1479
1480	# -----------------------------------------------------------------------------
1481
1482	def nospace(s):
1483	"""Replace spaces in string s with underscores
1484
1485	USAGE:
1486	nospace(s)
1487
1488	ARGUMENTS:
1489	s -- string
1490	"""
1491
1492	import string
1493
1494	newstr = ''
1495	for i in range(len(s)):
1496	if s[i] == ' ':
1497	newstr = newstr+'_'
1498	else:
1499	newstr = newstr+s[i]
1500
1501	return(newstr)
1502
1503	# -----------------------------------------------------------------------------
1504
1505	def get_funcname(func):
1506	"""Retrieve name of function object func (depending on its type)
1507
1508	USAGE:
1509	get_funcname(func)
1510	"""
1511
1512	import types, string
1513
1514	if type(func) == types.FunctionType:
1515	funcname = func.func_name
1516	elif type(func) == types.BuiltinFunctionType:
1517	funcname = func.__name__
1518	else:
1519	tab = string.maketrans("<>'"," ")
1520	tmp = string.translate(`func`,tab)
1521	tmp = string.split(tmp)
1522	funcname = string.join(tmp)
1523
1524	# Truncate memory address as in
1525	# class __main__.Dummy at 0x00A915D0
1526	index = funcname.find('at 0x')
1527	if index >= 0:
1528	funcname = funcname[:index+5] # Keep info that there is an address
1529
1530	funcname = nospace(funcname)
1531	return(funcname)
1532
1533	# -----------------------------------------------------------------------------
1534
1535	def get_bytecode(func):
1536	""" Get bytecode from function object.
1537
1538	USAGE:
1539	get_bytecode(func)
1540	"""
1541
1542	import types
1543
1544	if type(func) == types.FunctionType:
1545	bytecode = func.func_code.co_code
1546	consts = func.func_code.co_consts
1547	argcount = func.func_code.co_argcount
1548	defaults = func.func_defaults
1549	elif type(func) == types.MethodType:
1550	bytecode = func.im_func.func_code.co_code
1551	consts = func.im_func.func_code.co_consts
1552	argcount = func.im_func.func_code.co_argcount
1553	defaults = func.im_func.func_defaults
1554	else:
1555	#raise Exception #Test only
1556	bytecode = None #Built-in functions are assumed not to change
1557	consts = 0
1558	argcount = 0
1559	defaults = 0
1560
1561	return (bytecode, consts, argcount, defaults)
1562
1563	# -----------------------------------------------------------------------------
1564
1565	def get_depstats(dependencies):
1566	""" Build dictionary of dependency files and their size, mod. time and ctime.
1567
1568	USAGE:
1569	get_depstats(dependencies):
1570	"""
1571
1572	import types
1573
1574	#print 'Caching DEBUG: Dependencies are', dependencies
1575	d = {}
1576	if dependencies:
1577
1578	#Expand any wildcards
1579	import glob
1580	expanded_dependencies = []
1581	for FN in dependencies:
1582	expanded_FN = glob.glob(FN)
1583
1584	if expanded_FN == []:
1585	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1586	raise Exception, errmsg
1587
1588	expanded_dependencies += expanded_FN
1589
1590
1591	for FN in expanded_dependencies:
1592	if not type(FN) == types.StringType:
1593	errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1594	errmsg += ' Dependency given: %s' %FN
1595	raise Exception, errmsg
1596	if not os.access(FN,os.F_OK):
1597	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1598	raise Exception, errmsg
1599	(size,atime,mtime,ctime) = filestat(FN)
1600
1601	# We don't use atime because that would cause recomputation every time.
1602	# We don't use ctime because that is irrelevant and confusing for users.
1603	d.update({FN : (size,mtime)})
1604
1605	return(d)
1606
1607	# -----------------------------------------------------------------------------
1608
1609	def filestat(FN):
1610	"""A safe wrapper using os.stat to get basic file statistics
1611	The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1612
1613	USAGE:
1614	filestat(FN)
1615
1616	DESCRIPTION:
1617	Must compile Python with
1618	CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1619	configure
1620	as given in section 8.1.1 Large File Support in the Libray Reference
1621	"""
1622
1623	import os, time
1624
1625	try:
1626	stats = os.stat(FN)
1627	size = stats[6]
1628	atime = stats[7]
1629	mtime = stats[8]
1630	ctime = stats[9]
1631	except:
1632
1633	# Hack to get the results anyway (works only on Unix at the moment)
1634	#
1635	print 'Hack to get os.stat when files are too large'
1636
1637	if unix:
1638	tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1639	# Unique filename, FIXME: Use random number
1640
1641	# Get size and access time (atime)
1642	#
1643	exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1644	(size,atime) = get_lsline(tmp)
1645
1646	# Get size and modification time (mtime)
1647	#
1648	exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1649	(size,mtime) = get_lsline(tmp)
1650
1651	# Get size and ctime
1652	#
1653	exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1654	(size,ctime) = get_lsline(tmp)
1655
1656	try:
1657	exitcode=os.system('rm '+tmp)
1658	# FIXME: Gives error if file doesn't exist
1659	except:
1660	pass
1661	else:
1662	pass
1663	raise Exception # FIXME: Windows case
1664
1665	return(long(size),atime,mtime,ctime)
1666
1667	# -----------------------------------------------------------------------------
1668
1669	def get_lsline(FN):
1670	"""get size and time for filename
1671
1672	USAGE:
1673	get_lsline(file_name)
1674
1675	DESCRIPTION:
1676	Read in one line 'ls -la' item from file (generated by filestat) and
1677	convert time to seconds since epoch. Return file size and time.
1678	"""
1679
1680	import string, time
1681
1682	f = open(FN,'r')
1683	info = f.read()
1684	info = string.split(info)
1685
1686	size = info[4]
1687	week = info[5]
1688	mon = info[6]
1689	day = info[7]
1690	hour = info[8]
1691	year = info[9]
1692
1693	str = week+' '+mon+' '+day+' '+hour+' '+year
1694	timetup = time.strptime(str)
1695	t = time.mktime(timetup)
1696	return(size, t)
1697
1698	# -----------------------------------------------------------------------------
1699
1700	def checkdir(CD,verbose=None):
1701	"""Check or create caching directory
1702
1703	USAGE:
1704	checkdir(CD,verbose):
1705
1706	ARGUMENTS:
1707	CD -- Directory
1708	verbose -- Flag verbose output (default: None)
1709
1710	DESCRIPTION:
1711	If CD does not exist it will be created if possible
1712	"""
1713
1714	import os
1715	import os.path
1716
1717	if CD[-1] != os.sep:
1718	CD = CD + os.sep # Add separator for directories
1719
1720	CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1721	if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1722	try:
1723	exitcode=os.mkdir(CD)
1724
1725	# Change access rights if possible
1726	#
1727	if unix:
1728	exitcode=os.system('chmod 777 '+CD)
1729	else:
1730	pass # FIXME: What about acces rights under Windows?
1731	if verbose: print 'MESSAGE: Directory', CD, 'created.'
1732	except:
1733	print 'WARNING: Directory', CD, 'could not be created.'
1734	if unix:
1735	CD = '/tmp/'
1736	else:
1737	CD = 'C:'
1738	print 'Using directory %s instead' %CD
1739
1740	return(CD)
1741
1742	#==============================================================================
1743	# Statistics
1744	#==============================================================================
1745
1746	def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
1747	compression):
1748	"""Add stats entry
1749
1750	USAGE:
1751	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1752
1753	DESCRIPTION:
1754	Make one entry in the stats file about one cache hit recording time saved
1755	and other statistics. The data are used by the function cachestat.
1756	"""
1757
1758	import os, time
1759
1760	try:
1761	TimeTuple = time.localtime(time.time())
1762	extension = time.strftime('%b%Y',TimeTuple)
1763	SFN = CD+statsfile+'.'+extension
1764	#statfile = open(SFN,'a')
1765	(statfile, dummy) = myopen(SFN,'a',compression=0)
1766
1767	# Change access rights if possible
1768	#
1769	#if unix:
1770	# try:
1771	# exitcode=os.system('chmod 666 '+SFN)
1772	# except:
1773	# pass
1774	except:
1775	print 'Warning: Stat file could not be opened'
1776
1777	try:
1778	if os.environ.has_key('USER'):
1779	user = os.environ['USER']
1780	else:
1781	user = 'Nobody'
1782
1783	date = time.asctime(TimeTuple)
1784
1785	if Retrieved:
1786	hit = '1'
1787	else:
1788	hit = '0'
1789
1790	# Get size of result file
1791	#
1792	if compression:
1793	stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1794	else:
1795	stats = os.stat(CD+FN+'_'+file_types[0])
1796
1797	if stats:
1798	size = stats[6]
1799	else:
1800	size = -1 # Error condition, but don't crash. This is just statistics
1801
1802	# Build entry
1803	#
1804	entry = date + ',' +\
1805	user + ',' +\
1806	FN + ',' +\
1807	str(int(size)) + ',' +\
1808	str(compression) + ',' +\
1809	hit + ',' +\
1810	str(reason) + ',' +\
1811	str(round(comptime,4)) + ',' +\
1812	str(round(loadtime,4)) +\
1813	CR
1814
1815	statfile.write(entry)
1816	statfile.close()
1817	except:
1818	print 'Warning: Writing of stat file failed'
1819
1820	# -----------------------------------------------------------------------------
1821
1822	# FIXME: should take cachedir as an optional arg
1823	#
1824	def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1825	""" List caching statistics.
1826
1827	USAGE:
1828	__cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1829
1830	Generate statistics of caching efficiency.
1831	The parameter sortidx determines by what field lists are sorted.
1832	If the optional keyword period is set to -1,
1833	all available caching history is used.
1834	If it is 0 only the current month is used.
1835	Future versions will include more than one month....
1836	OMN 20/8/2000
1837	"""
1838
1839	import os
1840	import os.path
1841	from string import split, rstrip, find
1842	from time import strptime, localtime, strftime, mktime, ctime
1843
1844	# sortidx = 4 # Index into Fields[1:]. What to sort by.
1845
1846	Fields = ['Name', 'Hits', 'Exec(s)', \
1847	'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1848	Widths = [25,7,9,9,9,9,13]
1849	#Types = ['s','d','d','d','d','.2f','d']
1850	Types = ['s','d','.2f','.2f','.2f','.2f','d']
1851
1852	Dictnames = ['Function', 'User']
1853
1854	if not cachedir:
1855	cachedir = checkdir(options['cachedir'])
1856
1857	SD = os.path.expanduser(cachedir) # Expand ~ or ~user in pathname
1858
1859	if period == -1: # Take all available stats
1860	SFILENAME = statsfile
1861	else: # Only stats from current month
1862	# MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1863	TimeTuple = localtime(time())
1864	extension = strftime('%b%Y',TimeTuple)
1865	SFILENAME = statsfile+'.'+extension
1866
1867	DIRLIST = os.listdir(SD)
1868	SF = []
1869	for FN in DIRLIST:
1870	if find(FN,SFILENAME) >= 0:
1871	SF.append(FN)
1872
1873	blocksize = 15000000
1874	total_read = 0
1875	total_hits = 0
1876	total_discarded = 0
1877	firstday = mktime(strptime('2030','%Y'))
1878	# FIXME: strptime don't exist in WINDOWS ?
1879	lastday = 0
1880
1881	FuncDict = {}
1882	UserDict = {}
1883	for FN in SF:
1884	input = open(SD+FN,'r')
1885	print 'Reading file ', SD+FN
1886
1887	while 1:
1888	A = input.readlines(blocksize)
1889	if len(A) == 0: break
1890	total_read = total_read + len(A)
1891	for record in A:
1892	record = tuple(split(rstrip(record),','))
1893	#print record, len(record)
1894
1895	if len(record) == 9:
1896	timestamp = record[0]
1897
1898	try:
1899	t = mktime(strptime(timestamp))
1900	except:
1901	total_discarded = total_discarded + 1
1902	continue
1903
1904	if t > lastday:
1905	lastday = t
1906	if t < firstday:
1907	firstday = t
1908
1909	user = record[1]
1910	func = record[2]
1911
1912	# Strip hash-stamp off
1913	#
1914	i = find(func,'[')
1915	func = func[:i]
1916
1917	size = float(record[3])
1918
1919	# Compression kepword can be Boolean
1920	if record[4] in ['True', '1']:
1921	compression = 1
1922	elif record[4] in ['False', '0']:
1923	compression = 0
1924	else:
1925	print 'Unknown value of compression', record[4]
1926	print record
1927	total_discarded = total_discarded + 1
1928	continue
1929
1930	#compression = int(record[4]) # Can be Boolean
1931	hit = int(record[5])
1932	reason = int(record[6]) # Not used here
1933	cputime = float(record[7])
1934	loadtime = float(record[8])
1935
1936	if hit:
1937	total_hits = total_hits + 1
1938	saving = cputime-loadtime
1939
1940	if cputime != 0:
1941	rel_saving = round(100.0*saving/cputime,2)
1942	else:
1943	#rel_saving = round(1.0*saving,2)
1944	rel_saving = 100.0 - round(1.0*saving,2) # A bit of a hack
1945
1946	info = [1,cputime,loadtime,saving,rel_saving,size]
1947
1948	UpdateDict(UserDict,user,info)
1949	UpdateDict(FuncDict,func,info)
1950	else:
1951	pass #Stats on recomputations and their reasons could go in here
1952
1953	else:
1954	#print 'Record discarded'
1955	#print record
1956	total_discarded = total_discarded + 1
1957
1958	input.close()
1959
1960	# Compute averages of all sums and write list
1961	#
1962
1963	if total_read == 0:
1964	printline(Widths,'=')
1965	print 'CACHING STATISTICS: No valid records read'
1966	printline(Widths,'=')
1967	return
1968
1969	print
1970	printline(Widths,'=')
1971	print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
1972	printline(Widths,'=')
1973	#print ' Period:', ctime(firstday), 'to', ctime(lastday)
1974	print ' Total number of valid records', total_read
1975	print ' Total number of discarded records', total_discarded
1976	print ' Total number of hits', total_hits
1977	print
1978
1979	print ' Fields', Fields[2:], 'are averaged over number of hits'
1980	print ' Time is measured in seconds and size in bytes'
1981	print ' Tables are sorted by', Fields[1:][sortidx]
1982
1983	# printline(Widths,'-')
1984
1985	if showuser:
1986	Dictionaries = [FuncDict, UserDict]
1987	else:
1988	Dictionaries = [FuncDict]
1989
1990	i = 0
1991	for Dict in Dictionaries:
1992	for key in Dict.keys():
1993	rec = Dict[key]
1994	for n in range(len(rec)):
1995	if n > 0:
1996	rec[n] = round(1.0*rec[n]/rec[0],2)
1997	Dict[key] = rec
1998
1999	# Sort and output
2000	#
2001	keylist = SortDict(Dict,sortidx)
2002
2003	# Write Header
2004	#
2005	print
2006	#print Dictnames[i], 'statistics:'; i=i+1
2007	printline(Widths,'-')
2008	n = 0
2009	for s in Fields:
2010	if s == Fields[0]: # Left justify
2011	s = Dictnames[i] + ' ' + s; i=i+1
2012	exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
2013	else:
2014	exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
2015	print
2016	printline(Widths,'-')
2017
2018	# Output Values
2019	#
2020	for key in keylist:
2021	rec = Dict[key]
2022	n = 0
2023	if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
2024	exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
2025	for val in rec:
2026	exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
2027	print
2028	print
2029
2030	#==============================================================================
2031	# Auxiliary stats functions
2032	#==============================================================================
2033
2034	def UpdateDict(Dict,key,info):
2035	"""Update dictionary by adding new values to existing.
2036
2037	USAGE:
2038	UpdateDict(Dict,key,info)
2039	"""
2040
2041	if Dict.has_key(key):
2042	dinfo = Dict[key]
2043	for n in range(len(dinfo)):
2044	dinfo[n] = info[n] + dinfo[n]
2045	else:
2046	dinfo = info[:] # Make a copy of info list
2047
2048	Dict[key] = dinfo
2049	return Dict
2050
2051	# -----------------------------------------------------------------------------
2052
2053	def SortDict(Dict,sortidx=0):
2054	"""Sort dictionary
2055
2056	USAGE:
2057	SortDict(Dict,sortidx):
2058
2059	DESCRIPTION:
2060	Sort dictionary of lists according field number 'sortidx'
2061	"""
2062
2063	import types
2064
2065	sortlist = []
2066	keylist = Dict.keys()
2067	for key in keylist:
2068	rec = Dict[key]
2069	if not type(rec) in [types.ListType, types.TupleType]:
2070	rec = [rec]
2071
2072	if sortidx > len(rec)-1:
2073	if options['verbose']:
2074	print 'ERROR: Sorting index to large, sortidx = ', sortidx
2075	raise IndexError
2076
2077	val = rec[sortidx]
2078	sortlist.append(val)
2079
2080	A = map(None,sortlist,keylist)
2081	A.sort()
2082	keylist = map(lambda x: x[1], A) # keylist sorted by sortidx
2083
2084	return(keylist)
2085
2086	# -----------------------------------------------------------------------------
2087
2088	def printline(Widths,char):
2089	"""Print textline in fixed field.
2090
2091	USAGE:
2092	printline(Widths,char)
2093	"""
2094
2095	s = ''
2096	for n in range(len(Widths)):
2097	s = s+Widths[n]*char
2098	if n > 0:
2099	s = s+char
2100
2101	print s
2102
2103	#==============================================================================
2104	# Messages
2105	#==============================================================================
2106
2107	def msg1(funcname,args,kwargs,reason):
2108	"""Message 1
2109
2110	USAGE:
2111	msg1(funcname,args,kwargs,reason):
2112	"""
2113
2114	import string
2115	#print 'MESSAGE (caching.py): Evaluating function', funcname,
2116
2117	print_header_box('Evaluating function %s' %funcname)
2118
2119	msg7(args, kwargs)
2120	msg8(reason)
2121
2122	print_footer()
2123
2124	#
2125	# Old message
2126	#
2127	#args_present = 0
2128	#if args:
2129	# if len(args) == 1:
2130	# print 'with argument', mkargstr(args[0], textwidth2),
2131	# else:
2132	# print 'with arguments', mkargstr(args, textwidth2),
2133	# args_present = 1
2134	#
2135	#if kwargs:
2136	# if args_present:
2137	# word = 'and'
2138	# else:
2139	# word = 'with'
2140	#
2141	# if len(kwargs) == 1:
2142	# print word + ' keyword argument', mkargstr(kwargs, textwidth2)
2143	# else:
2144	# print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
2145	# args_present = 1
2146	#else:
2147	# print # Newline when no keyword args present
2148	#
2149	#if not args_present:
2150	# print '', # Default if no args or kwargs present
2151
2152
2153
2154	# -----------------------------------------------------------------------------
2155
2156	def msg2(funcname,args,kwargs,comptime,reason):
2157	"""Message 2
2158
2159	USAGE:
2160	msg2(funcname,args,kwargs,comptime,reason)
2161	"""
2162
2163	import string
2164
2165	#try:
2166	# R = Reason_msg[reason]
2167	#except:
2168	# R = 'Unknown reason'
2169
2170	#print_header_box('Caching statistics (storing) - %s' %R)
2171	print_header_box('Caching statistics (storing)')
2172
2173	msg6(funcname,args,kwargs)
2174	msg8(reason)
2175
2176	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2177
2178	# -----------------------------------------------------------------------------
2179
2180	def msg3(savetime, CD, FN, deps,compression):
2181	"""Message 3
2182
2183	USAGE:
2184	msg3(savetime, CD, FN, deps,compression)
2185	"""
2186
2187	import string
2188	print string.ljust('\| Loading time:', textwidth1) + str(round(savetime,2)) + \
2189	' seconds (estimated)'
2190	msg5(CD,FN,deps,compression)
2191
2192	# -----------------------------------------------------------------------------
2193
2194	def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
2195	"""Message 4
2196
2197	USAGE:
2198	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
2199	"""
2200
2201	import string
2202
2203	print_header_box('Caching statistics (retrieving)')
2204
2205	msg6(funcname,args,kwargs)
2206	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2207	print string.ljust('\| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
2208	print string.ljust('\| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
2209	' seconds'
2210	msg5(CD,FN,deps,compression)
2211
2212	# -----------------------------------------------------------------------------
2213
2214	def msg5(CD,FN,deps,compression):
2215	"""Message 5
2216
2217	USAGE:
2218	msg5(CD,FN,deps,compression)
2219
2220	DESCRIPTION:
2221	Print dependency stats. Used by msg3 and msg4
2222	"""
2223
2224	import os, time, string
2225
2226	print '\|'
2227	print string.ljust('\| Caching dir: ', textwidth1) + CD
2228
2229	if compression:
2230	suffix = '.z'
2231	bytetext = 'bytes, compressed'
2232	else:
2233	suffix = ''
2234	bytetext = 'bytes'
2235
2236	for file_type in file_types:
2237	file_name = FN + '_' + file_type + suffix
2238	print string.ljust('\| ' + file_type + ' file: ', textwidth1) + file_name,
2239	stats = os.stat(CD+file_name)
2240	print '('+ str(stats[6]) + ' ' + bytetext + ')'
2241
2242	print '\|'
2243	if len(deps) > 0:
2244	print '\| Dependencies: '
2245	dependencies = deps.keys()
2246	dlist = []; maxd = 0
2247	tlist = []; maxt = 0
2248	slist = []; maxs = 0
2249	for d in dependencies:
2250	stats = deps[d]
2251	t = time.ctime(stats[1])
2252	s = str(stats[0])
2253	#if s[-1] == 'L':
2254	# s = s[:-1] # Strip rightmost 'long integer' L off.
2255	# # FIXME: Unnecessary in versions later than 1.5.2
2256
2257	if len(d) > maxd: maxd = len(d)
2258	if len(t) > maxt: maxt = len(t)
2259	if len(s) > maxs: maxs = len(s)
2260	dlist.append(d)
2261	tlist.append(t)
2262	slist.append(s)
2263
2264	for n in range(len(dlist)):
2265	d = string.ljust(dlist[n]+':', maxd+1)
2266	t = string.ljust(tlist[n], maxt)
2267	s = string.rjust(slist[n], maxs)
2268
2269	print '\| ', d, t, ' ', s, 'bytes'
2270	else:
2271	print '\| No dependencies'
2272	print_footer()
2273
2274	# -----------------------------------------------------------------------------
2275
2276	def msg6(funcname,args,kwargs):
2277	"""Message 6
2278
2279	USAGE:
2280	msg6(funcname,args,kwargs)
2281	"""
2282
2283	import string
2284	print string.ljust('\| Function:', textwidth1) + funcname
2285
2286	msg7(args, kwargs)
2287
2288	# -----------------------------------------------------------------------------
2289
2290	def msg7(args, kwargs):
2291	"""Message 7
2292
2293	USAGE:
2294	msg7(args, kwargs):
2295	"""
2296
2297	import string
2298
2299	args_present = 0
2300	if args:
2301	if len(args) == 1:
2302	print string.ljust('\| Argument:', textwidth1) + mkargstr(args[0], \
2303	textwidth2)
2304	else:
2305	print string.ljust('\| Arguments:', textwidth1) + \
2306	mkargstr(args, textwidth2)
2307	args_present = 1
2308
2309	if kwargs:
2310	if len(kwargs) == 1:
2311	print string.ljust('\| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
2312	textwidth2)
2313	else:
2314	print string.ljust('\| Keyword Args:', textwidth1) + \
2315	mkargstr(kwargs, textwidth2)
2316	args_present = 1
2317
2318	if not args_present:
2319	print '\| No arguments' # Default if no args or kwargs present
2320
2321	# -----------------------------------------------------------------------------
2322
2323	def msg8(reason):
2324	"""Message 8
2325
2326	USAGE:
2327	msg8(reason):
2328	"""
2329
2330	import string
2331
2332	try:
2333	R = Reason_msg[reason]
2334	except:
2335	R = 'Unknown'
2336
2337	print string.ljust('\| Reason:', textwidth1) + R
2338
2339	# -----------------------------------------------------------------------------
2340
2341	def print_header_box(line):
2342	"""Print line in a nice box.
2343
2344	USAGE:
2345	print_header_box(line)
2346
2347	"""
2348	global textwidth3
2349
2350	import time
2351
2352	time_stamp = time.ctime(time.time())
2353	line = time_stamp + '. ' + line
2354
2355	N = len(line) + 1
2356	s = '+' + '-'*N + CR
2357
2358	print s + '\| ' + line + CR + s,
2359
2360	textwidth3 = N
2361
2362	# -----------------------------------------------------------------------------
2363
2364	def print_footer():
2365	"""Print line same width as that of print_header_box.
2366	"""
2367
2368	N = textwidth3
2369	s = '+' + '-'*N + CR
2370
2371	print s
2372
2373	# -----------------------------------------------------------------------------
2374
2375	def mkargstr(args, textwidth, argstr = '', level=0):
2376	""" Generate a string containing first textwidth characters of arguments.
2377
2378	USAGE:
2379	mkargstr(args, textwidth, argstr = '', level=0)
2380
2381	DESCRIPTION:
2382	Exactly the same as str(args) possibly followed by truncation,
2383	but faster if args is huge.
2384	"""
2385
2386	import types
2387
2388	if level > 10:
2389	# Protect against circular structures
2390	return '...'
2391
2392	WasTruncated = 0
2393
2394	if not type(args) in [types.TupleType, types.ListType, types.DictType]:
2395	if type(args) == types.StringType:
2396	argstr = argstr + "'"+str(args)+"'"
2397	else:
2398	# Truncate large Numeric arrays before using str()
2399	import Numeric
2400	if type(args) == Numeric.ArrayType:
2401	# if len(args.flat) > textwidth:
2402	# Changed by Duncan and Nick 21/2/07 .flat has problems with
2403	# non-contigous arrays and ravel is equal to .flat except it
2404	# can work with non-contiguous arrays
2405	if len(Numeric.ravel(args)) > textwidth:
2406	args = 'Array: ' + str(args.shape)
2407
2408	argstr = argstr + str(args)
2409	else:
2410	if type(args) == types.DictType:
2411	argstr = argstr + "{"
2412	for key in args.keys():
2413	argstr = argstr + mkargstr(key, textwidth, level=level+1) + ": " + \
2414	mkargstr(args[key], textwidth, level=level+1) + ", "
2415	if len(argstr) > textwidth:
2416	WasTruncated = 1
2417	break
2418	argstr = argstr[:-2] # Strip off trailing comma
2419	argstr = argstr + "}"
2420
2421	else:
2422	if type(args) == types.TupleType:
2423	lc = '('
2424	rc = ')'
2425	else:
2426	lc = '['
2427	rc = ']'
2428	argstr = argstr + lc
2429	for arg in args:
2430	argstr = argstr + mkargstr(arg, textwidth, level=level+1) + ', '
2431	if len(argstr) > textwidth:
2432	WasTruncated = 1
2433	break
2434
2435	# Strip off trailing comma and space unless singleton tuple
2436	#
2437	if type(args) == types.TupleType and len(args) == 1:
2438	argstr = argstr[:-1]
2439	else:
2440	argstr = argstr[:-2]
2441	argstr = argstr + rc
2442
2443	if len(argstr) > textwidth:
2444	WasTruncated = 1
2445
2446	if WasTruncated:
2447	argstr = argstr[:textwidth]+'...'
2448	return(argstr)
2449
2450	# -----------------------------------------------------------------------------
2451
2452	def test_OK(msg):
2453	"""Print OK msg if test is OK.
2454
2455	USAGE
2456	test_OK(message)
2457	"""
2458
2459	import string
2460
2461	print string.ljust(msg, textwidth4) + ' - OK'
2462
2463	#raise StandardError
2464
2465	# -----------------------------------------------------------------------------
2466
2467	def test_error(msg):
2468	"""Print error if test fails.
2469
2470	USAGE
2471	test_error(message)
2472	"""
2473
2474	print 'ERROR (caching.test): %s' %msg
2475	print 'Please send this code example and output to '
2476	print 'Ole.Nielsen@anu.edu.au'
2477	print
2478	print
2479
2480	#import sys
2481	#sys.exit()
2482	raise StandardError
2483
2484	#-------------------------------------------------------------
2485	if __name__ == "__main__":
2486	pass

Note: See TracBrowser for help on using the repository browser.

Download in other formats: