Context Navigation

source: anuga_core/source/anuga/caching/caching.py @ 5893

Last change on this file since 5893 was 5893, checked in by rwilson, 15 years ago
Initial numpy conversion.
File size: 72.2 KB

Line
1	# =============================================================================
2	# caching.py - Supervised caching of function results.
3	# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4	# Australian National University (1999-2003)
5	# Geoscience Australia (2003-present)
6	#
7	# This program is free software; you can redistribute it and/or modify
8	# it under the terms of the GNU General Public License as published by
9	# the Free Software Foundation; either version 2 of the License, or
10	# (at your option) any later version.
11	#
12	# This program is distributed in the hope that it will be useful,
13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	# GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16	# for more details.
17	#
18	# You should have received a copy of the GNU General Public License
19	# along with this program; if not, write to the Free Software
20	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
21	#
22	#
23	# Contact address: Ole.Nielsen@ga.gov.au
24	#
25	# Version 1.5.6 February 2002
26	# =============================================================================
27
28	"""Module caching.py - Supervised caching of function results.
29
30	Public functions:
31
32	cache(func,args) -- Cache values returned from func given args.
33	cachestat() -- Reports statistics about cache hits and time saved.
34	test() -- Conducts a basic test of the caching functionality.
35
36	See doc strings of individual functions for detailed documentation.
37	"""
38
39	# -----------------------------------------------------------------------------
40	# Initialisation code
41
42	# Determine platform
43	#
44	from os import getenv
45	import types
46
47	import os
48	if os.name in ['nt', 'dos', 'win32', 'what else?']:
49	unix = 0
50	else:
51	unix = 1
52
53	import numpy
54
55
56	cache_dir = '.python_cache'
57	# Make default caching directory name
58	#
59	if unix:
60	homedir = getenv('INUNDATIONHOME')
61	if homedir is None:
62	homedir = '~'
63	else:
64	homedir = homedir + os.sep + '.cache'
65	# Since homedir will be a group area, individually label the caches
66	user = getenv('LOGNAME')
67	if user is not None:
68	cache_dir += '_' + user
69
70	CR = '\n'
71	else:
72	homedir = 'c:'
73	CR = '\r\n' #FIXME: Not tested under windows
74
75	cachedir = homedir + os.sep + cache_dir + os.sep
76
77	# -----------------------------------------------------------------------------
78	# Options directory with default values - to be set by user
79	#
80
81	options = {
82	'cachedir': cachedir, # Default cache directory
83	'maxfiles': 1000000, # Maximum number of cached files
84	'savestat': 1, # Log caching info to stats file
85	'verbose': 1, # Write messages to standard output
86	'bin': 1, # Use binary format (more efficient)
87	'compression': 1, # Use zlib compression
88	'bytecode': 0, # Recompute if bytecode has changed
89	'expire': 0 # Automatically remove files that have been accessed
90	# least recently
91	}
92
93	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
94
95	def set_option(key, value):
96	"""Function to set values in the options directory.
97
98	USAGE:
99	set_option(key, value)
100
101	ARGUMENTS:
102	key -- Key in options dictionary. (Required)
103	value -- New value for key. (Required)
104
105	DESCRIPTION:
106	Function to set values in the options directory.
107	Raises an exception if key is not in options.
108	"""
109
110	if options.has_key(key):
111	options[key] = value
112	else:
113	raise KeyError(key) # Key not found, raise an exception
114
115	# -----------------------------------------------------------------------------
116	# Function cache - the main routine
117
118	def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
119	verbose=None, compression=None, evaluate=0, test=0, clear=0,
120	return_filename=0):
121	"""Supervised caching of function results.
122
123	USAGE:
124	result = cache(func, args, kwargs, dependencies, cachedir, verbose,
125	compression, evaluate, test, return_filename)
126
127	ARGUMENTS:
128	func -- Function object (Required)
129	args -- Arguments to func (Default: ())
130	kwargs -- Keyword arguments to func (Default: {})
131	dependencies -- Filenames that func depends on (Default: None)
132	cachedir -- Directory for cache files (Default: options['cachedir'])
133	verbose -- Flag verbose output to stdout
134	(Default: options['verbose'])
135	compression -- Flag zlib compression (Default: options['compression'])
136	evaluate -- Flag forced evaluation of func (Default: 0)
137	test -- Flag test for cached results (Default: 0)
138	clear -- Flag delete cached results (Default: 0)
139	return_filename -- Flag return of cache filename (Default: 0)
140
141	DESCRIPTION:
142	A Python function call of the form
143
144	result = func(arg1,...,argn)
145
146	can be replaced by
147
148	from caching import cache
149	result = cache(func,(arg1,...,argn))
150
151	The latter form returns the same output as the former but reuses cached
152	results if the function has been computed previously in the same context.
153	'result' and the arguments can be simple types, tuples, list, dictionaries or
154	objects, but not unhashable types such as functions or open file objects.
155	The function 'func' may be a member function of an object or a module.
156
157	This type of caching is particularly useful for computationally intensive
158	functions with few frequently used combinations of input arguments. Note that
159	if the inputs or output are very large caching might not save time because
160	disc access may dominate the execution time.
161
162	If the function definition changes after a result has been cached it will be
163	detected by examining the functions bytecode (co_code, co_consts,
164	func_defualts, co_argcount) and it will be recomputed.
165
166	LIMITATIONS:
167	1 Caching uses the apply function and will work with anything that can be
168	pickled, so any limitation in apply or pickle extends to caching.
169	2 A function to be cached should not depend on global variables
170	as wrong results may occur if globals are changed after a result has
171	been cached.
172
173	-----------------------------------------------------------------------------
174	Additional functionality:
175
176	Keyword args
177	Keyword arguments (kwargs) can be added as a dictionary of keyword: value
178	pairs, following the syntax of the built-in function apply().
179	A Python function call of the form
180
181	result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)
182
183	is then cached as follows
184
185	from caching import cache
186	result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
187
188	The default value of kwargs is {}
189
190	Explicit dependencies:
191	The call
192	cache(func,(arg1,...,argn),dependencies = <list of filenames>)
193	Checks the size, creation time and modification time of each listed file.
194	If any file has changed the function is recomputed and the results stored
195	again.
196
197	Specify caching directory:
198	The call
199	cache(func,(arg1,...,argn), cachedir = <cachedir>)
200	designates <cachedir> where cached data are stored. Use ~ to indicate users
201	home directory - not $HOME. The default is ~/.python_cache on a UNIX
202	platform and c:/.python_cache on a Win platform.
203
204	Silent operation:
205	The call
206	cache(func,(arg1,...,argn),verbose=0)
207	suppresses messages to standard output.
208
209	Compression:
210	The call
211	cache(func,(arg1,...,argn),compression=0)
212	disables compression. (Default: compression=1). If the requested compressed
213	or uncompressed file is not there, it'll try the other version.
214
215	Forced evaluation:
216	The call
217	cache(func,(arg1,...,argn),evaluate=1)
218	forces the function to evaluate even though cached data may exist.
219
220	Testing for presence of cached result:
221	The call
222	cache(func,(arg1,...,argn),test=1)
223	retrieves cached result if it exists, otherwise None. The function will not
224	be evaluated. If both evaluate and test are switched on, evaluate takes
225	precedence.
226	??NOTE: In case of hash collisions, this may return the wrong result as
227	??it only checks if a cached result is present.
228
229	Obtain cache filenames:
230	The call
231	cache(func,(arg1,...,argn),return_filename=1)
232	returns the hashed base filename under which this function and its
233	arguments would be cached
234
235	Clearing cached results:
236	The call
237	cache(func,'clear')
238	clears all cached data for 'func' and
239	cache('clear')
240	clears all cached data.
241
242	NOTE: The string 'clear' can be passed an argument to func using
243	cache(func,('clear',)) or cache(func,tuple(['clear'])).
244
245	New form of clear:
246	cache(func,(arg1,...,argn),clear=1)
247	clears cached data for particular combination func and args
248
249	"""
250
251	# Imports and input checks
252	#
253	import types, time, string
254
255	if not cachedir:
256	cachedir = options['cachedir']
257
258	if verbose == None: # Do NOT write 'if not verbose:', it could be zero.
259	verbose = options['verbose']
260
261	if compression == None: # Do NOT write 'if not compression:',
262	# it could be zero.
263	compression = options['compression']
264
265	# Create cache directory if needed
266	CD = checkdir(cachedir,verbose)
267
268	# Handle the case cache('clear')
269	if type(func) == types.StringType:
270	if string.lower(func) == 'clear':
271	clear_cache(CD,verbose=verbose)
272	return
273
274	# Handle the case cache(func, 'clear')
275	if type(args) == types.StringType:
276	if string.lower(args) == 'clear':
277	clear_cache(CD,func,verbose=verbose)
278	return
279
280	# Force singleton arg into a tuple
281	if type(args) != types.TupleType:
282	args = tuple([args])
283
284	# Check that kwargs is a dictionary
285	if type(kwargs) != types.DictType:
286	raise TypeError
287
288	# Hash arguments (and keyword args) to integer
289	arghash = myhash((args, kwargs))
290
291	# Get sizes and timestamps for files listed in dependencies.
292	# Force singletons into a tuple.
293	if dependencies and type(dependencies) != types.TupleType \
294	and type(dependencies) != types.ListType:
295	dependencies = tuple([dependencies])
296	deps = get_depstats(dependencies)
297
298	# Extract function name from func object
299	funcname = get_funcname(func)
300
301	# Create cache filename
302	FN = funcname+'['+`arghash`+']' # The symbol '(' does not work under unix
303
304	if return_filename:
305	return(FN)
306
307	if clear:
308	for file_type in file_types:
309	file_name = CD+FN+'_'+file_type
310	for fn in [file_name, file_name + '.z']:
311	if os.access(fn, os.F_OK):
312	if unix:
313	os.remove(fn)
314	else:
315	# FIXME: os.remove doesn't work under windows
316	os.system('del '+fn)
317	if verbose is True:
318	print 'MESSAGE (caching): File %s deleted' %fn
319	##else:
320	## print '%s was not accessed' %fn
321	return None
322
323
324	#-------------------------------------------------------------------
325
326	# Check if previous computation has been cached
327	if evaluate:
328	Retrieved = None # Force evaluation of func regardless of caching status.
329	reason = 5
330	else:
331	T, FN, Retrieved, reason, comptime, loadtime, compressed = \
332	CacheLookup(CD, FN, func,
333	args, kwargs,
334	deps,
335	verbose,
336	compression,
337	dependencies)
338
339	if not Retrieved:
340	if test: # Do not attempt to evaluate function
341	T = None
342	else: # Evaluate function and save to cache
343	if verbose:
344
345	msg1(funcname, args, kwargs,reason)
346
347	# Remove expired files automatically
348	if options['expire']:
349	DeleteOldFiles(CD,verbose)
350
351	# Save args before function is evaluated in case
352	# they are modified by function
353	save_args_to_cache(CD,FN,args,kwargs,compression)
354
355	# Execute and time function with supplied arguments
356	t0 = time.time()
357	T = apply(func,args,kwargs)
358	#comptime = round(time.time()-t0)
359	comptime = time.time()-t0
360
361	if verbose:
362	msg2(funcname,args,kwargs,comptime,reason)
363
364	# Save results and estimated loading time to cache
365	loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
366	funcname, dependencies, compression)
367	if verbose:
368	msg3(loadtime, CD, FN, deps, compression)
369	compressed = compression
370
371	if options['savestat'] and (not test or Retrieved):
372	##if options['savestat']:
373	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
374
375	return(T) # Return results in all cases
376
377	# -----------------------------------------------------------------------------
378
379	def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
380	"""Generate statistics of caching efficiency.
381
382	USAGE:
383	cachestat(sortidx, period, showuser, cachedir)
384
385	ARGUMENTS:
386	sortidx -- Index of field by which lists are (default: 4)
387	Legal values are
388	0: 'Name'
389	1: 'Hits'
390	2: 'CPU'
391	3: 'Time Saved'
392	4: 'Gain(%)'
393	5: 'Size'
394	period -- If set to -1 all available caching history is used.
395	If set 0 only the current month is used (default -1).
396	showuser -- Flag for additional table showing user statistics
397	(default: None).
398	cachedir -- Directory for cache files (default: options['cachedir']).
399
400	DESCRIPTION:
401	Logged caching statistics is converted into summaries of the form
402	--------------------------------------------------------------------------
403	Function Name Hits Exec(s) Cache(s) Saved(s) Gain(%) Size
404	--------------------------------------------------------------------------
405	"""
406
407	__cachestat(sortidx, period, showuser, cachedir)
408	return
409
410	# -----------------------------------------------------------------------------
411
412	#Has mostly been moved to proper unit test
413	def test(cachedir=None,verbose=0,compression=None):
414	"""Test the functionality of caching.
415
416	USAGE:
417	test(verbose)
418
419	ARGUMENTS:
420	verbose -- Flag whether caching will output its statistics (default=0)
421	cachedir -- Directory for cache files (Default: options['cachedir'])
422	compression -- Flag zlib compression (Default: options['compression'])
423	"""
424
425	import string, time
426
427	# Initialise
428	#
429	import caching
430	reload(caching)
431
432	if not cachedir:
433	cachedir = options['cachedir']
434
435	if verbose is None: # Do NOT write 'if not verbose:', it could be zero.
436	verbose = options['verbose']
437
438	if compression == None: # Do NOT write 'if not compression:',
439	# it could be zero.
440	compression = options['compression']
441	else:
442	try:
443	set_option('compression', compression)
444	except:
445	test_error('Set option failed')
446
447	try:
448	import zlib
449	except:
450	print
451	print '* Could not find zlib, default to no-compression *'
452	print '* Installing zlib will improve performance of caching *'
453	print
454	compression = 0
455	set_option('compression', compression)
456
457	print
458	print_header_box('Testing caching module - please stand by')
459	print
460
461	# Define a test function to be cached
462	#
463	def f(a,b,c,N,x=0,y='abcdefg'):
464	"""f(a,b,c,N)
465	Do something time consuming and produce a complex result.
466	"""
467
468	import string
469
470	B = []
471	for n in range(N):
472	s = str(n+2.0/(n + 4.0))+'.a'*10
473	B.append((a,b,c,s,n,x,y))
474	return(B)
475
476	# Check that default cachedir is OK
477	#
478	CD = checkdir(cachedir,verbose)
479
480
481	# Make a dependency file
482	#
483	try:
484	DepFN = CD + 'testfile.tmp'
485	DepFN_wildcard = CD + 'test*.tmp'
486	Depfile = open(DepFN,'w')
487	Depfile.write('We are the knights who say NI!')
488	Depfile.close()
489	test_OK('Wrote file %s' %DepFN)
490	except:
491	test_error('Could not open file %s for writing - check your environment' \
492	% DepFN)
493
494	# Check set_option (and switch stats off
495	#
496	try:
497	set_option('savestat',0)
498	assert(options['savestat'] == 0)
499	test_OK('Set option')
500	except:
501	test_error('Set option failed')
502
503	# Make some test input arguments
504	#
505	N = 5000 #Make N fairly small here
506
507	a = [1,2]
508	b = ('Thou shalt count the number three',4)
509	c = {'Five is right out': 6, (7,8): 9}
510	x = 3
511	y = 'holy hand granate'
512
513	# Test caching
514	#
515	if compression:
516	comprange = 2
517	else:
518	comprange = 1
519
520	for comp in range(comprange):
521
522	# Evaluate and store
523	#
524	try:
525	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
526	verbose=verbose, compression=comp)
527	if comp:
528	test_OK('Caching evaluation with compression')
529	else:
530	test_OK('Caching evaluation without compression')
531	except:
532	if comp:
533	test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
534	else:
535	test_error('Caching evaluation failed - try caching.test(verbose=1)')
536
537	# Retrieve
538	#
539	try:
540	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
541	compression=comp)
542
543	if comp:
544	test_OK('Caching retrieval with compression')
545	else:
546	test_OK('Caching retrieval without compression')
547	except:
548	if comp:
549	test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
550	else:
551	test_error('Caching retrieval failed - try caching.test(verbose=1)')
552
553	# Reference result
554	#
555	T3 = f(a,b,c,N,x=x,y=y) # Compute without caching
556
557	if T1 == T2 and T2 == T3:
558	if comp:
559	test_OK('Basic caching functionality (with compression)')
560	else:
561	test_OK('Basic caching functionality (without compression)')
562	else:
563	test_error('Cached result does not match computed result')
564
565
566	# Test return_filename
567	#
568	try:
569	FN = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
570	return_filename=1)
571	assert(FN[:2] == 'f[')
572	test_OK('Return of cache filename')
573	except:
574	test_error('Return of cache filename failed')
575
576	# Test existence of cachefiles
577	#
578	try:
579	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
580	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
581	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
582	test_OK('Presence of cache files')
583	datafile.close()
584	argsfile.close()
585	admfile.close()
586	except:
587	test_error('Expected cache files did not exist')
588
589	# Test 'test' function when cache is present
590	#
591	try:
592	#T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
593	# evaluate=1)
594	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
595	assert(T1 == T4)
596
597	test_OK("Option 'test' when cache file present")
598	except:
599	test_error("Option 'test' when cache file present failed")
600
601	# Test that 'clear' works
602	#
603	#try:
604	# caching.cache(f,'clear',verbose=verbose)
605	# test_OK('Clearing of cache files')
606	#except:
607	# test_error('Clear does not work')
608	try:
609	caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)
610	test_OK('Clearing of cache files')
611	except:
612	test_error('Clear does not work')
613
614
615
616	# Test 'test' function when cache is absent
617	#
618	try:
619	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
620	assert(T4 is None)
621	test_OK("Option 'test' when cache absent")
622	except:
623	test_error("Option 'test' when cache absent failed")
624
625	# Test dependencies
626	#
627	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
628	dependencies=DepFN)
629	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
630	dependencies=DepFN)
631
632	if T1 == T2:
633	test_OK('Basic dependencies functionality')
634	else:
635	test_error('Dependencies do not work')
636
637	# Test basic wildcard dependency
638	#
639	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
640	dependencies=DepFN_wildcard)
641
642	if T1 == T3:
643	test_OK('Basic dependencies with wildcard functionality')
644	else:
645	test_error('Dependencies with wildcards do not work')
646
647
648	# Test that changed timestamp in dependencies triggers recomputation
649
650	# Modify dependency file
651	Depfile = open(DepFN,'a')
652	Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
653	Depfile.close()
654
655	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
656	dependencies=DepFN, test = 1)
657
658	if T3 is None:
659	test_OK('Changed dependencies recognised')
660	else:
661	test_error('Changed dependencies not recognised')
662
663	# Test recomputation when dependencies have changed
664	#
665	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
666	dependencies=DepFN)
667	if T1 == T3:
668	test_OK('Recomputed value with changed dependencies')
669	else:
670	test_error('Recomputed value with changed dependencies failed')
671
672	# Performance test (with statistics)
673	# Don't really rely on this as it will depend on specific computer.
674	#
675
676	set_option('savestat',1)
677
678	N = 20*N #Should be large on fast computers...
679	tt = time.time()
680	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
681	t1 = time.time() - tt
682
683	tt = time.time()
684	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
685	t2 = time.time() - tt
686
687	if T1 == T2:
688	if t1 > t2:
689	test_OK('Performance test: relative time saved = %s pct' \
690	%str(round((t1-t2)*100/t1,2)))
691	#else:
692	# print 'WARNING: Performance a bit low - this could be specific to current platform'
693	else:
694	test_error('Basic caching failed for new problem')
695
696	# Test presence of statistics file
697	#
698	try:
699	DIRLIST = os.listdir(CD)
700	SF = []
701	for FN in DIRLIST:
702	if string.find(FN,statsfile) >= 0:
703	fid = open(CD+FN,'r')
704	fid.close()
705	test_OK('Statistics files present')
706	except:
707	test_OK('Statistics files cannot be opened')
708
709	print_header_box('Show sample output of the caching function:')
710
711	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
712	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
713	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
714
715	print_header_box('Show sample output of cachestat():')
716	if unix:
717	cachestat()
718	else:
719	try:
720	import time
721	t = time.strptime('2030','%Y')
722	cachestat()
723	except:
724	print 'caching.cachestat() does not work here, because it'
725	print 'relies on time.strptime() which is unavailable in Windows'
726
727	print
728	test_OK('Caching self test completed')
729
730
731	# Test setoption (not yet implemented)
732	#
733
734
735	#==============================================================================
736	# Auxiliary functions
737	#==============================================================================
738
739	# Import pickler
740	# cPickle is used by functions mysave, myload, and compare
741	#
742	import cPickle # 10 to 100 times faster than pickle
743	pickler = cPickle
744
745	# Local immutable constants
746	#
747	comp_level = 1 # Compression level for zlib.
748	# comp_level = 1 works well.
749	textwidth1 = 16 # Text width of key fields in report forms.
750	#textwidth2 = 132 # Maximal width of textual representation of
751	textwidth2 = 300 # Maximal width of textual representation of
752	# arguments.
753	textwidth3 = 16 # Initial width of separation lines. Is modified.
754	textwidth4 = 50 # Text width in test_OK()
755	statsfile = '.cache_stat' # Basefilename for cached statistics.
756	# It will reside in the chosen cache directory.
757
758	file_types = ['Result', # File name extension for cached function results.
759	'Args', # File name extension for stored function args.
760	'Admin'] # File name extension for administrative info.
761
762	Reason_msg = ['OK', # Verbose reasons for recomputation
763	'No cached result',
764	'Dependencies have changed',
765	'Arguments have changed',
766	'Bytecode has changed',
767	'Recomputation was requested by caller',
768	'Cached file was unreadable']
769
770	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
771
772	def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression,
773	dependencies):
774	"""Determine whether cached result exists and return info.
775
776	USAGE:
777	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
778	CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
779	dependencies)
780
781	INPUT ARGUMENTS:
782	CD -- Cache Directory
783	FN -- Suggested cache file name
784	func -- Function object
785	args -- Tuple of arguments
786	kwargs -- Dictionary of keyword arguments
787	deps -- Dependencies time stamps
788	verbose -- Flag text output
789	compression -- Flag zlib compression
790	dependencies -- Given list of dependencies
791
792	OUTPUT ARGUMENTS:
793	T -- Cached result if present otherwise None
794	FN -- File name under which new results must be saved
795	Retrieved -- True if a valid cached result was found
796	reason -- 0: OK (if Retrieved),
797	1: No cached result,
798	2: Dependencies have changed,
799	3: Arguments have changed
800	4: Bytecode has changed
801	5: Recomputation was forced
802	6: Unreadable file
803	comptime -- Number of seconds it took to computed cachged result
804	loadtime -- Number of seconds it took to load cached result
805	compressed -- Flag (0,1) if cached results were compressed or not
806
807	DESCRIPTION:
808	Determine if cached result exists as follows:
809	Load in saved arguments and bytecode stored under hashed filename.
810	If they are identical to current arguments and bytecode and if dependencies
811	have not changed their time stamp, then return cached result.
812
813	Otherwise return filename under which new results should be cached.
814	Hash collisions are handled recursively by calling CacheLookup again with a
815	modified filename.
816	"""
817
818	import time, string, types
819
820	# Assess whether cached result exists - compressed or not.
821	#
822	if verbose:
823	print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
824	%(CD+FN, file_types[0], file_types[1], file_types[2])
825	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
826	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
827	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
828
829	if verbose is True and deps is not None:
830	print 'Caching: Dependencies are', deps.keys()
831
832	if not (argsfile and datafile and admfile) or \
833	not (compressed0 == compressed1 and compressed0 == compressed2):
834	# Cached result does not exist or files were compressed differently
835	#
836	# This will ensure that evaluation will take place unless all files are
837	# present.
838
839	reason = 1
840	return(None,FN,None,reason,None,None,None) #Recompute using same filename
841
842	compressed = compressed0 # Remember if compressed files were actually used
843	datafile.close()
844
845	# Retrieve arguments and adm. info
846	#
847	R, reason = myload(argsfile, compressed) # The original arguments
848	argsfile.close()
849
850	if reason > 0:
851	# Recompute using same filename
852	return(None, FN, None, reason, None, None, None)
853	else:
854	(argsref, kwargsref) = R
855
856	R, reason = myload(admfile, compressed)
857	admfile.close()
858
859	if reason > 0:
860	return(None,FN,None,reason,None,None,None) #Recompute using same filename
861
862
863	depsref = R[0] # Dependency statistics
864	comptime = R[1] # The computation time
865	coderef = R[2] # The byte code
866	funcname = R[3] # The function name
867
868	# Check if dependencies have changed
869	#
870	if dependencies and not compare(depsref, deps):
871	if verbose:
872	print 'MESSAGE (caching.py): Dependencies', dependencies, \
873	'have changed - recomputing'
874	# Don't use cached file - recompute
875	reason = 2
876	return(None, FN, None, reason, None, None, None)
877
878	# Get bytecode from func
879	#
880	bytecode = get_bytecode(func)
881
882	#print compare(argsref,args),
883	#print compare(kwargsref,kwargs),
884	#print compare(bytecode,coderef)
885
886	# Check if arguments or bytecode have changed
887	if compare(argsref, args) and compare(kwargsref, kwargs) and \
888	(not options['bytecode'] or compare(bytecode, coderef)):
889
890	# Arguments and dependencies match. Get cached results
891	T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
892	if reason > 0:
893	# Recompute using same FN
894	return(None, FN, None, reason, None, None, None)
895
896	Retrieved = 1
897	reason = 0
898
899	if verbose:
900	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
901
902	if loadtime >= comptime:
903	print 'WARNING (caching.py): Caching did not yield any gain.'
904	print ' Consider executing function ',
905	print '('+funcname+') without caching.'
906	else:
907
908	# Non matching arguments or bytecodes signify a hash-collision.
909	# This is resolved by recursive search of cache filenames
910	# until either a matching or an unused filename is found.
911	#
912	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
913	CacheLookup(CD, FN+'x', func, args, kwargs, deps,
914	verbose, compression, dependencies)
915
916	# DEBUGGING
917	# if not Retrieved:
918	# print 'Arguments did not match'
919	# else:
920	# print 'Match found !'
921
922	# The real reason is that args or bytecodes have changed.
923	# Not that the recursive seach has found an unused filename
924	if not Retrieved:
925	if not compare(bytecode, coderef):
926	reason = 4 # Bytecode has changed
927	else:
928	reason = 3 # Arguments have changed
929
930
931	return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
932
933	# -----------------------------------------------------------------------------
934
935	def clear_cache(CD,func=None, verbose=None):
936	"""Clear cache for func.
937
938	USAGE:
939	clear(CD, func, verbose)
940
941	ARGUMENTS:
942	CD -- Caching directory (required)
943	func -- Function object (default: None)
944	verbose -- Flag verbose output (default: None)
945
946	DESCRIPTION:
947
948	If func == None, clear everything,
949	otherwise clear only files pertaining to func.
950	"""
951
952	import os, re
953
954	if CD[-1] != os.sep:
955	CD = CD+os.sep
956
957	if verbose == None:
958	verbose = options['verbose']
959
960	# FIXME: Windows version needs to be tested
961
962	if func:
963	funcname = get_funcname(func)
964	if verbose:
965	print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
966
967	file_names = os.listdir(CD)
968	for file_name in file_names:
969	#RE = re.search('^' + funcname,file_name) #Inefficient
970	#if RE:
971	if file_name[:len(funcname)] == funcname:
972	if unix:
973	os.remove(CD+file_name)
974	else:
975	os.system('del '+CD+file_name)
976	# FIXME: os.remove doesn't work under windows
977	else:
978	file_names = os.listdir(CD)
979	if len(file_names) > 0:
980	if verbose:
981	print 'MESSAGE (caching.py): Remove the following files:'
982	for file_name in file_names:
983	print file_name
984
985	A = raw_input('Delete (Y/N)[N] ?')
986	else:
987	A = 'Y'
988
989	if A == 'Y' or A == 'y':
990	for file_name in file_names:
991	if unix:
992	os.remove(CD+file_name)
993	else:
994	os.system('del '+CD+file_name)
995	# FIXME: os.remove doesn't work under windows
996	#exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
997
998	# -----------------------------------------------------------------------------
999
1000	def DeleteOldFiles(CD,verbose=None):
1001	"""Remove expired files
1002
1003	USAGE:
1004	DeleteOldFiles(CD,verbose=None)
1005	"""
1006
1007	if verbose == None:
1008	verbose = options['verbose']
1009
1010	maxfiles = options['maxfiles']
1011
1012	# FIXME: Windows version
1013
1014	import os
1015	block = 1000 # How many files to delete per invokation
1016	Files = os.listdir(CD)
1017	numfiles = len(Files)
1018	if not unix: return # FIXME: Windows case ?
1019
1020	if numfiles > maxfiles:
1021	delfiles = numfiles-maxfiles+block
1022	if verbose:
1023	print 'Deleting '+`delfiles`+' expired files:'
1024	os.system('ls -lur '+CD+'* \| head -' + `delfiles`) # List them
1025	os.system('ls -ur '+CD+'* \| head -' + `delfiles` + ' \| xargs /bin/rm')
1026	# Delete them
1027	# FIXME: Replace this with os.listdir and os.remove
1028
1029	# -----------------------------------------------------------------------------
1030
1031	def save_args_to_cache(CD, FN, args, kwargs, compression):
1032	"""Save arguments to cache
1033
1034	USAGE:
1035	save_args_to_cache(CD,FN,args,kwargs,compression)
1036	"""
1037
1038	import time, os, sys, types
1039
1040	(argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1041
1042	if argsfile is None:
1043	msg = 'ERROR (caching): Could not open argsfile for writing: %s' %FN
1044	raise IOError, msg
1045
1046	mysave((args,kwargs),argsfile,compression) # Save args and kwargs to cache
1047	argsfile.close()
1048
1049	# Change access rights if possible
1050	#
1051	#if unix:
1052	# try:
1053	# exitcode=os.system('chmod 666 '+argsfile.name)
1054	# except:
1055	# pass
1056	#else:
1057	# pass # FIXME: Take care of access rights under Windows
1058
1059	return
1060
1061	# -----------------------------------------------------------------------------
1062
1063	def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1064	dependencies, compression):
1065	"""Save computed results T and admin info to cache
1066
1067	USAGE:
1068	save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1069	dependencies, compression)
1070	"""
1071
1072	import time, os, sys, types
1073
1074	(datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1075	(admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1076
1077	if not datafile:
1078	if verbose:
1079	print 'ERROR (caching): Could not open %s' %datafile.name
1080	raise IOError
1081
1082	if not admfile:
1083	if verbose:
1084	print 'ERROR (caching): Could not open %s' %admfile.name
1085	raise IOError
1086
1087	t0 = time.time()
1088
1089	mysave(T,datafile,compression) # Save data to cache
1090	datafile.close()
1091	#savetime = round(time.time()-t0,2)
1092	savetime = time.time()-t0
1093
1094	bytecode = get_bytecode(func) # Get bytecode from function object
1095	admtup = (deps, comptime, bytecode, funcname) # Gather admin info
1096
1097	mysave(admtup,admfile,compression) # Save admin info to cache
1098	admfile.close()
1099
1100	# Change access rights if possible
1101	#
1102	#if unix:
1103	# try:
1104	# exitcode=os.system('chmod 666 '+datafile.name)
1105	# exitcode=os.system('chmod 666 '+admfile.name)
1106	# except:
1107	# pass
1108	#else:
1109	# pass # FIXME: Take care of access rights under Windows
1110
1111	return(savetime)
1112
1113	# -----------------------------------------------------------------------------
1114
1115	def load_from_cache(CD,FN,compression):
1116	"""Load previously cached data from file FN
1117
1118	USAGE:
1119	load_from_cache(CD,FN,compression)
1120	"""
1121
1122	import time
1123
1124	(datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1125	t0 = time.time()
1126	T, reason = myload(datafile,compressed)
1127
1128	loadtime = time.time()-t0
1129	datafile.close()
1130
1131	return T, loadtime, compressed, reason
1132
1133	# -----------------------------------------------------------------------------
1134
1135	def myopen(FN,mode,compression=1):
1136	"""Open file FN using given mode
1137
1138	USAGE:
1139	myopen(FN,mode,compression=1)
1140
1141	ARGUMENTS:
1142	FN -- File name to be opened
1143	mode -- Open mode (as in open)
1144	compression -- Flag zlib compression
1145
1146	DESCRIPTION:
1147	if compression
1148	Attempt first to open FN + '.z'
1149	If this fails try to open FN
1150	else do the opposite
1151	Return file handle plus info about whether it was compressed or not.
1152	"""
1153
1154	import string
1155
1156	# Determine if file exists already (if writing was requested)
1157	# This info is only used to determine if access modes should be set
1158	#
1159	if 'w' in mode or 'a' in mode:
1160	try:
1161	file = open(FN+'.z','r')
1162	file.close()
1163	new_file = 0
1164	except:
1165	try:
1166	file = open(FN,'r')
1167	file.close()
1168	new_file = 0
1169	except:
1170	new_file = 1
1171	else:
1172	new_file = 0 #Assume it exists if mode was not 'w'
1173
1174
1175	compressed = 0
1176	if compression:
1177	try:
1178	file = open(FN+'.z',mode)
1179	compressed = 1
1180	except:
1181	try:
1182	file = open(FN,mode)
1183	except:
1184	file = None
1185	else:
1186	try:
1187	file = open(FN,mode)
1188	except:
1189	try:
1190	file = open(FN+'.z',mode)
1191	compressed = 1
1192	except:
1193	file = None
1194
1195	# Now set access rights if it is a new file
1196	#
1197	if file and new_file:
1198	if unix:
1199	exitcode=os.system('chmod 666 '+file.name)
1200	else:
1201	pass # FIXME: Take care of access rights under Windows
1202
1203	return(file,compressed)
1204
1205	# -----------------------------------------------------------------------------
1206
1207	def myload(file, compressed):
1208	"""Load data from file
1209
1210	USAGE:
1211	myload(file, compressed)
1212	"""
1213
1214	reason = 0
1215	try:
1216	if compressed:
1217	import zlib
1218
1219	RsC = file.read()
1220	try:
1221	Rs = zlib.decompress(RsC)
1222	except:
1223	# File "./caching.py", line 1032, in load_from_cache
1224	# T = myload(datafile,compressed)
1225	# File "./caching.py", line 1124, in myload
1226	# Rs = zlib.decompress(RsC)
1227	# zlib.error: Error -5 while decompressing data
1228	#print 'ERROR (caching): Could not decompress ', file.name
1229	#raise Exception
1230	reason = 6 # Unreadable file
1231	return None, reason
1232
1233
1234	del RsC # Free up some space
1235	R = pickler.loads(Rs)
1236	else:
1237	try:
1238	R = pickler.load(file)
1239	#except EOFError, e:
1240	except:
1241	#Catch e.g., file with 0 length or corrupted
1242	reason = 6 # Unreadable file
1243	return None, reason
1244
1245	except MemoryError:
1246	import sys
1247	if options['verbose']:
1248	print 'ERROR (caching): Out of memory while loading %s, aborting' \
1249	%(file.name)
1250
1251	# Raise the error again for now
1252	#
1253	raise MemoryError
1254
1255	return R, reason
1256
1257	# -----------------------------------------------------------------------------
1258
1259	def mysave(T,file,compression):
1260	"""Save data T to file
1261
1262	USAGE:
1263	mysave(T,file,compression)
1264
1265	"""
1266
1267	bin = options['bin']
1268
1269	if compression:
1270	try:
1271	import zlib
1272	except:
1273	print
1274	print '* Could not find zlib *'
1275	print '* Try to run caching with compression off *'
1276	print "* caching.set_option('compression', 0) *"
1277	raise Exception
1278
1279
1280	try:
1281	Ts = pickler.dumps(T, bin)
1282	except MemoryError:
1283	msg = '****WARNING (caching.py): Could not pickle data for compression.'
1284	msg += ' Try using compression = False'
1285	raise MemoryError, msg
1286	else:
1287	# Compressed pickling
1288	TsC = zlib.compress(Ts, comp_level)
1289	file.write(TsC)
1290	else:
1291	# Uncompressed pickling
1292	pickler.dump(T, file, bin)
1293
1294	# FIXME: This may not work on Windoze network drives.
1295	# The error msg is IOError: [Errno 22] Invalid argument
1296	# Testing with small files was OK, though.
1297	# I think this is an OS problem.
1298
1299	# Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1300
1301	# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1302	#
1303	#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1304	#
1305	#The following extract is taken from the site http://www.python.org:
1306	#
1307	#---------------------------------------------------------------------------------------------
1308	#exception IOError
1309	#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1310	#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1311	#---------------------------------------------------------------------------------------------
1312	#
1313	#The error code(s) that accompany exceptions are described at:
1314	#http://www.python.org/dev/doc/devel//lib/module-errno.html
1315	#
1316	#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1317	#
1318	#exceptions.IOError invalid argument Errno 22
1319
1320	#try:
1321	# pickler.dump(T,file,bin)
1322	#except IOError, e:
1323	# print e
1324	# msg = 'Could not store to %s, bin=%s' %(file, bin)
1325	# raise msg
1326
1327
1328	# -----------------------------------------------------------------------------
1329
1330
1331	def myhash(T, ids=None):
1332	"""Compute hashed integer from a range of inputs.
1333	If T is not hashable being e.g. a tuple T, myhash will recursively
1334	hash the values individually
1335
1336	USAGE:
1337	myhash(T)
1338
1339	ARGUMENTS:
1340	T -- Anything
1341	"""
1342
1343	from types import TupleType, ListType, DictType, InstanceType
1344	## from numpy.oldnumeric import ArrayType, average
1345
1346
1347	if type(T) in [TupleType, ListType, DictType, InstanceType]:
1348	# Keep track of unique id's to protect against infinite recursion
1349	if ids is None: ids = []
1350
1351	# Check if T has already been encountered
1352	i = id(T)
1353
1354	if i in ids:
1355	return 0 # T has been hashed already
1356	else:
1357	ids.append(i)
1358
1359
1360
1361	# Start hashing
1362
1363	# On some architectures None, False and True gets different hash values
1364	if T is None:
1365	return(-1)
1366	if T is False:
1367	return(0)
1368	if T is True:
1369	return(1)
1370
1371	# Get hash values for hashable entries
1372	if type(T) in [TupleType, ListType]:
1373	hvals = []
1374	for t in T:
1375	h = myhash(t, ids)
1376	hvals.append(h)
1377	val = hash(tuple(hvals))
1378	elif type(T) == DictType:
1379	# Make dictionary ordering unique
1380	I = T.items()
1381	I.sort()
1382	val = myhash(I, ids)
1383	elif isinstance(T, numpy.ndarray):
1384	# Use mean value for efficiency
1385	val = hash(numpy.average(T.ravel()))
1386	elif type(T) == InstanceType:
1387	val = myhash(T.__dict__, ids)
1388	else:
1389	try:
1390	val = hash(T)
1391	except:
1392	val = 1
1393
1394	return(val)
1395
1396
1397
1398	def compare(A, B, ids=None):
1399	"""Safe comparison of general objects
1400
1401	USAGE:
1402	compare(A,B)
1403
1404	DESCRIPTION:
1405	Return 1 if A and B they are identical, 0 otherwise
1406	"""
1407
1408	from types import TupleType, ListType, DictType, InstanceType
1409	## from numpy.oldnumeric import ArrayType, alltrue
1410
1411	# Keep track of unique id's to protect against infinite recursion
1412	if ids is None: ids = {}
1413
1414	# Check if T has already been encountered
1415	iA = id(A)
1416	iB = id(B)
1417
1418	if ids.has_key((iA, iB)):
1419	# A and B have been compared already
1420	#print 'Found', (iA, iB), A, B
1421	return ids[(iA, iB)]
1422	else:
1423	ids[(iA, iB)] = True
1424
1425
1426	# Check if arguments are of same type
1427	if type(A) != type(B):
1428	return False
1429
1430
1431	# Compare recursively based on argument type
1432	if type(A) in [TupleType, ListType]:
1433	N = len(A)
1434	if len(B) != N:
1435	identical = False
1436	else:
1437	identical = True
1438	for i in range(N):
1439	if not compare(A[i], B[i], ids):
1440	identical = False; break
1441
1442	elif type(A) == DictType:
1443	if len(A) != len(B):
1444	identical = False
1445	else:
1446	# Make dictionary ordering unique
1447	a = A.items(); a.sort()
1448	b = B.items(); b.sort()
1449
1450	identical = compare(a, b, ids)
1451
1452	elif isinstance(A, numpy.ndarray):
1453	# Use element by element comparison
1454	identical = numpy.alltrue(A==B)
1455
1456	elif type(A) == InstanceType:
1457	# Take care of special case where elements are instances
1458	# Base comparison on attributes
1459	identical = compare(A.__dict__,
1460	B.__dict__,
1461	ids)
1462	else:
1463	# Fall back to general code
1464	try:
1465	identical = (A == B)
1466	except:
1467	import pickle
1468	# Use pickle to compare data
1469	# The native pickler must be used
1470	# since the faster cPickle does not
1471	# guarantee a unique translation
1472	try:
1473	identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
1474	except:
1475	identical = False
1476
1477	# Record result of comparison and return
1478	ids[(iA, iB)] = identical
1479
1480	return(identical)
1481
1482
1483	# -----------------------------------------------------------------------------
1484
1485	def nospace(s):
1486	"""Replace spaces in string s with underscores
1487
1488	USAGE:
1489	nospace(s)
1490
1491	ARGUMENTS:
1492	s -- string
1493	"""
1494
1495	import string
1496
1497	newstr = ''
1498	for i in range(len(s)):
1499	if s[i] == ' ':
1500	newstr = newstr+'_'
1501	else:
1502	newstr = newstr+s[i]
1503
1504	return(newstr)
1505
1506	# -----------------------------------------------------------------------------
1507
1508	def get_funcname(func):
1509	"""Retrieve name of function object func (depending on its type)
1510
1511	USAGE:
1512	get_funcname(func)
1513	"""
1514
1515	import types, string
1516
1517	if type(func) == types.FunctionType:
1518	funcname = func.func_name
1519	elif type(func) == types.BuiltinFunctionType:
1520	funcname = func.__name__
1521	else:
1522	tab = string.maketrans("<>'"," ")
1523	tmp = string.translate(`func`,tab)
1524	tmp = string.split(tmp)
1525	funcname = string.join(tmp)
1526
1527	# Truncate memory address as in
1528	# class __main__.Dummy at 0x00A915D0
1529	index = funcname.find('at 0x')
1530	if index >= 0:
1531	funcname = funcname[:index+5] # Keep info that there is an address
1532
1533	funcname = nospace(funcname)
1534	return(funcname)
1535
1536	# -----------------------------------------------------------------------------
1537
1538	def get_bytecode(func):
1539	""" Get bytecode from function object.
1540
1541	USAGE:
1542	get_bytecode(func)
1543	"""
1544
1545	import types
1546
1547	if type(func) == types.FunctionType:
1548	bytecode = func.func_code.co_code
1549	consts = func.func_code.co_consts
1550	argcount = func.func_code.co_argcount
1551	defaults = func.func_defaults
1552	elif type(func) == types.MethodType:
1553	bytecode = func.im_func.func_code.co_code
1554	consts = func.im_func.func_code.co_consts
1555	argcount = func.im_func.func_code.co_argcount
1556	defaults = func.im_func.func_defaults
1557	else:
1558	#raise Exception #Test only
1559	bytecode = None #Built-in functions are assumed not to change
1560	consts = 0
1561	argcount = 0
1562	defaults = 0
1563
1564	return (bytecode, consts, argcount, defaults)
1565
1566	# -----------------------------------------------------------------------------
1567
1568	def get_depstats(dependencies):
1569	""" Build dictionary of dependency files and their size, mod. time and ctime.
1570
1571	USAGE:
1572	get_depstats(dependencies):
1573	"""
1574
1575	import types
1576
1577	#print 'Caching DEBUG: Dependencies are', dependencies
1578	d = {}
1579	if dependencies:
1580
1581	#Expand any wildcards
1582	import glob
1583	expanded_dependencies = []
1584	for FN in dependencies:
1585	expanded_FN = glob.glob(FN)
1586
1587	if expanded_FN == []:
1588	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1589	raise Exception, errmsg
1590
1591	expanded_dependencies += expanded_FN
1592
1593
1594	for FN in expanded_dependencies:
1595	if not type(FN) == types.StringType:
1596	errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1597	errmsg += ' Dependency given: %s' %FN
1598	raise Exception, errmsg
1599	if not os.access(FN,os.F_OK):
1600	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1601	raise Exception, errmsg
1602	(size,atime,mtime,ctime) = filestat(FN)
1603
1604	# We don't use atime because that would cause recomputation every time.
1605	# We don't use ctime because that is irrelevant and confusing for users.
1606	d.update({FN : (size,mtime)})
1607
1608	return(d)
1609
1610	# -----------------------------------------------------------------------------
1611
1612	def filestat(FN):
1613	"""A safe wrapper using os.stat to get basic file statistics
1614	The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1615
1616	USAGE:
1617	filestat(FN)
1618
1619	DESCRIPTION:
1620	Must compile Python with
1621	CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1622	configure
1623	as given in section 8.1.1 Large File Support in the Libray Reference
1624	"""
1625
1626	import os, time
1627
1628	try:
1629	stats = os.stat(FN)
1630	size = stats[6]
1631	atime = stats[7]
1632	mtime = stats[8]
1633	ctime = stats[9]
1634	except:
1635
1636	# Hack to get the results anyway (works only on Unix at the moment)
1637	#
1638	print 'Hack to get os.stat when files are too large'
1639
1640	if unix:
1641	tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1642	# Unique filename, FIXME: Use random number
1643
1644	# Get size and access time (atime)
1645	#
1646	exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1647	(size,atime) = get_lsline(tmp)
1648
1649	# Get size and modification time (mtime)
1650	#
1651	exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1652	(size,mtime) = get_lsline(tmp)
1653
1654	# Get size and ctime
1655	#
1656	exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1657	(size,ctime) = get_lsline(tmp)
1658
1659	try:
1660	exitcode=os.system('rm '+tmp)
1661	# FIXME: Gives error if file doesn't exist
1662	except:
1663	pass
1664	else:
1665	pass
1666	raise Exception # FIXME: Windows case
1667
1668	return(long(size),atime,mtime,ctime)
1669
1670	# -----------------------------------------------------------------------------
1671
1672	def get_lsline(FN):
1673	"""get size and time for filename
1674
1675	USAGE:
1676	get_lsline(file_name)
1677
1678	DESCRIPTION:
1679	Read in one line 'ls -la' item from file (generated by filestat) and
1680	convert time to seconds since epoch. Return file size and time.
1681	"""
1682
1683	import string, time
1684
1685	f = open(FN,'r')
1686	info = f.read()
1687	info = string.split(info)
1688
1689	size = info[4]
1690	week = info[5]
1691	mon = info[6]
1692	day = info[7]
1693	hour = info[8]
1694	year = info[9]
1695
1696	str = week+' '+mon+' '+day+' '+hour+' '+year
1697	timetup = time.strptime(str)
1698	t = time.mktime(timetup)
1699	return(size, t)
1700
1701	# -----------------------------------------------------------------------------
1702
1703	def checkdir(CD,verbose=None):
1704	"""Check or create caching directory
1705
1706	USAGE:
1707	checkdir(CD,verbose):
1708
1709	ARGUMENTS:
1710	CD -- Directory
1711	verbose -- Flag verbose output (default: None)
1712
1713	DESCRIPTION:
1714	If CD does not exist it will be created if possible
1715	"""
1716
1717	import os
1718	import os.path
1719
1720	if CD[-1] != os.sep:
1721	CD = CD + os.sep # Add separator for directories
1722
1723	CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1724	if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1725	try:
1726	exitcode=os.mkdir(CD)
1727
1728	# Change access rights if possible
1729	#
1730	if unix:
1731	exitcode=os.system('chmod 777 '+CD)
1732	else:
1733	pass # FIXME: What about acces rights under Windows?
1734	if verbose: print 'MESSAGE: Directory', CD, 'created.'
1735	except:
1736	print 'WARNING: Directory', CD, 'could not be created.'
1737	if unix:
1738	CD = '/tmp/'
1739	else:
1740	CD = 'C:'
1741	print 'Using directory %s instead' %CD
1742
1743	return(CD)
1744
1745	#==============================================================================
1746	# Statistics
1747	#==============================================================================
1748
1749	def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
1750	compression):
1751	"""Add stats entry
1752
1753	USAGE:
1754	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1755
1756	DESCRIPTION:
1757	Make one entry in the stats file about one cache hit recording time saved
1758	and other statistics. The data are used by the function cachestat.
1759	"""
1760
1761	import os, time
1762
1763	try:
1764	TimeTuple = time.localtime(time.time())
1765	extension = time.strftime('%b%Y',TimeTuple)
1766	SFN = CD+statsfile+'.'+extension
1767	#statfile = open(SFN,'a')
1768	(statfile, dummy) = myopen(SFN,'a',compression=0)
1769
1770	# Change access rights if possible
1771	#
1772	#if unix:
1773	# try:
1774	# exitcode=os.system('chmod 666 '+SFN)
1775	# except:
1776	# pass
1777	except:
1778	print 'Warning: Stat file could not be opened'
1779
1780	try:
1781	if os.environ.has_key('USER'):
1782	user = os.environ['USER']
1783	else:
1784	user = 'Nobody'
1785
1786	date = time.asctime(TimeTuple)
1787
1788	if Retrieved:
1789	hit = '1'
1790	else:
1791	hit = '0'
1792
1793	# Get size of result file
1794	#
1795	if compression:
1796	stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1797	else:
1798	stats = os.stat(CD+FN+'_'+file_types[0])
1799
1800	if stats:
1801	size = stats[6]
1802	else:
1803	size = -1 # Error condition, but don't crash. This is just statistics
1804
1805	# Build entry
1806	#
1807	entry = date + ',' +\
1808	user + ',' +\
1809	FN + ',' +\
1810	str(int(size)) + ',' +\
1811	str(compression) + ',' +\
1812	hit + ',' +\
1813	str(reason) + ',' +\
1814	str(round(comptime,4)) + ',' +\
1815	str(round(loadtime,4)) +\
1816	CR
1817
1818	statfile.write(entry)
1819	statfile.close()
1820	except:
1821	print 'Warning: Writing of stat file failed'
1822
1823	# -----------------------------------------------------------------------------
1824
1825	# FIXME: should take cachedir as an optional arg
1826	#
1827	def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1828	""" List caching statistics.
1829
1830	USAGE:
1831	__cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1832
1833	Generate statistics of caching efficiency.
1834	The parameter sortidx determines by what field lists are sorted.
1835	If the optional keyword period is set to -1,
1836	all available caching history is used.
1837	If it is 0 only the current month is used.
1838	Future versions will include more than one month....
1839	OMN 20/8/2000
1840	"""
1841
1842	import os
1843	import os.path
1844	from string import split, rstrip, find
1845	from time import strptime, localtime, strftime, mktime, ctime
1846
1847	# sortidx = 4 # Index into Fields[1:]. What to sort by.
1848
1849	Fields = ['Name', 'Hits', 'Exec(s)', \
1850	'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1851	Widths = [25,7,9,9,9,9,13]
1852	#Types = ['s','d','d','d','d','.2f','d']
1853	Types = ['s','d','.2f','.2f','.2f','.2f','d']
1854
1855	Dictnames = ['Function', 'User']
1856
1857	if not cachedir:
1858	cachedir = checkdir(options['cachedir'])
1859
1860	SD = os.path.expanduser(cachedir) # Expand ~ or ~user in pathname
1861
1862	if period == -1: # Take all available stats
1863	SFILENAME = statsfile
1864	else: # Only stats from current month
1865	# MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1866	TimeTuple = localtime(time())
1867	extension = strftime('%b%Y',TimeTuple)
1868	SFILENAME = statsfile+'.'+extension
1869
1870	DIRLIST = os.listdir(SD)
1871	SF = []
1872	for FN in DIRLIST:
1873	if find(FN,SFILENAME) >= 0:
1874	SF.append(FN)
1875
1876	blocksize = 15000000
1877	total_read = 0
1878	total_hits = 0
1879	total_discarded = 0
1880	firstday = mktime(strptime('2030','%Y'))
1881	# FIXME: strptime don't exist in WINDOWS ?
1882	lastday = 0
1883
1884	FuncDict = {}
1885	UserDict = {}
1886	for FN in SF:
1887	input = open(SD+FN,'r')
1888	print 'Reading file ', SD+FN
1889
1890	while 1:
1891	A = input.readlines(blocksize)
1892	if len(A) == 0: break
1893	total_read = total_read + len(A)
1894	for record in A:
1895	record = tuple(split(rstrip(record),','))
1896	#print record, len(record)
1897
1898	if len(record) == 9:
1899	timestamp = record[0]
1900
1901	try:
1902	t = mktime(strptime(timestamp))
1903	except:
1904	total_discarded = total_discarded + 1
1905	continue
1906
1907	if t > lastday:
1908	lastday = t
1909	if t < firstday:
1910	firstday = t
1911
1912	user = record[1]
1913	func = record[2]
1914
1915	# Strip hash-stamp off
1916	#
1917	i = find(func,'[')
1918	func = func[:i]
1919
1920	size = float(record[3])
1921
1922	# Compression kepword can be Boolean
1923	if record[4] in ['True', '1']:
1924	compression = 1
1925	elif record[4] in ['False', '0']:
1926	compression = 0
1927	else:
1928	print 'Unknown value of compression', record[4]
1929	print record
1930	total_discarded = total_discarded + 1
1931	continue
1932
1933	#compression = int(record[4]) # Can be Boolean
1934	hit = int(record[5])
1935	reason = int(record[6]) # Not used here
1936	cputime = float(record[7])
1937	loadtime = float(record[8])
1938
1939	if hit:
1940	total_hits = total_hits + 1
1941	saving = cputime-loadtime
1942
1943	if cputime != 0:
1944	rel_saving = round(100.0*saving/cputime,2)
1945	else:
1946	#rel_saving = round(1.0*saving,2)
1947	rel_saving = 100.0 - round(1.0*saving,2) # A bit of a hack
1948
1949	info = [1,cputime,loadtime,saving,rel_saving,size]
1950
1951	UpdateDict(UserDict,user,info)
1952	UpdateDict(FuncDict,func,info)
1953	else:
1954	pass #Stats on recomputations and their reasons could go in here
1955
1956	else:
1957	#print 'Record discarded'
1958	#print record
1959	total_discarded = total_discarded + 1
1960
1961	input.close()
1962
1963	# Compute averages of all sums and write list
1964	#
1965
1966	if total_read == 0:
1967	printline(Widths,'=')
1968	print 'CACHING STATISTICS: No valid records read'
1969	printline(Widths,'=')
1970	return
1971
1972	print
1973	printline(Widths,'=')
1974	print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
1975	printline(Widths,'=')
1976	#print ' Period:', ctime(firstday), 'to', ctime(lastday)
1977	print ' Total number of valid records', total_read
1978	print ' Total number of discarded records', total_discarded
1979	print ' Total number of hits', total_hits
1980	print
1981
1982	print ' Fields', Fields[2:], 'are averaged over number of hits'
1983	print ' Time is measured in seconds and size in bytes'
1984	print ' Tables are sorted by', Fields[1:][sortidx]
1985
1986	# printline(Widths,'-')
1987
1988	if showuser:
1989	Dictionaries = [FuncDict, UserDict]
1990	else:
1991	Dictionaries = [FuncDict]
1992
1993	i = 0
1994	for Dict in Dictionaries:
1995	for key in Dict.keys():
1996	rec = Dict[key]
1997	for n in range(len(rec)):
1998	if n > 0:
1999	rec[n] = round(1.0*rec[n]/rec[0],2)
2000	Dict[key] = rec
2001
2002	# Sort and output
2003	#
2004	keylist = SortDict(Dict,sortidx)
2005
2006	# Write Header
2007	#
2008	print
2009	#print Dictnames[i], 'statistics:'; i=i+1
2010	printline(Widths,'-')
2011	n = 0
2012	for s in Fields:
2013	if s == Fields[0]: # Left justify
2014	s = Dictnames[i] + ' ' + s; i=i+1
2015	exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
2016	else:
2017	exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
2018	print
2019	printline(Widths,'-')
2020
2021	# Output Values
2022	#
2023	for key in keylist:
2024	rec = Dict[key]
2025	n = 0
2026	if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
2027	exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
2028	for val in rec:
2029	exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
2030	print
2031	print
2032
2033	#==============================================================================
2034	# Auxiliary stats functions
2035	#==============================================================================
2036
2037	def UpdateDict(Dict,key,info):
2038	"""Update dictionary by adding new values to existing.
2039
2040	USAGE:
2041	UpdateDict(Dict,key,info)
2042	"""
2043
2044	if Dict.has_key(key):
2045	dinfo = Dict[key]
2046	for n in range(len(dinfo)):
2047	dinfo[n] = info[n] + dinfo[n]
2048	else:
2049	dinfo = info[:] # Make a copy of info list
2050
2051	Dict[key] = dinfo
2052	return Dict
2053
2054	# -----------------------------------------------------------------------------
2055
2056	def SortDict(Dict,sortidx=0):
2057	"""Sort dictionary
2058
2059	USAGE:
2060	SortDict(Dict,sortidx):
2061
2062	DESCRIPTION:
2063	Sort dictionary of lists according field number 'sortidx'
2064	"""
2065
2066	import types
2067
2068	sortlist = []
2069	keylist = Dict.keys()
2070	for key in keylist:
2071	rec = Dict[key]
2072	if not type(rec) in [types.ListType, types.TupleType]:
2073	rec = [rec]
2074
2075	if sortidx > len(rec)-1:
2076	if options['verbose']:
2077	print 'ERROR: Sorting index to large, sortidx = ', sortidx
2078	raise IndexError
2079
2080	val = rec[sortidx]
2081	sortlist.append(val)
2082
2083	A = map(None,sortlist,keylist)
2084	A.sort()
2085	keylist = map(lambda x: x[1], A) # keylist sorted by sortidx
2086
2087	return(keylist)
2088
2089	# -----------------------------------------------------------------------------
2090
2091	def printline(Widths,char):
2092	"""Print textline in fixed field.
2093
2094	USAGE:
2095	printline(Widths,char)
2096	"""
2097
2098	s = ''
2099	for n in range(len(Widths)):
2100	s = s+Widths[n]*char
2101	if n > 0:
2102	s = s+char
2103
2104	print s
2105
2106	#==============================================================================
2107	# Messages
2108	#==============================================================================
2109
2110	def msg1(funcname,args,kwargs,reason):
2111	"""Message 1
2112
2113	USAGE:
2114	msg1(funcname,args,kwargs,reason):
2115	"""
2116
2117	import string
2118	#print 'MESSAGE (caching.py): Evaluating function', funcname,
2119
2120	print_header_box('Evaluating function %s' %funcname)
2121
2122	msg7(args, kwargs)
2123	msg8(reason)
2124
2125	print_footer()
2126
2127	#
2128	# Old message
2129	#
2130	#args_present = 0
2131	#if args:
2132	# if len(args) == 1:
2133	# print 'with argument', mkargstr(args[0], textwidth2),
2134	# else:
2135	# print 'with arguments', mkargstr(args, textwidth2),
2136	# args_present = 1
2137	#
2138	#if kwargs:
2139	# if args_present:
2140	# word = 'and'
2141	# else:
2142	# word = 'with'
2143	#
2144	# if len(kwargs) == 1:
2145	# print word + ' keyword argument', mkargstr(kwargs, textwidth2)
2146	# else:
2147	# print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
2148	# args_present = 1
2149	#else:
2150	# print # Newline when no keyword args present
2151	#
2152	#if not args_present:
2153	# print '', # Default if no args or kwargs present
2154
2155
2156
2157	# -----------------------------------------------------------------------------
2158
2159	def msg2(funcname,args,kwargs,comptime,reason):
2160	"""Message 2
2161
2162	USAGE:
2163	msg2(funcname,args,kwargs,comptime,reason)
2164	"""
2165
2166	import string
2167
2168	#try:
2169	# R = Reason_msg[reason]
2170	#except:
2171	# R = 'Unknown reason'
2172
2173	#print_header_box('Caching statistics (storing) - %s' %R)
2174	print_header_box('Caching statistics (storing)')
2175
2176	msg6(funcname,args,kwargs)
2177	msg8(reason)
2178
2179	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2180
2181	# -----------------------------------------------------------------------------
2182
2183	def msg3(savetime, CD, FN, deps,compression):
2184	"""Message 3
2185
2186	USAGE:
2187	msg3(savetime, CD, FN, deps,compression)
2188	"""
2189
2190	import string
2191	print string.ljust('\| Loading time:', textwidth1) + str(round(savetime,2)) + \
2192	' seconds (estimated)'
2193	msg5(CD,FN,deps,compression)
2194
2195	# -----------------------------------------------------------------------------
2196
2197	def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
2198	"""Message 4
2199
2200	USAGE:
2201	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
2202	"""
2203
2204	import string
2205
2206	print_header_box('Caching statistics (retrieving)')
2207
2208	msg6(funcname,args,kwargs)
2209	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2210	print string.ljust('\| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
2211	print string.ljust('\| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
2212	' seconds'
2213	msg5(CD,FN,deps,compression)
2214
2215	# -----------------------------------------------------------------------------
2216
2217	def msg5(CD,FN,deps,compression):
2218	"""Message 5
2219
2220	USAGE:
2221	msg5(CD,FN,deps,compression)
2222
2223	DESCRIPTION:
2224	Print dependency stats. Used by msg3 and msg4
2225	"""
2226
2227	import os, time, string
2228
2229	print '\|'
2230	print string.ljust('\| Caching dir: ', textwidth1) + CD
2231
2232	if compression:
2233	suffix = '.z'
2234	bytetext = 'bytes, compressed'
2235	else:
2236	suffix = ''
2237	bytetext = 'bytes'
2238
2239	for file_type in file_types:
2240	file_name = FN + '_' + file_type + suffix
2241	print string.ljust('\| ' + file_type + ' file: ', textwidth1) + file_name,
2242	stats = os.stat(CD+file_name)
2243	print '('+ str(stats[6]) + ' ' + bytetext + ')'
2244
2245	print '\|'
2246	if len(deps) > 0:
2247	print '\| Dependencies: '
2248	dependencies = deps.keys()
2249	dlist = []; maxd = 0
2250	tlist = []; maxt = 0
2251	slist = []; maxs = 0
2252	for d in dependencies:
2253	stats = deps[d]
2254	t = time.ctime(stats[1])
2255	s = str(stats[0])
2256	#if s[-1] == 'L':
2257	# s = s[:-1] # Strip rightmost 'long integer' L off.
2258	# # FIXME: Unnecessary in versions later than 1.5.2
2259
2260	if len(d) > maxd: maxd = len(d)
2261	if len(t) > maxt: maxt = len(t)
2262	if len(s) > maxs: maxs = len(s)
2263	dlist.append(d)
2264	tlist.append(t)
2265	slist.append(s)
2266
2267	for n in range(len(dlist)):
2268	d = string.ljust(dlist[n]+':', maxd+1)
2269	t = string.ljust(tlist[n], maxt)
2270	s = string.rjust(slist[n], maxs)
2271
2272	print '\| ', d, t, ' ', s, 'bytes'
2273	else:
2274	print '\| No dependencies'
2275	print_footer()
2276
2277	# -----------------------------------------------------------------------------
2278
2279	def msg6(funcname,args,kwargs):
2280	"""Message 6
2281
2282	USAGE:
2283	msg6(funcname,args,kwargs)
2284	"""
2285
2286	import string
2287	print string.ljust('\| Function:', textwidth1) + funcname
2288
2289	msg7(args, kwargs)
2290
2291	# -----------------------------------------------------------------------------
2292
2293	def msg7(args, kwargs):
2294	"""Message 7
2295
2296	USAGE:
2297	msg7(args, kwargs):
2298	"""
2299
2300	import string
2301
2302	args_present = 0
2303	if args:
2304	if len(args) == 1:
2305	print string.ljust('\| Argument:', textwidth1) + mkargstr(args[0], \
2306	textwidth2)
2307	else:
2308	print string.ljust('\| Arguments:', textwidth1) + \
2309	mkargstr(args, textwidth2)
2310	args_present = 1
2311
2312	if kwargs:
2313	if len(kwargs) == 1:
2314	print string.ljust('\| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
2315	textwidth2)
2316	else:
2317	print string.ljust('\| Keyword Args:', textwidth1) + \
2318	mkargstr(kwargs, textwidth2)
2319	args_present = 1
2320
2321	if not args_present:
2322	print '\| No arguments' # Default if no args or kwargs present
2323
2324	# -----------------------------------------------------------------------------
2325
2326	def msg8(reason):
2327	"""Message 8
2328
2329	USAGE:
2330	msg8(reason):
2331	"""
2332
2333	import string
2334
2335	try:
2336	R = Reason_msg[reason]
2337	except:
2338	R = 'Unknown'
2339
2340	print string.ljust('\| Reason:', textwidth1) + R
2341
2342	# -----------------------------------------------------------------------------
2343
2344	def print_header_box(line):
2345	"""Print line in a nice box.
2346
2347	USAGE:
2348	print_header_box(line)
2349
2350	"""
2351	global textwidth3
2352
2353	import time
2354
2355	time_stamp = time.ctime(time.time())
2356	line = time_stamp + '. ' + line
2357
2358	N = len(line) + 1
2359	s = '+' + '-'*N + CR
2360
2361	print s + '\| ' + line + CR + s,
2362
2363	textwidth3 = N
2364
2365	# -----------------------------------------------------------------------------
2366
2367	def print_footer():
2368	"""Print line same width as that of print_header_box.
2369	"""
2370
2371	N = textwidth3
2372	s = '+' + '-'*N + CR
2373
2374	print s
2375
2376	# -----------------------------------------------------------------------------
2377
2378	def mkargstr(args, textwidth, argstr = '', level=0):
2379	""" Generate a string containing first textwidth characters of arguments.
2380
2381	USAGE:
2382	mkargstr(args, textwidth, argstr = '', level=0)
2383
2384	DESCRIPTION:
2385	Exactly the same as str(args) possibly followed by truncation,
2386	but faster if args is huge.
2387	"""
2388
2389	import types
2390
2391	if level > 10:
2392	# Protect against circular structures
2393	return '...'
2394
2395	WasTruncated = 0
2396
2397	if not type(args) in [types.TupleType, types.ListType, types.DictType]:
2398	if type(args) == types.StringType:
2399	argstr = argstr + "'"+str(args)+"'"
2400	else:
2401	# Truncate large arrays before using str()
2402	if isinstance(args, numpy.ndarray):
2403	# if len(args.ravel()) > textwidth:
2404	# Changed by Duncan and Nick 21/2/07 .ravel() has problems with
2405	# non-contigous arrays and ravel is equal to .ravel() except it
2406	# can work with non-contiguous arrays
2407	if len(numpy.ravel(args)) > textwidth:
2408	args = 'Array: ' + str(args.shape)
2409
2410	argstr = argstr + str(args)
2411	else:
2412	if type(args) == types.DictType:
2413	argstr = argstr + "{"
2414	for key in args.keys():
2415	argstr = argstr + mkargstr(key, textwidth, level=level+1) + ": " + \
2416	mkargstr(args[key], textwidth, level=level+1) + ", "
2417	if len(argstr) > textwidth:
2418	WasTruncated = 1
2419	break
2420	argstr = argstr[:-2] # Strip off trailing comma
2421	argstr = argstr + "}"
2422
2423	else:
2424	if type(args) == types.TupleType:
2425	lc = '('
2426	rc = ')'
2427	else:
2428	lc = '['
2429	rc = ']'
2430	argstr = argstr + lc
2431	for arg in args:
2432	argstr = argstr + mkargstr(arg, textwidth, level=level+1) + ', '
2433	if len(argstr) > textwidth:
2434	WasTruncated = 1
2435	break
2436
2437	# Strip off trailing comma and space unless singleton tuple
2438	#
2439	if type(args) == types.TupleType and len(args) == 1:
2440	argstr = argstr[:-1]
2441	else:
2442	argstr = argstr[:-2]
2443	argstr = argstr + rc
2444
2445	if len(argstr) > textwidth:
2446	WasTruncated = 1
2447
2448	if WasTruncated:
2449	argstr = argstr[:textwidth]+'...'
2450	return(argstr)
2451
2452	# -----------------------------------------------------------------------------
2453
2454	def test_OK(msg):
2455	"""Print OK msg if test is OK.
2456
2457	USAGE
2458	test_OK(message)
2459	"""
2460
2461	import string
2462
2463	print string.ljust(msg, textwidth4) + ' - OK'
2464
2465	#raise StandardError
2466
2467	# -----------------------------------------------------------------------------
2468
2469	def test_error(msg):
2470	"""Print error if test fails.
2471
2472	USAGE
2473	test_error(message)
2474	"""
2475
2476	print 'ERROR (caching.test): %s' %msg
2477	print 'Please send this code example and output to '
2478	print 'Ole.Nielsen@anu.edu.au'
2479	print
2480	print
2481
2482	#import sys
2483	#sys.exit()
2484	raise StandardError
2485
2486	#-------------------------------------------------------------
2487	if __name__ == "__main__":
2488	pass

Note: See TracBrowser for help on using the repository browser.

Download in other formats: