Context Navigation

source: anuga_core/source_numpy_conversion/anuga/caching/caching.py @ 5909

Last change on this file since 5909 was 5909, checked in by rwilson, 15 years ago
NumPy? conversion.
File size: 72.0 KB

Line
1	# =============================================================================
2	# caching.py - Supervised caching of function results.
3	# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4	# Australian National University (1999-2003)
5	# Geoscience Australia (2003-present)
6	#
7	# This program is free software; you can redistribute it and/or modify
8	# it under the terms of the GNU General Public License as published by
9	# the Free Software Foundation; either version 2 of the License, or
10	# (at your option) any later version.
11	#
12	# This program is distributed in the hope that it will be useful,
13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	# GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16	# for more details.
17	#
18	# You should have received a copy of the GNU General Public License
19	# along with this program; if not, write to the Free Software
20	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
21	#
22	#
23	# Contact address: Ole.Nielsen@ga.gov.au
24	#
25	# Version 1.5.6 February 2002
26	# =============================================================================
27
28	"""Module caching.py - Supervised caching of function results.
29
30	Public functions:
31
32	cache(func,args) -- Cache values returned from func given args.
33	cachestat() -- Reports statistics about cache hits and time saved.
34	test() -- Conducts a basic test of the caching functionality.
35
36	See doc strings of individual functions for detailed documentation.
37	"""
38
39	# -----------------------------------------------------------------------------
40	# Initialisation code
41
42	# Determine platform
43	#
44	from os import getenv
45	import types
46
47	import os
48	if os.name in ['nt', 'dos', 'win32', 'what else?']:
49	unix = 0
50	else:
51	unix = 1
52
53	import numpy
54
55
56	cache_dir = '.python_cache'
57	# Make default caching directory name
58	#
59	if unix:
60	homedir = getenv('INUNDATIONHOME')
61	if homedir is None:
62	homedir = '~'
63	else:
64	homedir = homedir + os.sep + '.cache'
65	# Since homedir will be a group area, individually label the caches
66	user = getenv('LOGNAME')
67	if user is not None:
68	cache_dir += '_' + user
69
70	CR = '\n'
71	else:
72	homedir = 'c:'
73	CR = '\r\n' #FIXME: Not tested under windows
74
75	cachedir = homedir + os.sep + cache_dir + os.sep
76
77	# -----------------------------------------------------------------------------
78	# Options directory with default values - to be set by user
79	#
80
81	options = {
82	'cachedir': cachedir, # Default cache directory
83	'maxfiles': 1000000, # Maximum number of cached files
84	'savestat': 1, # Log caching info to stats file
85	'verbose': 1, # Write messages to standard output
86	'bin': 1, # Use binary format (more efficient)
87	'compression': 1, # Use zlib compression
88	'bytecode': 0, # Recompute if bytecode has changed
89	'expire': 0 # Automatically remove files that have been accessed
90	# least recently
91	}
92
93	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
94
95	def set_option(key, value):
96	"""Function to set values in the options directory.
97
98	USAGE:
99	set_option(key, value)
100
101	ARGUMENTS:
102	key -- Key in options dictionary. (Required)
103	value -- New value for key. (Required)
104
105	DESCRIPTION:
106	Function to set values in the options directory.
107	Raises an exception if key is not in options.
108	"""
109
110	if options.has_key(key):
111	options[key] = value
112	else:
113	raise KeyError(key) # Key not found, raise an exception
114
115	# -----------------------------------------------------------------------------
116	# Function cache - the main routine
117
118	def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
119	verbose=None, compression=None, evaluate=0, test=0, clear=0,
120	return_filename=0):
121	"""Supervised caching of function results.
122
123	USAGE:
124	result = cache(func, args, kwargs, dependencies, cachedir, verbose,
125	compression, evaluate, test, return_filename)
126
127	ARGUMENTS:
128	func -- Function object (Required)
129	args -- Arguments to func (Default: ())
130	kwargs -- Keyword arguments to func (Default: {})
131	dependencies -- Filenames that func depends on (Default: None)
132	cachedir -- Directory for cache files (Default: options['cachedir'])
133	verbose -- Flag verbose output to stdout
134	(Default: options['verbose'])
135	compression -- Flag zlib compression (Default: options['compression'])
136	evaluate -- Flag forced evaluation of func (Default: 0)
137	test -- Flag test for cached results (Default: 0)
138	clear -- Flag delete cached results (Default: 0)
139	return_filename -- Flag return of cache filename (Default: 0)
140
141	DESCRIPTION:
142	A Python function call of the form
143
144	result = func(arg1,...,argn)
145
146	can be replaced by
147
148	from caching import cache
149	result = cache(func,(arg1,...,argn))
150
151	The latter form returns the same output as the former but reuses cached
152	results if the function has been computed previously in the same context.
153	'result' and the arguments can be simple types, tuples, list, dictionaries or
154	objects, but not unhashable types such as functions or open file objects.
155	The function 'func' may be a member function of an object or a module.
156
157	This type of caching is particularly useful for computationally intensive
158	functions with few frequently used combinations of input arguments. Note that
159	if the inputs or output are very large caching might not save time because
160	disc access may dominate the execution time.
161
162	If the function definition changes after a result has been cached it will be
163	detected by examining the functions bytecode (co_code, co_consts,
164	func_defualts, co_argcount) and it will be recomputed.
165
166	LIMITATIONS:
167	1 Caching uses the apply function and will work with anything that can be
168	pickled, so any limitation in apply or pickle extends to caching.
169	2 A function to be cached should not depend on global variables
170	as wrong results may occur if globals are changed after a result has
171	been cached.
172
173	-----------------------------------------------------------------------------
174	Additional functionality:
175
176	Keyword args
177	Keyword arguments (kwargs) can be added as a dictionary of keyword: value
178	pairs, following the syntax of the built-in function apply().
179	A Python function call of the form
180
181	result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)
182
183	is then cached as follows
184
185	from caching import cache
186	result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
187
188	The default value of kwargs is {}
189
190	Explicit dependencies:
191	The call
192	cache(func,(arg1,...,argn),dependencies = <list of filenames>)
193	Checks the size, creation time and modification time of each listed file.
194	If any file has changed the function is recomputed and the results stored
195	again.
196
197	Specify caching directory:
198	The call
199	cache(func,(arg1,...,argn), cachedir = <cachedir>)
200	designates <cachedir> where cached data are stored. Use ~ to indicate users
201	home directory - not $HOME. The default is ~/.python_cache on a UNIX
202	platform and c:/.python_cache on a Win platform.
203
204	Silent operation:
205	The call
206	cache(func,(arg1,...,argn),verbose=0)
207	suppresses messages to standard output.
208
209	Compression:
210	The call
211	cache(func,(arg1,...,argn),compression=0)
212	disables compression. (Default: compression=1). If the requested compressed
213	or uncompressed file is not there, it'll try the other version.
214
215	Forced evaluation:
216	The call
217	cache(func,(arg1,...,argn),evaluate=1)
218	forces the function to evaluate even though cached data may exist.
219
220	Testing for presence of cached result:
221	The call
222	cache(func,(arg1,...,argn),test=1)
223	retrieves cached result if it exists, otherwise None. The function will not
224	be evaluated. If both evaluate and test are switched on, evaluate takes
225	precedence.
226	??NOTE: In case of hash collisions, this may return the wrong result as
227	??it only checks if a cached result is present.
228
229	Obtain cache filenames:
230	The call
231	cache(func,(arg1,...,argn),return_filename=1)
232	returns the hashed base filename under which this function and its
233	arguments would be cached
234
235	Clearing cached results:
236	The call
237	cache(func,'clear')
238	clears all cached data for 'func' and
239	cache('clear')
240	clears all cached data.
241
242	NOTE: The string 'clear' can be passed an argument to func using
243	cache(func,('clear',)) or cache(func,tuple(['clear'])).
244
245	New form of clear:
246	cache(func,(arg1,...,argn),clear=1)
247	clears cached data for particular combination func and args
248
249	"""
250
251	# Imports and input checks
252	#
253	import types, time, string
254
255	if not cachedir:
256	cachedir = options['cachedir']
257
258	if verbose == None: # Do NOT write 'if not verbose:', it could be zero.
259	verbose = options['verbose']
260
261	if compression == None: # Do NOT write 'if not compression:',
262	# it could be zero.
263	compression = options['compression']
264
265	# Create cache directory if needed
266	CD = checkdir(cachedir,verbose)
267
268	# Handle the case cache('clear')
269	if type(func) == types.StringType:
270	if string.lower(func) == 'clear':
271	clear_cache(CD,verbose=verbose)
272	return
273
274	# Handle the case cache(func, 'clear')
275	if type(args) == types.StringType:
276	if string.lower(args) == 'clear':
277	clear_cache(CD,func,verbose=verbose)
278	return
279
280	# Force singleton arg into a tuple
281	if type(args) != types.TupleType:
282	args = tuple([args])
283
284	# Check that kwargs is a dictionary
285	if type(kwargs) != types.DictType:
286	raise TypeError
287
288	# Hash arguments (and keyword args) to integer
289	arghash = myhash((args, kwargs))
290
291	# Get sizes and timestamps for files listed in dependencies.
292	# Force singletons into a tuple.
293	if dependencies and type(dependencies) != types.TupleType \
294	and type(dependencies) != types.ListType:
295	dependencies = tuple([dependencies])
296	deps = get_depstats(dependencies)
297
298	# Extract function name from func object
299	funcname = get_funcname(func)
300
301	# Create cache filename
302	FN = funcname+'['+`arghash`+']' # The symbol '(' does not work under unix
303
304	if return_filename:
305	return(FN)
306
307	if clear:
308	for file_type in file_types:
309	file_name = CD+FN+'_'+file_type
310	for fn in [file_name, file_name + '.z']:
311	if os.access(fn, os.F_OK):
312	if unix:
313	os.remove(fn)
314	else:
315	# FIXME: os.remove doesn't work under windows
316	os.system('del '+fn)
317	if verbose is True:
318	print 'MESSAGE (caching): File %s deleted' %fn
319	##else:
320	## print '%s was not accessed' %fn
321	return None
322
323
324	#-------------------------------------------------------------------
325
326	# Check if previous computation has been cached
327	if evaluate:
328	Retrieved = None # Force evaluation of func regardless of caching status.
329	reason = 5
330	else:
331	T, FN, Retrieved, reason, comptime, loadtime, compressed = \
332	CacheLookup(CD, FN, func,
333	args, kwargs,
334	deps,
335	verbose,
336	compression,
337	dependencies)
338
339	if not Retrieved:
340	if test: # Do not attempt to evaluate function
341	T = None
342	else: # Evaluate function and save to cache
343	if verbose:
344
345	msg1(funcname, args, kwargs,reason)
346
347	# Remove expired files automatically
348	if options['expire']:
349	DeleteOldFiles(CD,verbose)
350
351	# Save args before function is evaluated in case
352	# they are modified by function
353	save_args_to_cache(CD,FN,args,kwargs,compression)
354
355	# Execute and time function with supplied arguments
356	t0 = time.time()
357	T = apply(func,args,kwargs)
358	#comptime = round(time.time()-t0)
359	comptime = time.time()-t0
360
361	if verbose:
362	msg2(funcname,args,kwargs,comptime,reason)
363
364	# Save results and estimated loading time to cache
365	loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
366	funcname, dependencies, compression)
367	if verbose:
368	msg3(loadtime, CD, FN, deps, compression)
369	compressed = compression
370
371	if options['savestat'] and (not test or Retrieved):
372	##if options['savestat']:
373	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
374
375	return(T) # Return results in all cases
376
377	# -----------------------------------------------------------------------------
378
379	def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
380	"""Generate statistics of caching efficiency.
381
382	USAGE:
383	cachestat(sortidx, period, showuser, cachedir)
384
385	ARGUMENTS:
386	sortidx -- Index of field by which lists are (default: 4)
387	Legal values are
388	0: 'Name'
389	1: 'Hits'
390	2: 'CPU'
391	3: 'Time Saved'
392	4: 'Gain(%)'
393	5: 'Size'
394	period -- If set to -1 all available caching history is used.
395	If set 0 only the current month is used (default -1).
396	showuser -- Flag for additional table showing user statistics
397	(default: None).
398	cachedir -- Directory for cache files (default: options['cachedir']).
399
400	DESCRIPTION:
401	Logged caching statistics is converted into summaries of the form
402	--------------------------------------------------------------------------
403	Function Name Hits Exec(s) Cache(s) Saved(s) Gain(%) Size
404	--------------------------------------------------------------------------
405	"""
406
407	__cachestat(sortidx, period, showuser, cachedir)
408	return
409
410	# -----------------------------------------------------------------------------
411
412	#Has mostly been moved to proper unit test
413	def test(cachedir=None,verbose=0,compression=None):
414	"""Test the functionality of caching.
415
416	USAGE:
417	test(verbose)
418
419	ARGUMENTS:
420	verbose -- Flag whether caching will output its statistics (default=0)
421	cachedir -- Directory for cache files (Default: options['cachedir'])
422	compression -- Flag zlib compression (Default: options['compression'])
423	"""
424
425	import string, time
426
427	# Initialise
428	#
429	import caching
430	reload(caching)
431
432	if not cachedir:
433	cachedir = options['cachedir']
434
435	if verbose is None: # Do NOT write 'if not verbose:', it could be zero.
436	verbose = options['verbose']
437
438	if compression == None: # Do NOT write 'if not compression:',
439	# it could be zero.
440	compression = options['compression']
441	else:
442	try:
443	set_option('compression', compression)
444	except:
445	test_error('Set option failed')
446
447	try:
448	import zlib
449	except:
450	print
451	print '* Could not find zlib, default to no-compression *'
452	print '* Installing zlib will improve performance of caching *'
453	print
454	compression = 0
455	set_option('compression', compression)
456
457	print
458	print_header_box('Testing caching module - please stand by')
459	print
460
461	# Define a test function to be cached
462	#
463	def f(a,b,c,N,x=0,y='abcdefg'):
464	"""f(a,b,c,N)
465	Do something time consuming and produce a complex result.
466	"""
467
468	import string
469
470	B = []
471	for n in range(N):
472	s = str(n+2.0/(n + 4.0))+'.a'*10
473	B.append((a,b,c,s,n,x,y))
474	return(B)
475
476	# Check that default cachedir is OK
477	#
478	CD = checkdir(cachedir,verbose)
479
480
481	# Make a dependency file
482	#
483	try:
484	DepFN = CD + 'testfile.tmp'
485	DepFN_wildcard = CD + 'test*.tmp'
486	Depfile = open(DepFN,'w')
487	Depfile.write('We are the knights who say NI!')
488	Depfile.close()
489	test_OK('Wrote file %s' %DepFN)
490	except:
491	test_error('Could not open file %s for writing - check your environment' \
492	% DepFN)
493
494	# Check set_option (and switch stats off
495	#
496	try:
497	set_option('savestat',0)
498	assert(options['savestat'] == 0)
499	test_OK('Set option')
500	except:
501	test_error('Set option failed')
502
503	# Make some test input arguments
504	#
505	N = 5000 #Make N fairly small here
506
507	a = [1,2]
508	b = ('Thou shalt count the number three',4)
509	c = {'Five is right out': 6, (7,8): 9}
510	x = 3
511	y = 'holy hand granate'
512
513	# Test caching
514	#
515	if compression:
516	comprange = 2
517	else:
518	comprange = 1
519
520	for comp in range(comprange):
521
522	# Evaluate and store
523	#
524	try:
525	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
526	verbose=verbose, compression=comp)
527	if comp:
528	test_OK('Caching evaluation with compression')
529	else:
530	test_OK('Caching evaluation without compression')
531	except:
532	if comp:
533	test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
534	else:
535	test_error('Caching evaluation failed - try caching.test(verbose=1)')
536
537	# Retrieve
538	#
539	try:
540	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
541	compression=comp)
542
543	if comp:
544	test_OK('Caching retrieval with compression')
545	else:
546	test_OK('Caching retrieval without compression')
547	except:
548	if comp:
549	test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
550	else:
551	test_error('Caching retrieval failed - try caching.test(verbose=1)')
552
553	# Reference result
554	#
555	T3 = f(a,b,c,N,x=x,y=y) # Compute without caching
556
557	if T1 == T2 and T2 == T3:
558	if comp:
559	test_OK('Basic caching functionality (with compression)')
560	else:
561	test_OK('Basic caching functionality (without compression)')
562	else:
563	test_error('Cached result does not match computed result')
564
565
566	# Test return_filename
567	#
568	try:
569	FN = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
570	return_filename=1)
571	assert(FN[:2] == 'f[')
572	test_OK('Return of cache filename')
573	except:
574	test_error('Return of cache filename failed')
575
576	# Test existence of cachefiles
577	#
578	try:
579	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
580	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
581	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
582	test_OK('Presence of cache files')
583	datafile.close()
584	argsfile.close()
585	admfile.close()
586	except:
587	test_error('Expected cache files did not exist')
588
589	# Test 'test' function when cache is present
590	#
591	try:
592	#T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
593	# evaluate=1)
594	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
595	assert(T1 == T4)
596
597	test_OK("Option 'test' when cache file present")
598	except:
599	test_error("Option 'test' when cache file present failed")
600
601	# Test that 'clear' works
602	#
603	#try:
604	# caching.cache(f,'clear',verbose=verbose)
605	# test_OK('Clearing of cache files')
606	#except:
607	# test_error('Clear does not work')
608	try:
609	caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)
610	test_OK('Clearing of cache files')
611	except:
612	test_error('Clear does not work')
613
614
615
616	# Test 'test' function when cache is absent
617	#
618	try:
619	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
620	assert(T4 is None)
621	test_OK("Option 'test' when cache absent")
622	except:
623	test_error("Option 'test' when cache absent failed")
624
625	# Test dependencies
626	#
627	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
628	dependencies=DepFN)
629	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
630	dependencies=DepFN)
631
632	if T1 == T2:
633	test_OK('Basic dependencies functionality')
634	else:
635	test_error('Dependencies do not work')
636
637	# Test basic wildcard dependency
638	#
639	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
640	dependencies=DepFN_wildcard)
641
642	if T1 == T3:
643	test_OK('Basic dependencies with wildcard functionality')
644	else:
645	test_error('Dependencies with wildcards do not work')
646
647
648	# Test that changed timestamp in dependencies triggers recomputation
649
650	# Modify dependency file
651	Depfile = open(DepFN,'a')
652	Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
653	Depfile.close()
654
655	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
656	dependencies=DepFN, test = 1)
657
658	if T3 is None:
659	test_OK('Changed dependencies recognised')
660	else:
661	test_error('Changed dependencies not recognised')
662
663	# Test recomputation when dependencies have changed
664	#
665	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
666	dependencies=DepFN)
667	if T1 == T3:
668	test_OK('Recomputed value with changed dependencies')
669	else:
670	test_error('Recomputed value with changed dependencies failed')
671
672	# Performance test (with statistics)
673	# Don't really rely on this as it will depend on specific computer.
674	#
675
676	set_option('savestat',1)
677
678	N = 20*N #Should be large on fast computers...
679	tt = time.time()
680	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
681	t1 = time.time() - tt
682
683	tt = time.time()
684	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
685	t2 = time.time() - tt
686
687	if T1 == T2:
688	if t1 > t2:
689	test_OK('Performance test: relative time saved = %s pct' \
690	%str(round((t1-t2)*100/t1,2)))
691	#else:
692	# print 'WARNING: Performance a bit low - this could be specific to current platform'
693	else:
694	test_error('Basic caching failed for new problem')
695
696	# Test presence of statistics file
697	#
698	try:
699	DIRLIST = os.listdir(CD)
700	SF = []
701	for FN in DIRLIST:
702	if string.find(FN,statsfile) >= 0:
703	fid = open(CD+FN,'r')
704	fid.close()
705	test_OK('Statistics files present')
706	except:
707	test_OK('Statistics files cannot be opened')
708
709	print_header_box('Show sample output of the caching function:')
710
711	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
712	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
713	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
714
715	print_header_box('Show sample output of cachestat():')
716	if unix:
717	cachestat()
718	else:
719	try:
720	import time
721	t = time.strptime('2030','%Y')
722	cachestat()
723	except:
724	print 'caching.cachestat() does not work here, because it'
725	print 'relies on time.strptime() which is unavailable in Windows'
726
727	print
728	test_OK('Caching self test completed')
729
730
731	# Test setoption (not yet implemented)
732	#
733
734
735	#==============================================================================
736	# Auxiliary functions
737	#==============================================================================
738
739	# Import pickler
740	# cPickle is used by functions mysave, myload, and compare
741	#
742	import cPickle # 10 to 100 times faster than pickle
743	pickler = cPickle
744
745	# Local immutable constants
746	#
747	comp_level = 1 # Compression level for zlib.
748	# comp_level = 1 works well.
749	textwidth1 = 16 # Text width of key fields in report forms.
750	#textwidth2 = 132 # Maximal width of textual representation of
751	textwidth2 = 300 # Maximal width of textual representation of
752	# arguments.
753	textwidth3 = 16 # Initial width of separation lines. Is modified.
754	textwidth4 = 50 # Text width in test_OK()
755	statsfile = '.cache_stat' # Basefilename for cached statistics.
756	# It will reside in the chosen cache directory.
757
758	file_types = ['Result', # File name extension for cached function results.
759	'Args', # File name extension for stored function args.
760	'Admin'] # File name extension for administrative info.
761
762	Reason_msg = ['OK', # Verbose reasons for recomputation
763	'No cached result',
764	'Dependencies have changed',
765	'Arguments have changed',
766	'Bytecode has changed',
767	'Recomputation was requested by caller',
768	'Cached file was unreadable']
769
770	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
771
772	def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression,
773	dependencies):
774	"""Determine whether cached result exists and return info.
775
776	USAGE:
777	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
778	CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
779	dependencies)
780
781	INPUT ARGUMENTS:
782	CD -- Cache Directory
783	FN -- Suggested cache file name
784	func -- Function object
785	args -- Tuple of arguments
786	kwargs -- Dictionary of keyword arguments
787	deps -- Dependencies time stamps
788	verbose -- Flag text output
789	compression -- Flag zlib compression
790	dependencies -- Given list of dependencies
791
792	OUTPUT ARGUMENTS:
793	T -- Cached result if present otherwise None
794	FN -- File name under which new results must be saved
795	Retrieved -- True if a valid cached result was found
796	reason -- 0: OK (if Retrieved),
797	1: No cached result,
798	2: Dependencies have changed,
799	3: Arguments have changed
800	4: Bytecode has changed
801	5: Recomputation was forced
802	6: Unreadable file
803	comptime -- Number of seconds it took to computed cachged result
804	loadtime -- Number of seconds it took to load cached result
805	compressed -- Flag (0,1) if cached results were compressed or not
806
807	DESCRIPTION:
808	Determine if cached result exists as follows:
809	Load in saved arguments and bytecode stored under hashed filename.
810	If they are identical to current arguments and bytecode and if dependencies
811	have not changed their time stamp, then return cached result.
812
813	Otherwise return filename under which new results should be cached.
814	Hash collisions are handled recursively by calling CacheLookup again with a
815	modified filename.
816	"""
817
818	import time, string, types
819
820	# Assess whether cached result exists - compressed or not.
821	#
822	if verbose:
823	print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
824	%(CD+FN, file_types[0], file_types[1], file_types[2])
825	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
826	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
827	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
828
829	if verbose is True and deps is not None:
830	print 'Caching: Dependencies are', deps.keys()
831
832	if not (argsfile and datafile and admfile) or \
833	not (compressed0 == compressed1 and compressed0 == compressed2):
834	# Cached result does not exist or files were compressed differently
835	#
836	# This will ensure that evaluation will take place unless all files are
837	# present.
838
839	reason = 1
840	return(None,FN,None,reason,None,None,None) #Recompute using same filename
841
842	compressed = compressed0 # Remember if compressed files were actually used
843	datafile.close()
844
845	# Retrieve arguments and adm. info
846	#
847	R, reason = myload(argsfile, compressed) # The original arguments
848	argsfile.close()
849
850	if reason > 0:
851	# Recompute using same filename
852	return(None, FN, None, reason, None, None, None)
853	else:
854	(argsref, kwargsref) = R
855
856	R, reason = myload(admfile, compressed)
857	admfile.close()
858
859	if reason > 0:
860	return(None,FN,None,reason,None,None,None) #Recompute using same filename
861
862
863	depsref = R[0] # Dependency statistics
864	comptime = R[1] # The computation time
865	coderef = R[2] # The byte code
866	funcname = R[3] # The function name
867
868	# Check if dependencies have changed
869	#
870	if dependencies and not compare(depsref, deps):
871	if verbose:
872	print 'MESSAGE (caching.py): Dependencies', dependencies, \
873	'have changed - recomputing'
874	# Don't use cached file - recompute
875	reason = 2
876	return(None, FN, None, reason, None, None, None)
877
878	# Get bytecode from func
879	#
880	bytecode = get_bytecode(func)
881
882	#print compare(argsref,args),
883	#print compare(kwargsref,kwargs),
884	#print compare(bytecode,coderef)
885
886	# Check if arguments or bytecode have changed
887	if compare(argsref, args) and compare(kwargsref, kwargs) and \
888	(not options['bytecode'] or compare(bytecode, coderef)):
889
890	# Arguments and dependencies match. Get cached results
891	T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
892	if reason > 0:
893	# Recompute using same FN
894	return(None, FN, None, reason, None, None, None)
895
896	Retrieved = 1
897	reason = 0
898
899	if verbose:
900	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
901
902	if loadtime >= comptime:
903	print 'WARNING (caching.py): Caching did not yield any gain.'
904	print ' Consider executing function ',
905	print '('+funcname+') without caching.'
906	else:
907
908	# Non matching arguments or bytecodes signify a hash-collision.
909	# This is resolved by recursive search of cache filenames
910	# until either a matching or an unused filename is found.
911	#
912	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
913	CacheLookup(CD, FN+'x', func, args, kwargs, deps,
914	verbose, compression, dependencies)
915
916	# DEBUGGING
917	# if not Retrieved:
918	# print 'Arguments did not match'
919	# else:
920	# print 'Match found !'
921
922	# The real reason is that args or bytecodes have changed.
923	# Not that the recursive seach has found an unused filename
924	if not Retrieved:
925	if not compare(bytecode, coderef):
926	reason = 4 # Bytecode has changed
927	else:
928	reason = 3 # Arguments have changed
929
930
931	return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
932
933	# -----------------------------------------------------------------------------
934
935	def clear_cache(CD,func=None, verbose=None):
936	"""Clear cache for func.
937
938	USAGE:
939	clear(CD, func, verbose)
940
941	ARGUMENTS:
942	CD -- Caching directory (required)
943	func -- Function object (default: None)
944	verbose -- Flag verbose output (default: None)
945
946	DESCRIPTION:
947
948	If func == None, clear everything,
949	otherwise clear only files pertaining to func.
950	"""
951
952	import os, re
953
954	if CD[-1] != os.sep:
955	CD = CD+os.sep
956
957	if verbose == None:
958	verbose = options['verbose']
959
960	# FIXME: Windows version needs to be tested
961
962	if func:
963	funcname = get_funcname(func)
964	if verbose:
965	print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
966
967	file_names = os.listdir(CD)
968	for file_name in file_names:
969	#RE = re.search('^' + funcname,file_name) #Inefficient
970	#if RE:
971	if file_name[:len(funcname)] == funcname:
972	if unix:
973	os.remove(CD+file_name)
974	else:
975	os.system('del '+CD+file_name)
976	# FIXME: os.remove doesn't work under windows
977	else:
978	file_names = os.listdir(CD)
979	if len(file_names) > 0:
980	if verbose:
981	print 'MESSAGE (caching.py): Remove the following files:'
982	for file_name in file_names:
983	print file_name
984
985	A = raw_input('Delete (Y/N)[N] ?')
986	else:
987	A = 'Y'
988
989	if A == 'Y' or A == 'y':
990	for file_name in file_names:
991	if unix:
992	os.remove(CD+file_name)
993	else:
994	os.system('del '+CD+file_name)
995	# FIXME: os.remove doesn't work under windows
996	#exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
997
998	# -----------------------------------------------------------------------------
999
1000	def DeleteOldFiles(CD,verbose=None):
1001	"""Remove expired files
1002
1003	USAGE:
1004	DeleteOldFiles(CD,verbose=None)
1005	"""
1006
1007	if verbose == None:
1008	verbose = options['verbose']
1009
1010	maxfiles = options['maxfiles']
1011
1012	# FIXME: Windows version
1013
1014	import os
1015	block = 1000 # How many files to delete per invokation
1016	Files = os.listdir(CD)
1017	numfiles = len(Files)
1018	if not unix: return # FIXME: Windows case ?
1019
1020	if numfiles > maxfiles:
1021	delfiles = numfiles-maxfiles+block
1022	if verbose:
1023	print 'Deleting '+`delfiles`+' expired files:'
1024	os.system('ls -lur '+CD+'* \| head -' + `delfiles`) # List them
1025	os.system('ls -ur '+CD+'* \| head -' + `delfiles` + ' \| xargs /bin/rm')
1026	# Delete them
1027	# FIXME: Replace this with os.listdir and os.remove
1028
1029	# -----------------------------------------------------------------------------
1030
1031	def save_args_to_cache(CD, FN, args, kwargs, compression):
1032	"""Save arguments to cache
1033
1034	USAGE:
1035	save_args_to_cache(CD,FN,args,kwargs,compression)
1036	"""
1037
1038	import time, os, sys, types
1039
1040	(argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1041
1042	if argsfile is None:
1043	msg = 'ERROR (caching): Could not open argsfile for writing: %s' %FN
1044	raise IOError, msg
1045
1046	mysave((args,kwargs),argsfile,compression) # Save args and kwargs to cache
1047	argsfile.close()
1048
1049	# Change access rights if possible
1050	#
1051	#if unix:
1052	# try:
1053	# exitcode=os.system('chmod 666 '+argsfile.name)
1054	# except:
1055	# pass
1056	#else:
1057	# pass # FIXME: Take care of access rights under Windows
1058
1059	return
1060
1061	# -----------------------------------------------------------------------------
1062
1063	def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1064	dependencies, compression):
1065	"""Save computed results T and admin info to cache
1066
1067	USAGE:
1068	save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1069	dependencies, compression)
1070	"""
1071
1072	import time, os, sys, types
1073
1074	(datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1075	(admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1076
1077	if not datafile:
1078	if verbose:
1079	print 'ERROR (caching): Could not open %s' %datafile.name
1080	raise IOError
1081
1082	if not admfile:
1083	if verbose:
1084	print 'ERROR (caching): Could not open %s' %admfile.name
1085	raise IOError
1086
1087	t0 = time.time()
1088
1089	mysave(T,datafile,compression) # Save data to cache
1090	datafile.close()
1091	#savetime = round(time.time()-t0,2)
1092	savetime = time.time()-t0
1093
1094	bytecode = get_bytecode(func) # Get bytecode from function object
1095	admtup = (deps, comptime, bytecode, funcname) # Gather admin info
1096
1097	mysave(admtup,admfile,compression) # Save admin info to cache
1098	admfile.close()
1099
1100	# Change access rights if possible
1101	#
1102	#if unix:
1103	# try:
1104	# exitcode=os.system('chmod 666 '+datafile.name)
1105	# exitcode=os.system('chmod 666 '+admfile.name)
1106	# except:
1107	# pass
1108	#else:
1109	# pass # FIXME: Take care of access rights under Windows
1110
1111	return(savetime)
1112
1113	# -----------------------------------------------------------------------------
1114
1115	def load_from_cache(CD,FN,compression):
1116	"""Load previously cached data from file FN
1117
1118	USAGE:
1119	load_from_cache(CD,FN,compression)
1120	"""
1121
1122	import time
1123
1124	(datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1125	t0 = time.time()
1126	T, reason = myload(datafile,compressed)
1127
1128	loadtime = time.time()-t0
1129	datafile.close()
1130
1131	return T, loadtime, compressed, reason
1132
1133	# -----------------------------------------------------------------------------
1134
1135	def myopen(FN,mode,compression=1):
1136	"""Open file FN using given mode
1137
1138	USAGE:
1139	myopen(FN,mode,compression=1)
1140
1141	ARGUMENTS:
1142	FN -- File name to be opened
1143	mode -- Open mode (as in open)
1144	compression -- Flag zlib compression
1145
1146	DESCRIPTION:
1147	if compression
1148	Attempt first to open FN + '.z'
1149	If this fails try to open FN
1150	else do the opposite
1151	Return file handle plus info about whether it was compressed or not.
1152	"""
1153
1154	import string
1155
1156	# Determine if file exists already (if writing was requested)
1157	# This info is only used to determine if access modes should be set
1158	#
1159	if 'w' in mode or 'a' in mode:
1160	try:
1161	file = open(FN+'.z','r')
1162	file.close()
1163	new_file = 0
1164	except:
1165	try:
1166	file = open(FN,'r')
1167	file.close()
1168	new_file = 0
1169	except:
1170	new_file = 1
1171	else:
1172	new_file = 0 #Assume it exists if mode was not 'w'
1173
1174
1175	compressed = 0
1176	if compression:
1177	try:
1178	file = open(FN+'.z',mode)
1179	compressed = 1
1180	except:
1181	try:
1182	file = open(FN,mode)
1183	except:
1184	file = None
1185	else:
1186	try:
1187	file = open(FN,mode)
1188	except:
1189	try:
1190	file = open(FN+'.z',mode)
1191	compressed = 1
1192	except:
1193	file = None
1194
1195	# Now set access rights if it is a new file
1196	#
1197	if file and new_file:
1198	if unix:
1199	exitcode=os.system('chmod 666 '+file.name)
1200	else:
1201	pass # FIXME: Take care of access rights under Windows
1202
1203	return(file,compressed)
1204
1205	# -----------------------------------------------------------------------------
1206
1207	def myload(file, compressed):
1208	"""Load data from file
1209
1210	USAGE:
1211	myload(file, compressed)
1212	"""
1213
1214	reason = 0
1215	try:
1216	if compressed:
1217	import zlib
1218
1219	RsC = file.read()
1220	try:
1221	Rs = zlib.decompress(RsC)
1222	except:
1223	# File "./caching.py", line 1032, in load_from_cache
1224	# T = myload(datafile,compressed)
1225	# File "./caching.py", line 1124, in myload
1226	# Rs = zlib.decompress(RsC)
1227	# zlib.error: Error -5 while decompressing data
1228	#print 'ERROR (caching): Could not decompress ', file.name
1229	#raise Exception
1230	reason = 6 # Unreadable file
1231	return None, reason
1232
1233
1234	del RsC # Free up some space
1235	R = pickler.loads(Rs)
1236	else:
1237	try:
1238	R = pickler.load(file)
1239	#except EOFError, e:
1240	except:
1241	#Catch e.g., file with 0 length or corrupted
1242	reason = 6 # Unreadable file
1243	return None, reason
1244
1245	except MemoryError:
1246	import sys
1247	if options['verbose']:
1248	print 'ERROR (caching): Out of memory while loading %s, aborting' \
1249	%(file.name)
1250
1251	# Raise the error again for now
1252	#
1253	raise MemoryError
1254
1255	return R, reason
1256
1257	# -----------------------------------------------------------------------------
1258
1259	def mysave(T,file,compression):
1260	"""Save data T to file
1261
1262	USAGE:
1263	mysave(T,file,compression)
1264
1265	"""
1266
1267	bin = options['bin']
1268
1269	if compression:
1270	try:
1271	import zlib
1272	except:
1273	print
1274	print '* Could not find zlib *'
1275	print '* Try to run caching with compression off *'
1276	print "* caching.set_option('compression', 0) *"
1277	raise Exception
1278
1279
1280	try:
1281	Ts = pickler.dumps(T, bin)
1282	except MemoryError:
1283	msg = '****WARNING (caching.py): Could not pickle data for compression.'
1284	msg += ' Try using compression = False'
1285	raise MemoryError, msg
1286	else:
1287	# Compressed pickling
1288	TsC = zlib.compress(Ts, comp_level)
1289	file.write(TsC)
1290	else:
1291	# Uncompressed pickling
1292	pickler.dump(T, file, bin)
1293
1294	# FIXME: This may not work on Windoze network drives.
1295	# The error msg is IOError: [Errno 22] Invalid argument
1296	# Testing with small files was OK, though.
1297	# I think this is an OS problem.
1298
1299	# Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1300
1301	# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1302	#
1303	#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1304	#
1305	#The following extract is taken from the site http://www.python.org:
1306	#
1307	#---------------------------------------------------------------------------------------------
1308	#exception IOError
1309	#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1310	#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1311	#---------------------------------------------------------------------------------------------
1312	#
1313	#The error code(s) that accompany exceptions are described at:
1314	#http://www.python.org/dev/doc/devel//lib/module-errno.html
1315	#
1316	#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1317	#
1318	#exceptions.IOError invalid argument Errno 22
1319
1320	#try:
1321	# pickler.dump(T,file,bin)
1322	#except IOError, e:
1323	# print e
1324	# msg = 'Could not store to %s, bin=%s' %(file, bin)
1325	# raise msg
1326
1327
1328	# -----------------------------------------------------------------------------
1329
1330
1331	def myhash(T, ids=None):
1332	"""Compute hashed integer from a range of inputs.
1333	If T is not hashable being e.g. a tuple T, myhash will recursively
1334	hash the values individually
1335
1336	USAGE:
1337	myhash(T)
1338
1339	ARGUMENTS:
1340	T -- Anything
1341	"""
1342
1343	from types import TupleType, ListType, DictType, InstanceType
1344
1345
1346	if type(T) in [TupleType, ListType, DictType, InstanceType]:
1347	# Keep track of unique id's to protect against infinite recursion
1348	if ids is None: ids = []
1349
1350	# Check if T has already been encountered
1351	i = id(T)
1352
1353	if i in ids:
1354	return 0 # T has been hashed already
1355	else:
1356	ids.append(i)
1357
1358
1359
1360	# Start hashing
1361
1362	# On some architectures None, False and True gets different hash values
1363	if T is None:
1364	return(-1)
1365	if T is False:
1366	return(0)
1367	if T is True:
1368	return(1)
1369
1370	# Get hash values for hashable entries
1371	if type(T) in [TupleType, ListType]:
1372	hvals = []
1373	for t in T:
1374	h = myhash(t, ids)
1375	hvals.append(h)
1376	val = hash(tuple(hvals))
1377	elif type(T) == DictType:
1378	# Make dictionary ordering unique
1379	I = T.items()
1380	I.sort()
1381	val = myhash(I, ids)
1382	elif isinstance(T, numpy.ndarray):
1383	# Use mean value for efficiency
1384	val = hash(numpy.average(T.ravel()))
1385	elif type(T) == InstanceType:
1386	val = myhash(T.__dict__, ids)
1387	else:
1388	try:
1389	val = hash(T)
1390	except:
1391	val = 1
1392
1393	return(val)
1394
1395
1396
1397	def compare(A, B, ids=None):
1398	"""Safe comparison of general objects
1399
1400	USAGE:
1401	compare(A,B)
1402
1403	DESCRIPTION:
1404	Return 1 if A and B they are identical, 0 otherwise
1405	"""
1406
1407	from types import TupleType, ListType, DictType, InstanceType
1408
1409	# Keep track of unique id's to protect against infinite recursion
1410	if ids is None: ids = {}
1411
1412	# Check if T has already been encountered
1413	iA = id(A)
1414	iB = id(B)
1415
1416	if ids.has_key((iA, iB)):
1417	# A and B have been compared already
1418	#print 'Found', (iA, iB), A, B
1419	return ids[(iA, iB)]
1420	else:
1421	ids[(iA, iB)] = True
1422
1423
1424	# Check if arguments are of same type
1425	if type(A) != type(B):
1426	return False
1427
1428
1429	# Compare recursively based on argument type
1430	if type(A) in [TupleType, ListType]:
1431	N = len(A)
1432	if len(B) != N:
1433	identical = False
1434	else:
1435	identical = True
1436	for i in range(N):
1437	if not compare(A[i], B[i], ids):
1438	identical = False; break
1439
1440	elif type(A) == DictType:
1441	if len(A) != len(B):
1442	identical = False
1443	else:
1444	# Make dictionary ordering unique
1445	a = A.items(); a.sort()
1446	b = B.items(); b.sort()
1447
1448	identical = compare(a, b, ids)
1449
1450	elif isinstance(A, numpy.ndarray):
1451	# Use element by element comparison
1452	identical = numpy.alltrue(A==B)
1453
1454	elif type(A) == InstanceType:
1455	# Take care of special case where elements are instances
1456	# Base comparison on attributes
1457	identical = compare(A.__dict__,
1458	B.__dict__,
1459	ids)
1460	else:
1461	# Fall back to general code
1462	try:
1463	identical = (A == B)
1464	except:
1465	import pickle
1466	# Use pickle to compare data
1467	# The native pickler must be used
1468	# since the faster cPickle does not
1469	# guarantee a unique translation
1470	try:
1471	identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
1472	except:
1473	identical = False
1474
1475	# Record result of comparison and return
1476	ids[(iA, iB)] = identical
1477
1478	return(identical)
1479
1480
1481	# -----------------------------------------------------------------------------
1482
1483	def nospace(s):
1484	"""Replace spaces in string s with underscores
1485
1486	USAGE:
1487	nospace(s)
1488
1489	ARGUMENTS:
1490	s -- string
1491	"""
1492
1493	import string
1494
1495	newstr = ''
1496	for i in range(len(s)):
1497	if s[i] == ' ':
1498	newstr = newstr+'_'
1499	else:
1500	newstr = newstr+s[i]
1501
1502	return(newstr)
1503
1504	# -----------------------------------------------------------------------------
1505
1506	def get_funcname(func):
1507	"""Retrieve name of function object func (depending on its type)
1508
1509	USAGE:
1510	get_funcname(func)
1511	"""
1512
1513	import types, string
1514
1515	if type(func) == types.FunctionType:
1516	funcname = func.func_name
1517	elif type(func) == types.BuiltinFunctionType:
1518	funcname = func.__name__
1519	else:
1520	tab = string.maketrans("<>'"," ")
1521	tmp = string.translate(`func`,tab)
1522	tmp = string.split(tmp)
1523	funcname = string.join(tmp)
1524
1525	# Truncate memory address as in
1526	# class __main__.Dummy at 0x00A915D0
1527	index = funcname.find('at 0x')
1528	if index >= 0:
1529	funcname = funcname[:index+5] # Keep info that there is an address
1530
1531	funcname = nospace(funcname)
1532	return(funcname)
1533
1534	# -----------------------------------------------------------------------------
1535
1536	def get_bytecode(func):
1537	""" Get bytecode from function object.
1538
1539	USAGE:
1540	get_bytecode(func)
1541	"""
1542
1543	import types
1544
1545	if type(func) == types.FunctionType:
1546	bytecode = func.func_code.co_code
1547	consts = func.func_code.co_consts
1548	argcount = func.func_code.co_argcount
1549	defaults = func.func_defaults
1550	elif type(func) == types.MethodType:
1551	bytecode = func.im_func.func_code.co_code
1552	consts = func.im_func.func_code.co_consts
1553	argcount = func.im_func.func_code.co_argcount
1554	defaults = func.im_func.func_defaults
1555	else:
1556	#raise Exception #Test only
1557	bytecode = None #Built-in functions are assumed not to change
1558	consts = 0
1559	argcount = 0
1560	defaults = 0
1561
1562	return (bytecode, consts, argcount, defaults)
1563
1564	# -----------------------------------------------------------------------------
1565
1566	def get_depstats(dependencies):
1567	""" Build dictionary of dependency files and their size, mod. time and ctime.
1568
1569	USAGE:
1570	get_depstats(dependencies):
1571	"""
1572
1573	import types
1574
1575	#print 'Caching DEBUG: Dependencies are', dependencies
1576	d = {}
1577	if dependencies:
1578
1579	#Expand any wildcards
1580	import glob
1581	expanded_dependencies = []
1582	for FN in dependencies:
1583	expanded_FN = glob.glob(FN)
1584
1585	if expanded_FN == []:
1586	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1587	raise Exception, errmsg
1588
1589	expanded_dependencies += expanded_FN
1590
1591
1592	for FN in expanded_dependencies:
1593	if not type(FN) == types.StringType:
1594	errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1595	errmsg += ' Dependency given: %s' %FN
1596	raise Exception, errmsg
1597	if not os.access(FN,os.F_OK):
1598	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1599	raise Exception, errmsg
1600	(size,atime,mtime,ctime) = filestat(FN)
1601
1602	# We don't use atime because that would cause recomputation every time.
1603	# We don't use ctime because that is irrelevant and confusing for users.
1604	d.update({FN : (size,mtime)})
1605
1606	return(d)
1607
1608	# -----------------------------------------------------------------------------
1609
1610	def filestat(FN):
1611	"""A safe wrapper using os.stat to get basic file statistics
1612	The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1613
1614	USAGE:
1615	filestat(FN)
1616
1617	DESCRIPTION:
1618	Must compile Python with
1619	CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1620	configure
1621	as given in section 8.1.1 Large File Support in the Libray Reference
1622	"""
1623
1624	import os, time
1625
1626	try:
1627	stats = os.stat(FN)
1628	size = stats[6]
1629	atime = stats[7]
1630	mtime = stats[8]
1631	ctime = stats[9]
1632	except:
1633
1634	# Hack to get the results anyway (works only on Unix at the moment)
1635	#
1636	print 'Hack to get os.stat when files are too large'
1637
1638	if unix:
1639	tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1640	# Unique filename, FIXME: Use random number
1641
1642	# Get size and access time (atime)
1643	#
1644	exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1645	(size,atime) = get_lsline(tmp)
1646
1647	# Get size and modification time (mtime)
1648	#
1649	exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1650	(size,mtime) = get_lsline(tmp)
1651
1652	# Get size and ctime
1653	#
1654	exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1655	(size,ctime) = get_lsline(tmp)
1656
1657	try:
1658	exitcode=os.system('rm '+tmp)
1659	# FIXME: Gives error if file doesn't exist
1660	except:
1661	pass
1662	else:
1663	pass
1664	raise Exception # FIXME: Windows case
1665
1666	return(long(size),atime,mtime,ctime)
1667
1668	# -----------------------------------------------------------------------------
1669
1670	def get_lsline(FN):
1671	"""get size and time for filename
1672
1673	USAGE:
1674	get_lsline(file_name)
1675
1676	DESCRIPTION:
1677	Read in one line 'ls -la' item from file (generated by filestat) and
1678	convert time to seconds since epoch. Return file size and time.
1679	"""
1680
1681	import string, time
1682
1683	f = open(FN,'r')
1684	info = f.read()
1685	info = string.split(info)
1686
1687	size = info[4]
1688	week = info[5]
1689	mon = info[6]
1690	day = info[7]
1691	hour = info[8]
1692	year = info[9]
1693
1694	str = week+' '+mon+' '+day+' '+hour+' '+year
1695	timetup = time.strptime(str)
1696	t = time.mktime(timetup)
1697	return(size, t)
1698
1699	# -----------------------------------------------------------------------------
1700
1701	def checkdir(CD,verbose=None):
1702	"""Check or create caching directory
1703
1704	USAGE:
1705	checkdir(CD,verbose):
1706
1707	ARGUMENTS:
1708	CD -- Directory
1709	verbose -- Flag verbose output (default: None)
1710
1711	DESCRIPTION:
1712	If CD does not exist it will be created if possible
1713	"""
1714
1715	import os
1716	import os.path
1717
1718	if CD[-1] != os.sep:
1719	CD = CD + os.sep # Add separator for directories
1720
1721	CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1722	if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1723	try:
1724	exitcode=os.mkdir(CD)
1725
1726	# Change access rights if possible
1727	#
1728	if unix:
1729	exitcode=os.system('chmod 777 '+CD)
1730	else:
1731	pass # FIXME: What about acces rights under Windows?
1732	if verbose: print 'MESSAGE: Directory', CD, 'created.'
1733	except:
1734	print 'WARNING: Directory', CD, 'could not be created.'
1735	if unix:
1736	CD = '/tmp/'
1737	else:
1738	CD = 'C:'
1739	print 'Using directory %s instead' %CD
1740
1741	return(CD)
1742
1743	#==============================================================================
1744	# Statistics
1745	#==============================================================================
1746
1747	def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
1748	compression):
1749	"""Add stats entry
1750
1751	USAGE:
1752	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1753
1754	DESCRIPTION:
1755	Make one entry in the stats file about one cache hit recording time saved
1756	and other statistics. The data are used by the function cachestat.
1757	"""
1758
1759	import os, time
1760
1761	try:
1762	TimeTuple = time.localtime(time.time())
1763	extension = time.strftime('%b%Y',TimeTuple)
1764	SFN = CD+statsfile+'.'+extension
1765	#statfile = open(SFN,'a')
1766	(statfile, dummy) = myopen(SFN,'a',compression=0)
1767
1768	# Change access rights if possible
1769	#
1770	#if unix:
1771	# try:
1772	# exitcode=os.system('chmod 666 '+SFN)
1773	# except:
1774	# pass
1775	except:
1776	print 'Warning: Stat file could not be opened'
1777
1778	try:
1779	if os.environ.has_key('USER'):
1780	user = os.environ['USER']
1781	else:
1782	user = 'Nobody'
1783
1784	date = time.asctime(TimeTuple)
1785
1786	if Retrieved:
1787	hit = '1'
1788	else:
1789	hit = '0'
1790
1791	# Get size of result file
1792	#
1793	if compression:
1794	stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1795	else:
1796	stats = os.stat(CD+FN+'_'+file_types[0])
1797
1798	if stats:
1799	size = stats[6]
1800	else:
1801	size = -1 # Error condition, but don't crash. This is just statistics
1802
1803	# Build entry
1804	#
1805	entry = date + ',' +\
1806	user + ',' +\
1807	FN + ',' +\
1808	str(int(size)) + ',' +\
1809	str(compression) + ',' +\
1810	hit + ',' +\
1811	str(reason) + ',' +\
1812	str(round(comptime,4)) + ',' +\
1813	str(round(loadtime,4)) +\
1814	CR
1815
1816	statfile.write(entry)
1817	statfile.close()
1818	except:
1819	print 'Warning: Writing of stat file failed'
1820
1821	# -----------------------------------------------------------------------------
1822
1823	# FIXME: should take cachedir as an optional arg
1824	#
1825	def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1826	""" List caching statistics.
1827
1828	USAGE:
1829	__cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1830
1831	Generate statistics of caching efficiency.
1832	The parameter sortidx determines by what field lists are sorted.
1833	If the optional keyword period is set to -1,
1834	all available caching history is used.
1835	If it is 0 only the current month is used.
1836	Future versions will include more than one month....
1837	OMN 20/8/2000
1838	"""
1839
1840	import os
1841	import os.path
1842	from string import split, rstrip, find
1843	from time import strptime, localtime, strftime, mktime, ctime
1844
1845	# sortidx = 4 # Index into Fields[1:]. What to sort by.
1846
1847	Fields = ['Name', 'Hits', 'Exec(s)', \
1848	'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1849	Widths = [25,7,9,9,9,9,13]
1850	#Types = ['s','d','d','d','d','.2f','d']
1851	Types = ['s','d','.2f','.2f','.2f','.2f','d']
1852
1853	Dictnames = ['Function', 'User']
1854
1855	if not cachedir:
1856	cachedir = checkdir(options['cachedir'])
1857
1858	SD = os.path.expanduser(cachedir) # Expand ~ or ~user in pathname
1859
1860	if period == -1: # Take all available stats
1861	SFILENAME = statsfile
1862	else: # Only stats from current month
1863	# MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1864	TimeTuple = localtime(time())
1865	extension = strftime('%b%Y',TimeTuple)
1866	SFILENAME = statsfile+'.'+extension
1867
1868	DIRLIST = os.listdir(SD)
1869	SF = []
1870	for FN in DIRLIST:
1871	if find(FN,SFILENAME) >= 0:
1872	SF.append(FN)
1873
1874	blocksize = 15000000
1875	total_read = 0
1876	total_hits = 0
1877	total_discarded = 0
1878	firstday = mktime(strptime('2030','%Y'))
1879	# FIXME: strptime don't exist in WINDOWS ?
1880	lastday = 0
1881
1882	FuncDict = {}
1883	UserDict = {}
1884	for FN in SF:
1885	input = open(SD+FN,'r')
1886	print 'Reading file ', SD+FN
1887
1888	while 1:
1889	A = input.readlines(blocksize)
1890	if len(A) == 0: break
1891	total_read = total_read + len(A)
1892	for record in A:
1893	record = tuple(split(rstrip(record),','))
1894	#print record, len(record)
1895
1896	if len(record) == 9:
1897	timestamp = record[0]
1898
1899	try:
1900	t = mktime(strptime(timestamp))
1901	except:
1902	total_discarded = total_discarded + 1
1903	continue
1904
1905	if t > lastday:
1906	lastday = t
1907	if t < firstday:
1908	firstday = t
1909
1910	user = record[1]
1911	func = record[2]
1912
1913	# Strip hash-stamp off
1914	#
1915	i = find(func,'[')
1916	func = func[:i]
1917
1918	size = float(record[3])
1919
1920	# Compression kepword can be Boolean
1921	if record[4] in ['True', '1']:
1922	compression = 1
1923	elif record[4] in ['False', '0']:
1924	compression = 0
1925	else:
1926	print 'Unknown value of compression', record[4]
1927	print record
1928	total_discarded = total_discarded + 1
1929	continue
1930
1931	#compression = int(record[4]) # Can be Boolean
1932	hit = int(record[5])
1933	reason = int(record[6]) # Not used here
1934	cputime = float(record[7])
1935	loadtime = float(record[8])
1936
1937	if hit:
1938	total_hits = total_hits + 1
1939	saving = cputime-loadtime
1940
1941	if cputime != 0:
1942	rel_saving = round(100.0*saving/cputime,2)
1943	else:
1944	#rel_saving = round(1.0*saving,2)
1945	rel_saving = 100.0 - round(1.0*saving,2) # A bit of a hack
1946
1947	info = [1,cputime,loadtime,saving,rel_saving,size]
1948
1949	UpdateDict(UserDict,user,info)
1950	UpdateDict(FuncDict,func,info)
1951	else:
1952	pass #Stats on recomputations and their reasons could go in here
1953
1954	else:
1955	#print 'Record discarded'
1956	#print record
1957	total_discarded = total_discarded + 1
1958
1959	input.close()
1960
1961	# Compute averages of all sums and write list
1962	#
1963
1964	if total_read == 0:
1965	printline(Widths,'=')
1966	print 'CACHING STATISTICS: No valid records read'
1967	printline(Widths,'=')
1968	return
1969
1970	print
1971	printline(Widths,'=')
1972	print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
1973	printline(Widths,'=')
1974	#print ' Period:', ctime(firstday), 'to', ctime(lastday)
1975	print ' Total number of valid records', total_read
1976	print ' Total number of discarded records', total_discarded
1977	print ' Total number of hits', total_hits
1978	print
1979
1980	print ' Fields', Fields[2:], 'are averaged over number of hits'
1981	print ' Time is measured in seconds and size in bytes'
1982	print ' Tables are sorted by', Fields[1:][sortidx]
1983
1984	# printline(Widths,'-')
1985
1986	if showuser:
1987	Dictionaries = [FuncDict, UserDict]
1988	else:
1989	Dictionaries = [FuncDict]
1990
1991	i = 0
1992	for Dict in Dictionaries:
1993	for key in Dict.keys():
1994	rec = Dict[key]
1995	for n in range(len(rec)):
1996	if n > 0:
1997	rec[n] = round(1.0*rec[n]/rec[0],2)
1998	Dict[key] = rec
1999
2000	# Sort and output
2001	#
2002	keylist = SortDict(Dict,sortidx)
2003
2004	# Write Header
2005	#
2006	print
2007	#print Dictnames[i], 'statistics:'; i=i+1
2008	printline(Widths,'-')
2009	n = 0
2010	for s in Fields:
2011	if s == Fields[0]: # Left justify
2012	s = Dictnames[i] + ' ' + s; i=i+1
2013	exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
2014	else:
2015	exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
2016	print
2017	printline(Widths,'-')
2018
2019	# Output Values
2020	#
2021	for key in keylist:
2022	rec = Dict[key]
2023	n = 0
2024	if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
2025	exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
2026	for val in rec:
2027	exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
2028	print
2029	print
2030
2031	#==============================================================================
2032	# Auxiliary stats functions
2033	#==============================================================================
2034
2035	def UpdateDict(Dict,key,info):
2036	"""Update dictionary by adding new values to existing.
2037
2038	USAGE:
2039	UpdateDict(Dict,key,info)
2040	"""
2041
2042	if Dict.has_key(key):
2043	dinfo = Dict[key]
2044	for n in range(len(dinfo)):
2045	dinfo[n] = info[n] + dinfo[n]
2046	else:
2047	dinfo = info[:] # Make a copy of info list
2048
2049	Dict[key] = dinfo
2050	return Dict
2051
2052	# -----------------------------------------------------------------------------
2053
2054	def SortDict(Dict,sortidx=0):
2055	"""Sort dictionary
2056
2057	USAGE:
2058	SortDict(Dict,sortidx):
2059
2060	DESCRIPTION:
2061	Sort dictionary of lists according field number 'sortidx'
2062	"""
2063
2064	import types
2065
2066	sortlist = []
2067	keylist = Dict.keys()
2068	for key in keylist:
2069	rec = Dict[key]
2070	if not type(rec) in [types.ListType, types.TupleType]:
2071	rec = [rec]
2072
2073	if sortidx > len(rec)-1:
2074	if options['verbose']:
2075	print 'ERROR: Sorting index to large, sortidx = ', sortidx
2076	raise IndexError
2077
2078	val = rec[sortidx]
2079	sortlist.append(val)
2080
2081	A = map(None,sortlist,keylist)
2082	A.sort()
2083	keylist = map(lambda x: x[1], A) # keylist sorted by sortidx
2084
2085	return(keylist)
2086
2087	# -----------------------------------------------------------------------------
2088
2089	def printline(Widths,char):
2090	"""Print textline in fixed field.
2091
2092	USAGE:
2093	printline(Widths,char)
2094	"""
2095
2096	s = ''
2097	for n in range(len(Widths)):
2098	s = s+Widths[n]*char
2099	if n > 0:
2100	s = s+char
2101
2102	print s
2103
2104	#==============================================================================
2105	# Messages
2106	#==============================================================================
2107
2108	def msg1(funcname,args,kwargs,reason):
2109	"""Message 1
2110
2111	USAGE:
2112	msg1(funcname,args,kwargs,reason):
2113	"""
2114
2115	import string
2116	#print 'MESSAGE (caching.py): Evaluating function', funcname,
2117
2118	print_header_box('Evaluating function %s' %funcname)
2119
2120	msg7(args, kwargs)
2121	msg8(reason)
2122
2123	print_footer()
2124
2125	#
2126	# Old message
2127	#
2128	#args_present = 0
2129	#if args:
2130	# if len(args) == 1:
2131	# print 'with argument', mkargstr(args[0], textwidth2),
2132	# else:
2133	# print 'with arguments', mkargstr(args, textwidth2),
2134	# args_present = 1
2135	#
2136	#if kwargs:
2137	# if args_present:
2138	# word = 'and'
2139	# else:
2140	# word = 'with'
2141	#
2142	# if len(kwargs) == 1:
2143	# print word + ' keyword argument', mkargstr(kwargs, textwidth2)
2144	# else:
2145	# print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
2146	# args_present = 1
2147	#else:
2148	# print # Newline when no keyword args present
2149	#
2150	#if not args_present:
2151	# print '', # Default if no args or kwargs present
2152
2153
2154
2155	# -----------------------------------------------------------------------------
2156
2157	def msg2(funcname,args,kwargs,comptime,reason):
2158	"""Message 2
2159
2160	USAGE:
2161	msg2(funcname,args,kwargs,comptime,reason)
2162	"""
2163
2164	import string
2165
2166	#try:
2167	# R = Reason_msg[reason]
2168	#except:
2169	# R = 'Unknown reason'
2170
2171	#print_header_box('Caching statistics (storing) - %s' %R)
2172	print_header_box('Caching statistics (storing)')
2173
2174	msg6(funcname,args,kwargs)
2175	msg8(reason)
2176
2177	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2178
2179	# -----------------------------------------------------------------------------
2180
2181	def msg3(savetime, CD, FN, deps,compression):
2182	"""Message 3
2183
2184	USAGE:
2185	msg3(savetime, CD, FN, deps,compression)
2186	"""
2187
2188	import string
2189	print string.ljust('\| Loading time:', textwidth1) + str(round(savetime,2)) + \
2190	' seconds (estimated)'
2191	msg5(CD,FN,deps,compression)
2192
2193	# -----------------------------------------------------------------------------
2194
2195	def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
2196	"""Message 4
2197
2198	USAGE:
2199	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
2200	"""
2201
2202	import string
2203
2204	print_header_box('Caching statistics (retrieving)')
2205
2206	msg6(funcname,args,kwargs)
2207	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2208	print string.ljust('\| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
2209	print string.ljust('\| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
2210	' seconds'
2211	msg5(CD,FN,deps,compression)
2212
2213	# -----------------------------------------------------------------------------
2214
2215	def msg5(CD,FN,deps,compression):
2216	"""Message 5
2217
2218	USAGE:
2219	msg5(CD,FN,deps,compression)
2220
2221	DESCRIPTION:
2222	Print dependency stats. Used by msg3 and msg4
2223	"""
2224
2225	import os, time, string
2226
2227	print '\|'
2228	print string.ljust('\| Caching dir: ', textwidth1) + CD
2229
2230	if compression:
2231	suffix = '.z'
2232	bytetext = 'bytes, compressed'
2233	else:
2234	suffix = ''
2235	bytetext = 'bytes'
2236
2237	for file_type in file_types:
2238	file_name = FN + '_' + file_type + suffix
2239	print string.ljust('\| ' + file_type + ' file: ', textwidth1) + file_name,
2240	stats = os.stat(CD+file_name)
2241	print '('+ str(stats[6]) + ' ' + bytetext + ')'
2242
2243	print '\|'
2244	if len(deps) > 0:
2245	print '\| Dependencies: '
2246	dependencies = deps.keys()
2247	dlist = []; maxd = 0
2248	tlist = []; maxt = 0
2249	slist = []; maxs = 0
2250	for d in dependencies:
2251	stats = deps[d]
2252	t = time.ctime(stats[1])
2253	s = str(stats[0])
2254	#if s[-1] == 'L':
2255	# s = s[:-1] # Strip rightmost 'long integer' L off.
2256	# # FIXME: Unnecessary in versions later than 1.5.2
2257
2258	if len(d) > maxd: maxd = len(d)
2259	if len(t) > maxt: maxt = len(t)
2260	if len(s) > maxs: maxs = len(s)
2261	dlist.append(d)
2262	tlist.append(t)
2263	slist.append(s)
2264
2265	for n in range(len(dlist)):
2266	d = string.ljust(dlist[n]+':', maxd+1)
2267	t = string.ljust(tlist[n], maxt)
2268	s = string.rjust(slist[n], maxs)
2269
2270	print '\| ', d, t, ' ', s, 'bytes'
2271	else:
2272	print '\| No dependencies'
2273	print_footer()
2274
2275	# -----------------------------------------------------------------------------
2276
2277	def msg6(funcname,args,kwargs):
2278	"""Message 6
2279
2280	USAGE:
2281	msg6(funcname,args,kwargs)
2282	"""
2283
2284	import string
2285	print string.ljust('\| Function:', textwidth1) + funcname
2286
2287	msg7(args, kwargs)
2288
2289	# -----------------------------------------------------------------------------
2290
2291	def msg7(args, kwargs):
2292	"""Message 7
2293
2294	USAGE:
2295	msg7(args, kwargs):
2296	"""
2297
2298	import string
2299
2300	args_present = 0
2301	if args:
2302	if len(args) == 1:
2303	print string.ljust('\| Argument:', textwidth1) + mkargstr(args[0], \
2304	textwidth2)
2305	else:
2306	print string.ljust('\| Arguments:', textwidth1) + \
2307	mkargstr(args, textwidth2)
2308	args_present = 1
2309
2310	if kwargs:
2311	if len(kwargs) == 1:
2312	print string.ljust('\| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
2313	textwidth2)
2314	else:
2315	print string.ljust('\| Keyword Args:', textwidth1) + \
2316	mkargstr(kwargs, textwidth2)
2317	args_present = 1
2318
2319	if not args_present:
2320	print '\| No arguments' # Default if no args or kwargs present
2321
2322	# -----------------------------------------------------------------------------
2323
2324	def msg8(reason):
2325	"""Message 8
2326
2327	USAGE:
2328	msg8(reason):
2329	"""
2330
2331	import string
2332
2333	try:
2334	R = Reason_msg[reason]
2335	except:
2336	R = 'Unknown'
2337
2338	print string.ljust('\| Reason:', textwidth1) + R
2339
2340	# -----------------------------------------------------------------------------
2341
2342	def print_header_box(line):
2343	"""Print line in a nice box.
2344
2345	USAGE:
2346	print_header_box(line)
2347
2348	"""
2349	global textwidth3
2350
2351	import time
2352
2353	time_stamp = time.ctime(time.time())
2354	line = time_stamp + '. ' + line
2355
2356	N = len(line) + 1
2357	s = '+' + '-'*N + CR
2358
2359	print s + '\| ' + line + CR + s,
2360
2361	textwidth3 = N
2362
2363	# -----------------------------------------------------------------------------
2364
2365	def print_footer():
2366	"""Print line same width as that of print_header_box.
2367	"""
2368
2369	N = textwidth3
2370	s = '+' + '-'*N + CR
2371
2372	print s
2373
2374	# -----------------------------------------------------------------------------
2375
2376	def mkargstr(args, textwidth, argstr = '', level=0):
2377	""" Generate a string containing first textwidth characters of arguments.
2378
2379	USAGE:
2380	mkargstr(args, textwidth, argstr = '', level=0)
2381
2382	DESCRIPTION:
2383	Exactly the same as str(args) possibly followed by truncation,
2384	but faster if args is huge.
2385	"""
2386
2387	import types
2388
2389	if level > 10:
2390	# Protect against circular structures
2391	return '...'
2392
2393	WasTruncated = 0
2394
2395	if not type(args) in [types.TupleType, types.ListType, types.DictType]:
2396	if type(args) == types.StringType:
2397	argstr = argstr + "'"+str(args)+"'"
2398	else:
2399	# Truncate large arrays before using str()
2400	if isinstance(args, numpy.ndarray):
2401	# if len(args.ravel()) > textwidth:
2402	# Changed by Duncan and Nick 21/2/07 .ravel() has problems with
2403	# non-contigous arrays and ravel is equal to .ravel() except it
2404	# can work with non-contiguous arrays
2405	if len(numpy.ravel(args)) > textwidth:
2406	args = 'Array: ' + str(args.shape)
2407
2408	argstr = argstr + str(args)
2409	else:
2410	if type(args) == types.DictType:
2411	argstr = argstr + "{"
2412	for key in args.keys():
2413	argstr = argstr + mkargstr(key, textwidth, level=level+1) + ": " + \
2414	mkargstr(args[key], textwidth, level=level+1) + ", "
2415	if len(argstr) > textwidth:
2416	WasTruncated = 1
2417	break
2418	argstr = argstr[:-2] # Strip off trailing comma
2419	argstr = argstr + "}"
2420
2421	else:
2422	if type(args) == types.TupleType:
2423	lc = '('
2424	rc = ')'
2425	else:
2426	lc = '['
2427	rc = ']'
2428	argstr = argstr + lc
2429	for arg in args:
2430	argstr = argstr + mkargstr(arg, textwidth, level=level+1) + ', '
2431	if len(argstr) > textwidth:
2432	WasTruncated = 1
2433	break
2434
2435	# Strip off trailing comma and space unless singleton tuple
2436	#
2437	if type(args) == types.TupleType and len(args) == 1:
2438	argstr = argstr[:-1]
2439	else:
2440	argstr = argstr[:-2]
2441	argstr = argstr + rc
2442
2443	if len(argstr) > textwidth:
2444	WasTruncated = 1
2445
2446	if WasTruncated:
2447	argstr = argstr[:textwidth]+'...'
2448	return(argstr)
2449
2450	# -----------------------------------------------------------------------------
2451
2452	def test_OK(msg):
2453	"""Print OK msg if test is OK.
2454
2455	USAGE
2456	test_OK(message)
2457	"""
2458
2459	import string
2460
2461	print string.ljust(msg, textwidth4) + ' - OK'
2462
2463	#raise StandardError
2464
2465	# -----------------------------------------------------------------------------
2466
2467	def test_error(msg):
2468	"""Print error if test fails.
2469
2470	USAGE
2471	test_error(message)
2472	"""
2473
2474	print 'ERROR (caching.test): %s' %msg
2475	print 'Please send this code example and output to '
2476	print 'Ole.Nielsen@anu.edu.au'
2477	print
2478	print
2479
2480	#import sys
2481	#sys.exit()
2482	raise StandardError
2483
2484	#-------------------------------------------------------------
2485	if __name__ == "__main__":
2486	pass

Note: See TracBrowser for help on using the repository browser.

Download in other formats: