Context Navigation

source: inundation/caching/caching.py @ 2920

Last change on this file since 2920 was 2043, checked in by ole, 19 years ago
Investigated file write error on networked drives
File size: 68.0 KB

Line
1	# =============================================================================
2	# caching.py - Supervised caching of function results.
3	# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
4	# Australian National University (1999-2003)
5	# Geoscience Australia (2003-present)
6	#
7	# This program is free software; you can redistribute it and/or modify
8	# it under the terms of the GNU General Public License as published by
9	# the Free Software Foundation; either version 2 of the License, or
10	# (at your option) any later version.
11	#
12	# This program is distributed in the hope that it will be useful,
13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	# GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
16	# for more details.
17	#
18	# You should have received a copy of the GNU General Public License
19	# along with this program; if not, write to the Free Software
20	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
21	#
22	#
23	# Contact address: Ole.Nielsen@ga.gov.au
24	#
25	# Version 1.5.6 February 2002
26	# =============================================================================
27
28	"""Module caching.py - Supervised caching of function results.
29
30	Public functions:
31
32	cache(func,args) -- Cache values returned from func given args.
33	cachestat() -- Reports statistics about cache hits and time saved.
34	test() -- Conducts a basic test of the caching functionality.
35
36	See doc strings of individual functions for detailed documentation.
37	"""
38
39	# -----------------------------------------------------------------------------
40	# Initialisation code
41
42	# Determine platform
43	#
44	import os
45	if os.name in ['nt', 'dos', 'win32', 'what else?']:
46	unix = 0
47	else:
48	unix = 1
49
50	# Make default caching directory name
51	#
52	if unix:
53	homedir = '~'
54	CR = '\n'
55	else:
56	homedir = 'c:'
57	CR = '\r\n' #FIXME: Not tested under windows
58
59	cachedir = homedir + os.sep + '.python_cache' + os.sep
60
61	# -----------------------------------------------------------------------------
62	# Options directory with default values - to be set by user
63	#
64
65	options = {
66	'cachedir': cachedir, # Default cache directory
67	'maxfiles': 1000000, # Maximum number of cached files
68	'savestat': 1, # Log caching info to stats file
69	'verbose': 1, # Write messages to standard output
70	'bin': 1, # Use binary format (more efficient)
71	'compression': 1, # Use zlib compression
72	'bytecode': 0, # Recompute if bytecode has changed
73	'expire': 0 # Automatically remove files that have been accessed
74	# least recently
75	}
76
77	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
78
79	def set_option(key, value):
80	"""Function to set values in the options directory.
81
82	USAGE:
83	set_option(key, value)
84
85	ARGUMENTS:
86	key -- Key in options dictionary. (Required)
87	value -- New value for key. (Required)
88
89	DESCRIPTION:
90	Function to set values in the options directory.
91	Raises an exception if key is not in options.
92	"""
93
94	if options.has_key(key):
95	options[key] = value
96	else:
97	raise KeyError(key) # Key not found, raise an exception
98
99	# -----------------------------------------------------------------------------
100	# Function cache - the main routine
101
102	def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
103	verbose=None, compression=None, evaluate=0, test=0, clear=0,
104	return_filename=0):
105	"""Supervised caching of function results.
106
107	USAGE:
108	result = cache(func, args, kwargs, dependencies, cachedir, verbose,
109	compression, evaluate, test, return_filename)
110
111	ARGUMENTS:
112	func -- Function object (Required)
113	args -- Arguments to func (Default: ())
114	kwargs -- Keyword arguments to func (Default: {})
115	dependencies -- Filenames that func depends on (Default: None)
116	cachedir -- Directory for cache files (Default: options['cachedir'])
117	verbose -- Flag verbose output to stdout
118	(Default: options['verbose'])
119	compression -- Flag zlib compression (Default: options['compression'])
120	evaluate -- Flag forced evaluation of func (Default: 0)
121	test -- Flag test for cached results (Default: 0)
122	clear -- Flag delete cached results (Default: 0)
123	return_filename -- Flag return of cache filename (Default: 0)
124
125	DESCRIPTION:
126	A Python function call of the form
127
128	result = func(arg1,...,argn)
129
130	can be replaced by
131
132	from caching import cache
133	result = cache(func,(arg1,...,argn))
134
135	The latter form returns the same output as the former but reuses cached
136	results if the function has been computed previously in the same context.
137	'result' and the arguments can be simple types, tuples, list, dictionaries or
138	objects, but not unhashable types such as functions or open file objects.
139	The function 'func' may be a member function of an object or a module.
140
141	This type of caching is particularly useful for computationally intensive
142	functions with few frequently used combinations of input arguments. Note that
143	if the inputs or output are very large caching might not save time because
144	disc access may dominate the execution time.
145
146	If the function definition changes after a result has been cached it will be
147	detected by examining the functions bytecode (co_code, co_consts,
148	func_defualts, co_argcount) and it will be recomputed.
149
150	LIMITATIONS:
151	1 Caching uses the apply function and will work with anything that can be
152	pickled, so any limitation in apply or pickle extends to caching.
153	2 A function to be cached should not depend on global variables
154	as wrong results may occur if globals are changed after a result has
155	been cached.
156
157	-----------------------------------------------------------------------------
158	Additional functionality:
159
160	Keyword args
161	Keyword arguments (kwargs) can be added as a dictionary of keyword: value
162	pairs, following the syntax of the built-in function apply().
163	A Python function call of the form
164
165	result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)
166
167	is then cached as follows
168
169	from caching import cache
170	result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
171
172	The default value of kwargs is {}
173
174	Explicit dependencies:
175	The call
176	cache(func,(arg1,...,argn),dependencies = <list of filenames>)
177	Checks the size, creation time and modification time of each listed file.
178	If any file has changed the function is recomputed and the results stored
179	again.
180
181	Specify caching directory:
182	The call
183	cache(func,(arg1,...,argn), cachedir = <cachedir>)
184	designates <cachedir> where cached data are stored. Use ~ to indicate users
185	home directory - not $HOME. The default is ~/.python_cache on a UNIX
186	platform and c:/.python_cache on a Win platform.
187
188	Silent operation:
189	The call
190	cache(func,(arg1,...,argn),verbose=0)
191	suppresses messages to standard output.
192
193	Compression:
194	The call
195	cache(func,(arg1,...,argn),compression=0)
196	disables compression. (Default: compression=1). If the requested compressed
197	or uncompressed file is not there, it'll try the other version.
198
199	Forced evaluation:
200	The call
201	cache(func,(arg1,...,argn),evaluate=1)
202	forces the function to evaluate even though cached data may exist.
203
204	Testing for presence of cached result:
205	The call
206	cache(func,(arg1,...,argn),test=1)
207	retrieves cached result if it exists, otherwise None. The function will not
208	be evaluated. If both evaluate and test are switched on, evaluate takes
209	precedence.
210
211	Obtain cache filenames:
212	The call
213	cache(func,(arg1,...,argn),return_filename=1)
214	returns the hashed base filename under which this function and its
215	arguments would be cached
216
217	Clearing cached results:
218	The call
219	cache(func,'clear')
220	clears all cached data for 'func' and
221	cache('clear')
222	clears all cached data.
223
224	NOTE: The string 'clear' can be passed an argument to func using
225	cache(func,('clear',)) or cache(func,tuple(['clear'])).
226
227	New form of clear:
228	cache(func,(arg1,...,argn),clear=1)
229	clears cached data for particular combination func and args
230
231	"""
232
233	# Imports and input checks
234	#
235	import types, time, string
236
237	if not cachedir:
238	cachedir = options['cachedir']
239
240	if verbose == None: # Do NOT write 'if not verbose:', it could be zero.
241	verbose = options['verbose']
242
243	if compression == None: # Do NOT write 'if not compression:',
244	# it could be zero.
245	compression = options['compression']
246
247	# Create cache directory if needed
248	#
249	CD = checkdir(cachedir,verbose)
250
251	# Handle the case cache('clear')
252	#
253	if type(func) == types.StringType:
254	if string.lower(func) == 'clear':
255	clear_cache(CD,verbose=verbose)
256	return
257
258	# Handle the case cache(func, 'clear')
259	#
260	if type(args) == types.StringType:
261	if string.lower(args) == 'clear':
262	clear_cache(CD,func,verbose=verbose)
263	return
264
265	# Force singleton arg into a tuple
266	#
267	if type(args) != types.TupleType:
268	args = tuple([args])
269
270	# Check that kwargs is a dictionary
271	#
272	if type(kwargs) != types.DictType:
273	raise TypeError
274
275	#print 'hashing' #FIXME: make faster hashing function
276
277	# Hash arguments (and keyword args) to integer
278	#
279	arghash = myhash((args,kwargs))
280
281	# Get sizes and timestamps for files listed in dependencies.
282	# Force singletons into a tuple.
283	#
284	if dependencies and type(dependencies) != types.TupleType \
285	and type(dependencies) != types.ListType:
286	dependencies = tuple([dependencies])
287	deps = get_depstats(dependencies)
288
289	# Extract function name from func object
290	#
291	funcname = get_funcname(func)
292
293	# Create cache filename
294	#
295	FN = funcname+'['+`arghash`+']' # The symbol '(' does not work under unix
296
297	if return_filename:
298	return(FN)
299
300	if clear:
301	for file_type in file_types:
302	file_name = CD+FN+'_'+file_type
303	for fn in [file_name, file_name + '.z']:
304	if os.access(fn, os.F_OK):
305	if unix:
306	os.remove(fn)
307	else:
308	# FIXME: os.remove doesn't work under windows
309	os.system('del '+fn)
310	if verbose:
311	print 'MESSAGE (caching): File %s deleted' %fn
312	##else:
313	## print '%s was not accessed' %fn
314	return None
315
316
317	#-------------------------------------------------------------------
318
319	# Check if previous computation has been cached
320	#
321	if evaluate:
322	Retrieved = None # Force evaluation of func regardless of caching status.
323	reason = 4
324	else:
325	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
326	CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
327	dependencies)
328
329	if not Retrieved:
330	if test: # Do not attempt to evaluate function
331	T = None
332	else: # Evaluate function and save to cache
333	if verbose:
334	msg1(funcname, args, kwargs,reason)
335
336	# Remove expired files automatically
337	#
338	if options['expire']:
339	DeleteOldFiles(CD,verbose)
340
341	# Save args before function is evaluated in case
342	# they are modified by function
343	#
344	save_args_to_cache(CD,FN,args,kwargs,compression)
345
346	# Execute and time function with supplied arguments
347	#
348	t0 = time.time()
349	T = apply(func,args,kwargs)
350	#comptime = round(time.time()-t0)
351	comptime = time.time()-t0
352
353	if verbose:
354	msg2(funcname,args,kwargs,comptime,reason)
355
356	# Save results and estimated loading time to cache
357	#
358	loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
359	funcname, dependencies, compression)
360	if verbose:
361	msg3(loadtime, CD, FN, deps, compression)
362	compressed = compression
363
364	if options['savestat'] and (not test or Retrieved):
365	##if options['savestat']:
366	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
367
368	return(T) # Return results in all cases
369
370	# -----------------------------------------------------------------------------
371
372	def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
373	"""Generate statistics of caching efficiency.
374
375	USAGE:
376	cachestat(sortidx, period, showuser, cachedir)
377
378	ARGUMENTS:
379	sortidx -- Index of field by which lists are (default: 4)
380	Legal values are
381	0: 'Name'
382	1: 'Hits'
383	2: 'CPU'
384	3: 'Time Saved'
385	4: 'Gain(%)'
386	5: 'Size'
387	period -- If set to -1 all available caching history is used.
388	If set 0 only the current month is used (default -1).
389	showuser -- Flag for additional table showing user statistics
390	(default: None).
391	cachedir -- Directory for cache files (default: options['cachedir']).
392
393	DESCRIPTION:
394	Logged caching statistics is converted into summaries of the form
395	--------------------------------------------------------------------------
396	Function Name Hits Exec(s) Cache(s) Saved(s) Gain(%) Size
397	--------------------------------------------------------------------------
398	"""
399
400	__cachestat(sortidx, period, showuser, cachedir)
401	return
402
403	# -----------------------------------------------------------------------------
404
405	#Has mostly been moved to proper unit test
406	def test(cachedir=None,verbose=0,compression=None):
407	"""Test the functionality of caching.
408
409	USAGE:
410	test(verbose)
411
412	ARGUMENTS:
413	verbose -- Flag whether caching will output its statistics (default=0)
414	cachedir -- Directory for cache files (Default: options['cachedir'])
415	compression -- Flag zlib compression (Default: options['compression'])
416	"""
417
418	import string, time
419
420	# Initialise
421	#
422	import caching
423	reload(caching)
424
425	if not cachedir:
426	cachedir = options['cachedir']
427
428	if verbose is None: # Do NOT write 'if not verbose:', it could be zero.
429	verbose = options['verbose']
430
431	if compression == None: # Do NOT write 'if not compression:',
432	# it could be zero.
433	compression = options['compression']
434	else:
435	try:
436	set_option('compression', compression)
437	except:
438	test_error('Set option failed')
439
440	try:
441	import zlib
442	except:
443	print
444	print '* Could not find zlib, default to no-compression *'
445	print '* Installing zlib will improve performance of caching *'
446	print
447	compression = 0
448	set_option('compression', compression)
449
450	print
451	print_header_box('Testing caching module - please stand by')
452	print
453
454	# Define a test function to be cached
455	#
456	def f(a,b,c,N,x=0,y='abcdefg'):
457	"""f(a,b,c,N)
458	Do something time consuming and produce a complex result.
459	"""
460
461	import string
462
463	B = []
464	for n in range(N):
465	s = str(n+2.0/(n + 4.0))+'.a'*10
466	B.append((a,b,c,s,n,x,y))
467	return(B)
468
469	# Check that default cachedir is OK
470	#
471	CD = checkdir(cachedir,verbose)
472
473
474	# Make a dependency file
475	#
476	try:
477	DepFN = CD + 'testfile.tmp'
478	DepFN_wildcard = CD + 'test*.tmp'
479	Depfile = open(DepFN,'w')
480	Depfile.write('We are the knights who say NI!')
481	Depfile.close()
482	test_OK('Wrote file %s' %DepFN)
483	except:
484	test_error('Could not open file %s for writing - check your environment' \
485	% DepFN)
486
487	# Check set_option (and switch stats off
488	#
489	try:
490	set_option('savestat',0)
491	assert(options['savestat'] == 0)
492	test_OK('Set option')
493	except:
494	test_error('Set option failed')
495
496	# Make some test input arguments
497	#
498	N = 5000 #Make N fairly small here
499
500	a = [1,2]
501	b = ('Thou shalt count the number three',4)
502	c = {'Five is right out': 6, (7,8): 9}
503	x = 3
504	y = 'holy hand granate'
505
506	# Test caching
507	#
508	if compression:
509	comprange = 2
510	else:
511	comprange = 1
512
513	for comp in range(comprange):
514
515	# Evaluate and store
516	#
517	try:
518	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
519	verbose=verbose, compression=comp)
520	if comp:
521	test_OK('Caching evaluation with compression')
522	else:
523	test_OK('Caching evaluation without compression')
524	except:
525	if comp:
526	test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
527	else:
528	test_error('Caching evaluation failed - try caching.test(verbose=1)')
529
530	# Retrieve
531	#
532	try:
533	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
534	compression=comp)
535
536	if comp:
537	test_OK('Caching retrieval with compression')
538	else:
539	test_OK('Caching retrieval without compression')
540	except:
541	if comp:
542	test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
543	else:
544	test_error('Caching retrieval failed - try caching.test(verbose=1)')
545
546	# Reference result
547	#
548	T3 = f(a,b,c,N,x=x,y=y) # Compute without caching
549
550	if T1 == T2 and T2 == T3:
551	if comp:
552	test_OK('Basic caching functionality (with compression)')
553	else:
554	test_OK('Basic caching functionality (without compression)')
555	else:
556	test_error('Cached result does not match computed result')
557
558
559	# Test return_filename
560	#
561	try:
562	FN = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
563	return_filename=1)
564	assert(FN[:2] == 'f[')
565	test_OK('Return of cache filename')
566	except:
567	test_error('Return of cache filename failed')
568
569	# Test existence of cachefiles
570	#
571	try:
572	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
573	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
574	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
575	test_OK('Presence of cache files')
576	datafile.close()
577	argsfile.close()
578	admfile.close()
579	except:
580	test_error('Expected cache files did not exist')
581
582	# Test 'test' function when cache is present
583	#
584	try:
585	#T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
586	# evaluate=1)
587	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
588	assert(T1 == T4)
589
590	test_OK("Option 'test' when cache file present")
591	except:
592	test_error("Option 'test' when cache file present failed")
593
594	# Test that 'clear' works
595	#
596	#try:
597	# caching.cache(f,'clear',verbose=verbose)
598	# test_OK('Clearing of cache files')
599	#except:
600	# test_error('Clear does not work')
601	try:
602	caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)
603	test_OK('Clearing of cache files')
604	except:
605	test_error('Clear does not work')
606
607
608
609	# Test 'test' function when cache is absent
610	#
611	try:
612	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
613	assert(T4 is None)
614	test_OK("Option 'test' when cache absent")
615	except:
616	test_error("Option 'test' when cache absent failed")
617
618	# Test dependencies
619	#
620	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
621	dependencies=DepFN)
622	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
623	dependencies=DepFN)
624
625	if T1 == T2:
626	test_OK('Basic dependencies functionality')
627	else:
628	test_error('Dependencies do not work')
629
630	# Test basic wildcard dependency
631	#
632	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
633	dependencies=DepFN_wildcard)
634
635	if T1 == T3:
636	test_OK('Basic dependencies with wildcard functionality')
637	else:
638	test_error('Dependencies with wildcards do not work')
639
640
641	# Test that changed timestamp in dependencies triggers recomputation
642
643	# Modify dependency file
644	Depfile = open(DepFN,'a')
645	Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
646	Depfile.close()
647
648	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
649	dependencies=DepFN, test = 1)
650
651	if T3 is None:
652	test_OK('Changed dependencies recognised')
653	else:
654	test_error('Changed dependencies not recognised')
655
656	# Test recomputation when dependencies have changed
657	#
658	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
659	dependencies=DepFN)
660	if T1 == T3:
661	test_OK('Recomputed value with changed dependencies')
662	else:
663	test_error('Recomputed value with changed dependencies failed')
664
665	# Performance test (with statistics)
666	# Don't really rely on this as it will depend on specific computer.
667	#
668
669	set_option('savestat',1)
670
671	N = 20*N #Should be large on fast computers...
672	tt = time.time()
673	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
674	t1 = time.time() - tt
675
676	tt = time.time()
677	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
678	t2 = time.time() - tt
679
680	if T1 == T2:
681	if t1 > t2:
682	test_OK('Performance test: relative time saved = %s pct' \
683	%str(round((t1-t2)*100/t1,2)))
684	#else:
685	# print 'WARNING: Performance a bit low - this could be specific to current platform'
686	else:
687	test_error('Basic caching failed for new problem')
688
689	# Test presence of statistics file
690	#
691	try:
692	DIRLIST = os.listdir(CD)
693	SF = []
694	for FN in DIRLIST:
695	if string.find(FN,statsfile) >= 0:
696	fid = open(CD+FN,'r')
697	fid.close()
698	test_OK('Statistics files present')
699	except:
700	test_OK('Statistics files cannot be opened')
701
702	print_header_box('Show sample output of the caching function:')
703
704	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
705	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
706	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
707
708	print_header_box('Show sample output of cachestat():')
709	if unix:
710	cachestat()
711	else:
712	try:
713	import time
714	t = time.strptime('2030','%Y')
715	cachestat()
716	except:
717	print 'caching.cachestat() does not work here, because it'
718	print 'relies on time.strptime() which is unavailable in Windows'
719
720	print
721	test_OK('Caching self test completed')
722
723
724	# Test setoption (not yet implemented)
725	#
726
727
728	#==============================================================================
729	# Auxiliary functions
730	#==============================================================================
731
732	# Import pickler
733	# cPickle is used by functions mysave, myload, and compare
734	#
735	import cPickle # 10 to 100 times faster than pickle
736	pickler = cPickle
737
738	# Local immutable constants
739	#
740	comp_level = 1 # Compression level for zlib.
741	# comp_level = 1 works well.
742	textwidth1 = 16 # Text width of key fields in report forms.
743	textwidth2 = 132 # Maximal width of textual representation of
744	# arguments.
745	textwidth3 = 16 # Initial width of separation lines. Is modified.
746	textwidth4 = 50 # Text width in test_OK()
747	statsfile = '.cache_stat' # Basefilename for cached statistics.
748	# It will reside in the chosen cache directory.
749
750	file_types = ['Result', # File name extension for cached function results.
751	'Args', # File name extension for stored function args.
752	'Admin'] # File name extension for administrative info.
753
754	Reason_msg = ['OK', # Verbose reasons for recomputation
755	'No cached result',
756	'Dependencies have changed',
757	'Byte code or arguments have changed',
758	'Recomputation was requested by caller',
759	'Cached file was unreadable']
760
761	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
762
763	def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression,
764	dependencies):
765	"""Determine whether cached result exists and return info.
766
767	USAGE:
768	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
769	CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
770	dependencies)
771
772	INPUT ARGUMENTS:
773	CD -- Cache Directory
774	FN -- Suggested cache file name
775	func -- Function object
776	args -- Tuple of arguments
777	kwargs -- Dictionary of keyword arguments
778	deps -- Dependencies time stamps
779	verbose -- Flag text output
780	compression -- Flag zlib compression
781	dependencies -- Given list of dependencies
782
783	OUTPUT ARGUMENTS:
784	T -- Cached result if present otherwise None
785	FN -- File name under which new results must be saved
786	Retrieved -- True if a valid cached result was found
787	reason -- 0: OK (if Retrieved),
788	1: No cached result,
789	2: Dependencies have changed,
790	3: Arguments or Bytecode have changed
791	4: Recomputation was forced
792	comptime -- Number of seconds it took to computed cachged result
793	loadtime -- Number of seconds it took to load cached result
794	compressed -- Flag (0,1) if cached results were compressed or not
795
796	DESCRIPTION:
797	Determine if cached result exists as follows:
798	Load in saved arguments and bytecode stored under hashed filename.
799	If they are identical to current arguments and bytecode and if dependencies
800	have not changed their time stamp, then return cached result.
801
802	Otherwise return filename under which new results should be cached.
803	Hash collisions are handled recursively by calling CacheLookup again with a
804	modified filename.
805	"""
806
807	import time, string, types
808
809	# Assess whether cached result exists - compressed or not.
810	#
811	if verbose:
812	print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
813	%(CD+FN, file_types[0], file_types[1], file_types[2])
814	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
815	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
816	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
817
818	if not (argsfile and datafile and admfile) or \
819	not (compressed0 == compressed1 and compressed0 == compressed2):
820	# Cached result does not exist or files were compressed differently
821	#
822	# This will ensure that evaluation will take place unless all files are
823	# present.
824
825	reason = 1
826	return(None,FN,None,reason,None,None,None) #Recompute using same filename
827
828	compressed = compressed0 # Remember if compressed files were actually used
829	datafile.close()
830
831	# Retrieve arguments and adm. info
832	#
833	R, reason = myload(argsfile,compressed) # The original arguments
834	argsfile.close()
835
836	##if R == None and reason > 0:
837	if reason > 0:
838	return(None,FN,None,reason,None,None,None) #Recompute using same filename
839	else:
840	(argsref, kwargsref) = R
841
842	R, reason = myload(admfile,compressed)
843	admfile.close()
844	##if R == None and reason > 0:
845	if reason > 0:
846	return(None,FN,None,reason,None,None,None) #Recompute using same filename
847
848
849	depsref = R[0] # Dependency statistics
850	comptime = R[1] # The computation time
851	coderef = R[2] # The byte code
852	funcname = R[3] # The function name
853
854	# Check if dependencies have changed
855	#
856	if dependencies and not compare(depsref,deps):
857	if verbose:
858	print 'MESSAGE (caching.py): Dependencies', dependencies, \
859	'have changed - recomputing'
860	# Don't use cached file - recompute
861	reason = 2
862	return(None,FN,None,reason,None,None,None)
863
864	# Get bytecode from func
865	#
866	bytecode = get_bytecode(func)
867
868	#print compare(argsref,args),
869	#print compare(kwargsref,kwargs),
870	#print compare(bytecode,coderef)
871
872	# Check if arguments or bytecode have changed
873	#
874	if compare(argsref,args) and compare(kwargsref,kwargs) and \
875	(not options['bytecode'] or compare(bytecode,coderef)):
876
877	# Arguments and dependencies match. Get cached results
878	#
879	T, loadtime, compressed, reason = load_from_cache(CD,FN,compressed)
880	###if T == None and reason > 0: #This doesn't work if T is a numeric array
881	if reason > 0:
882	return(None,FN,None,reason,None,None,None) #Recompute using same FN
883
884	Retrieved = 1
885	reason = 0
886
887	if verbose:
888	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
889
890	if loadtime >= comptime:
891	print 'WARNING (caching.py): Caching did not yield any gain.'
892	print ' Consider executing function ',
893	print '('+funcname+') without caching.'
894	else:
895
896	# Non matching arguments or bytecodes signify a hash-collision.
897	# This is resolved by recursive search of cache filenames
898	# until either a matching or an unused filename is found.
899	#
900	(T,FN,Retrieved,reason,comptime,loadtime,compressed) = \
901	CacheLookup(CD,FN+'x',func,args,kwargs,deps,verbose,compression, \
902	dependencies)
903
904	# DEBUGGING
905	# if not Retrieved:
906	# print 'Arguments did not match'
907	# else:
908	# print 'Match found !'
909	if not Retrieved:
910	reason = 3 #The real reason is that args or bytecodes have changed.
911	#Not that the recursive seach has found an unused filename
912
913	return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
914
915	# -----------------------------------------------------------------------------
916
917	def clear_cache(CD,func=None, verbose=None):
918	"""Clear cache for func.
919
920	USAGE:
921	clear(CD, func, verbose)
922
923	ARGUMENTS:
924	CD -- Caching directory (required)
925	func -- Function object (default: None)
926	verbose -- Flag verbose output (default: None)
927
928	DESCRIPTION:
929
930	If func == None, clear everything,
931	otherwise clear only files pertaining to func.
932	"""
933
934	import os, re
935
936	if CD[-1] != os.sep:
937	CD = CD+os.sep
938
939	if verbose == None:
940	verbose = options['verbose']
941
942	# FIXME: Windows version needs to be tested
943
944	if func:
945	funcname = get_funcname(func)
946	if verbose:
947	print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
948
949	file_names = os.listdir(CD)
950	for file_name in file_names:
951	#RE = re.search('^' + funcname,file_name) #Inefficient
952	#if RE:
953	if file_name[:len(funcname)] == funcname:
954	if unix:
955	os.remove(CD+file_name)
956	else:
957	os.system('del '+CD+file_name)
958	# FIXME: os.remove doesn't work under windows
959	else:
960	file_names = os.listdir(CD)
961	if len(file_names) > 0:
962	if verbose:
963	print 'MESSAGE (caching.py): Remove the following files:'
964	for file_name in file_names:
965	print file_name
966
967	A = raw_input('Delete (Y/N)[N] ?')
968	else:
969	A = 'Y'
970
971	if A == 'Y' or A == 'y':
972	for file_name in file_names:
973	if unix:
974	os.remove(CD+file_name)
975	else:
976	os.system('del '+CD+file_name)
977	# FIXME: os.remove doesn't work under windows
978	#exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
979
980	# -----------------------------------------------------------------------------
981
982	def DeleteOldFiles(CD,verbose=None):
983	"""Remove expired files
984
985	USAGE:
986	DeleteOldFiles(CD,verbose=None)
987	"""
988
989	if verbose == None:
990	verbose = options['verbose']
991
992	maxfiles = options['maxfiles']
993
994	# FIXME: Windows version
995
996	import os
997	block = 1000 # How many files to delete per invokation
998	Files = os.listdir(CD)
999	numfiles = len(Files)
1000	if not unix: return # FIXME: Windows case ?
1001
1002	if numfiles > maxfiles:
1003	delfiles = numfiles-maxfiles+block
1004	if verbose:
1005	print 'Deleting '+`delfiles`+' expired files:'
1006	os.system('ls -lur '+CD+'* \| head -' + `delfiles`) # List them
1007	os.system('ls -ur '+CD+'* \| head -' + `delfiles` + ' \| xargs /bin/rm')
1008	# Delete them
1009	# FIXME: Replace this with os.listdir and os.remove
1010
1011	# -----------------------------------------------------------------------------
1012
1013	def save_args_to_cache(CD,FN,args,kwargs,compression):
1014	"""Save arguments to cache
1015
1016	USAGE:
1017	save_args_to_cache(CD,FN,args,kwargs,compression)
1018	"""
1019
1020	import time, os, sys, types
1021
1022	(argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
1023
1024	if not argsfile:
1025	if verbose:
1026	print 'ERROR (caching): Could not open %s' %argsfile.name
1027	raise IOError
1028
1029	mysave((args,kwargs),argsfile,compression) # Save args and kwargs to cache
1030	argsfile.close()
1031
1032	# Change access rights if possible
1033	#
1034	#if unix:
1035	# try:
1036	# exitcode=os.system('chmod 666 '+argsfile.name)
1037	# except:
1038	# pass
1039	#else:
1040	# pass # FIXME: Take care of access rights under Windows
1041
1042	return
1043
1044	# -----------------------------------------------------------------------------
1045
1046	def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1047	dependencies, compression):
1048	"""Save computed results T and admin info to cache
1049
1050	USAGE:
1051	save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
1052	dependencies, compression)
1053	"""
1054
1055	import time, os, sys, types
1056
1057	(datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
1058	(admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
1059
1060	if not datafile:
1061	if verbose:
1062	print 'ERROR (caching): Could not open %s' %datafile.name
1063	raise IOError
1064
1065	if not admfile:
1066	if verbose:
1067	print 'ERROR (caching): Could not open %s' %admfile.name
1068	raise IOError
1069
1070	t0 = time.time()
1071
1072	mysave(T,datafile,compression) # Save data to cache
1073	datafile.close()
1074	#savetime = round(time.time()-t0,2)
1075	savetime = time.time()-t0
1076
1077	bytecode = get_bytecode(func) # Get bytecode from function object
1078	admtup = (deps, comptime, bytecode, funcname) # Gather admin info
1079
1080	mysave(admtup,admfile,compression) # Save admin info to cache
1081	admfile.close()
1082
1083	# Change access rights if possible
1084	#
1085	#if unix:
1086	# try:
1087	# exitcode=os.system('chmod 666 '+datafile.name)
1088	# exitcode=os.system('chmod 666 '+admfile.name)
1089	# except:
1090	# pass
1091	#else:
1092	# pass # FIXME: Take care of access rights under Windows
1093
1094	return(savetime)
1095
1096	# -----------------------------------------------------------------------------
1097
1098	def load_from_cache(CD,FN,compression):
1099	"""Load previously cached data from file FN
1100
1101	USAGE:
1102	load_from_cache(CD,FN,compression)
1103	"""
1104
1105	import time
1106
1107	(datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
1108	t0 = time.time()
1109	T, reason = myload(datafile,compressed)
1110	#loadtime = round(time.time()-t0,2)
1111	loadtime = time.time()-t0
1112	datafile.close()
1113
1114	return T, loadtime, compressed, reason
1115
1116	# -----------------------------------------------------------------------------
1117
1118	def myopen(FN,mode,compression=1):
1119	"""Open file FN using given mode
1120
1121	USAGE:
1122	myopen(FN,mode,compression=1)
1123
1124	ARGUMENTS:
1125	FN -- File name to be opened
1126	mode -- Open mode (as in open)
1127	compression -- Flag zlib compression
1128
1129	DESCRIPTION:
1130	if compression
1131	Attempt first to open FN + '.z'
1132	If this fails try to open FN
1133	else do the opposite
1134	Return file handle plus info about whether it was compressed or not.
1135	"""
1136
1137	import string
1138
1139	# Determine if file exists already (if writing was requested)
1140	# This info is only used to determine if access modes should be set
1141	#
1142	if 'w' in mode or 'a' in mode:
1143	try:
1144	file = open(FN+'.z','r')
1145	file.close()
1146	new_file = 0
1147	except:
1148	try:
1149	file = open(FN,'r')
1150	file.close()
1151	new_file = 0
1152	except:
1153	new_file = 1
1154	else:
1155	new_file = 0 #Assume it exists if mode was not 'w'
1156
1157
1158	compressed = 0
1159	if compression:
1160	try:
1161	file = open(FN+'.z',mode)
1162	compressed = 1
1163	except:
1164	try:
1165	file = open(FN,mode)
1166	except:
1167	file = None
1168	else:
1169	try:
1170	file = open(FN,mode)
1171	except:
1172	try:
1173	file = open(FN+'.z',mode)
1174	compressed = 1
1175	except:
1176	file = None
1177
1178	# Now set access rights if it is a new file
1179	#
1180	if file and new_file:
1181	if unix:
1182	exitcode=os.system('chmod 666 '+file.name)
1183	else:
1184	pass # FIXME: Take care of access rights under Windows
1185
1186	return(file,compressed)
1187
1188	# -----------------------------------------------------------------------------
1189
1190	def myload(file, compressed):
1191	"""Load data from file
1192
1193	USAGE:
1194	myload(file, compressed)
1195	"""
1196
1197	reason = 0
1198	try:
1199	if compressed:
1200	import zlib
1201
1202	RsC = file.read()
1203	try:
1204	Rs = zlib.decompress(RsC)
1205	except:
1206	# File "./caching.py", line 1032, in load_from_cache
1207	# T = myload(datafile,compressed)
1208	# File "./caching.py", line 1124, in myload
1209	# Rs = zlib.decompress(RsC)
1210	# zlib.error: Error -5 while decompressing data
1211	#print 'ERROR (caching): Could not decompress ', file.name
1212	#raise Exception
1213	reason = 5 #(Unreadable file)
1214	return None, reason
1215
1216
1217	del RsC # Free up some space
1218	R = pickler.loads(Rs)
1219	else:
1220	try:
1221	R = pickler.load(file)
1222	#except EOFError, e:
1223	except:
1224	#Catch e.g., file with 0 length or corrupted
1225	reason = 5 #(Unreadable file)
1226	return None, reason
1227
1228	except MemoryError:
1229	import sys
1230	if options['verbose']:
1231	print 'ERROR (caching): Out of memory while loading %s, aborting' \
1232	%(file.name)
1233
1234	# Raise the error again for now
1235	#
1236	raise MemoryError
1237
1238	return R, reason
1239
1240	# -----------------------------------------------------------------------------
1241
1242	def mysave(T,file,compression):
1243	"""Save data T to file
1244
1245	USAGE:
1246	mysave(T,file,compression)
1247
1248	"""
1249
1250	bin = options['bin']
1251
1252	if compression:
1253	try:
1254	import zlib
1255	except:
1256	print
1257	print '* Could not find zlib *'
1258	print '* Try to run caching with compression off *'
1259	print "* caching.set_option('compression', 0) *"
1260	raise Exception
1261
1262
1263	try:
1264	Ts = pickler.dumps(T, bin)
1265	except MemoryError:
1266	msg = '****WARNING (caching.py): Could not pickle data for compression.'
1267	msg += ' Try using compression = False'
1268	raise MemoryError, msg
1269	else:
1270	#Compressed pickling
1271	TsC = zlib.compress(Ts, comp_level)
1272	file.write(TsC)
1273	else:
1274	#Uncompressed pickling
1275	pickler.dump(T, file, bin)
1276
1277	# FIXME: This may not work on Windoze network drives.
1278	# The error msg is IOError: [Errno 22] Invalid argument
1279	# Testing with small files was OK, though.
1280	# I think this is an OS problem.
1281
1282	# Excerpt from http://www.ultraseek.com/support/faqs/4173.html
1283
1284	# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
1285	#
1286	#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
1287	#
1288	#The following extract is taken from the site http://www.python.org:
1289	#
1290	#---------------------------------------------------------------------------------------------
1291	#exception IOError
1292	#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
1293	#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
1294	#---------------------------------------------------------------------------------------------
1295	#
1296	#The error code(s) that accompany exceptions are described at:
1297	#http://www.python.org/dev/doc/devel//lib/module-errno.html
1298	#
1299	#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
1300	#
1301	#exceptions.IOError invalid argument Errno 22
1302
1303	#try:
1304	# pickler.dump(T,file,bin)
1305	#except IOError, e:
1306	# print e
1307	# msg = 'Could not store to %s, bin=%s' %(file, bin)
1308	# raise msg
1309
1310
1311	# -----------------------------------------------------------------------------
1312
1313	def myhash(T):
1314	"""Compute hashed integer from hashable values of tuple T
1315
1316	USAGE:
1317	myhash(T)
1318
1319	ARGUMENTS:
1320	T -- Tuple
1321	"""
1322
1323	import types
1324
1325	# Get hash vals for hashable entries
1326	#
1327	if type(T) == types.TupleType or type(T) == types.ListType:
1328	hvals = []
1329	for k in range(len(T)):
1330	h = myhash(T[k])
1331	hvals.append(h)
1332	val = hash(tuple(hvals))
1333	elif type(T) == types.DictType:
1334	val = dicthash(T)
1335	else:
1336	try:
1337	val = hash(T)
1338	except:
1339	val = 1
1340	try:
1341	import Numeric
1342	if type(T) == Numeric.ArrayType:
1343	hvals = []
1344	for e in T:
1345	h = myhash(e)
1346	hvals.append(h)
1347	val = hash(tuple(hvals))
1348	else:
1349	val = 1 #Could implement other Numeric types here
1350	except:
1351	pass
1352
1353	return(val)
1354
1355	# -----------------------------------------------------------------------------
1356
1357	def dicthash(D):
1358	"""Compute hashed integer from hashable values of dictionary D
1359
1360	USAGE:
1361	dicthash(D)
1362	"""
1363
1364	keys = D.keys()
1365
1366	# Get hash values for hashable entries
1367	#
1368	hvals = []
1369	for k in range(len(keys)):
1370	try:
1371	h = hash(D[keys[k]])
1372	hvals.append(h)
1373	except:
1374	pass
1375
1376	# Hash obtained values into one value
1377	#
1378	return(hash(tuple(hvals)))
1379
1380	# -----------------------------------------------------------------------------
1381
1382	def compare(A,B):
1383	"""Safe comparison of general objects
1384
1385	USAGE:
1386	compare(A,B)
1387
1388	DESCRIPTION:
1389	Return 1 if A and B they are identical, 0 otherwise
1390	"""
1391
1392	try:
1393	identical = (A == B)
1394	except:
1395	try:
1396	identical = (pickler.dumps(A) == pickler.dumps(B))
1397	except:
1398	identical = 0
1399
1400	return(identical)
1401
1402	# -----------------------------------------------------------------------------
1403
1404	def nospace(s):
1405	"""Replace spaces in string s with underscores
1406
1407	USAGE:
1408	nospace(s)
1409
1410	ARGUMENTS:
1411	s -- string
1412	"""
1413
1414	import string
1415
1416	newstr = ''
1417	for i in range(len(s)):
1418	if s[i] == ' ':
1419	newstr = newstr+'_'
1420	else:
1421	newstr = newstr+s[i]
1422
1423	return(newstr)
1424
1425	# -----------------------------------------------------------------------------
1426
1427	def get_funcname(func):
1428	"""Retrieve name of function object func (depending on its type)
1429
1430	USAGE:
1431	get_funcname(func)
1432	"""
1433
1434	import types, string
1435
1436	if type(func) == types.FunctionType:
1437	funcname = func.func_name
1438	elif type(func) == types.BuiltinFunctionType:
1439	funcname = func.__name__
1440	else:
1441	tab = string.maketrans("<>'"," ")
1442	tmp = string.translate(`func`,tab)
1443	tmp = string.split(tmp)
1444	funcname = string.join(tmp)
1445
1446	funcname = nospace(funcname)
1447	return(funcname)
1448
1449	# -----------------------------------------------------------------------------
1450
1451	def get_bytecode(func):
1452	""" Get bytecode from function object.
1453
1454	USAGE:
1455	get_bytecode(func)
1456	"""
1457
1458	import types
1459
1460	if type(func) == types.FunctionType:
1461	bytecode = func.func_code.co_code
1462	consts = func.func_code.co_consts
1463	argcount = func.func_code.co_argcount
1464	defaults = func.func_defaults
1465	elif type(func) == types.MethodType:
1466	bytecode = func.im_func.func_code.co_code
1467	consts = func.im_func.func_code.co_consts
1468	argcount = func.im_func.func_code.co_argcount
1469	defaults = func.im_func.func_defaults
1470	else:
1471	#raise Exception #Test only
1472	bytecode = None #Built-in functions are assumed not to change
1473	consts = 0
1474	argcount = 0
1475	defaults = 0
1476
1477	return (bytecode, consts, argcount, defaults)
1478
1479	# -----------------------------------------------------------------------------
1480
1481	def get_depstats(dependencies):
1482	""" Build dictionary of dependency files and their size, mod. time and ctime.
1483
1484	USAGE:
1485	get_depstats(dependencies):
1486	"""
1487
1488	import types
1489
1490	d = {}
1491	if dependencies:
1492
1493	#Expand any wildcards
1494	import glob
1495	expanded_dependencies = []
1496	for FN in dependencies:
1497	expanded_FN = glob.glob(FN)
1498
1499	expanded_dependencies += expanded_FN
1500
1501
1502	for FN in expanded_dependencies:
1503	if not type(FN) == types.StringType:
1504	errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
1505	errmsg += ' Dependency given: %s' %FN
1506	raise Exception, errmsg
1507	if not os.access(FN,os.F_OK):
1508	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
1509	raise Exception, errmsg
1510	(size,atime,mtime,ctime) = filestat(FN)
1511
1512	# We don't use atime because that would cause recomputation every time.
1513	# We don't use ctime because that is irrelevant and confusing for users.
1514	d.update({FN : (size,mtime)})
1515
1516	return(d)
1517
1518	# -----------------------------------------------------------------------------
1519
1520	def filestat(FN):
1521	"""A safe wrapper using os.stat to get basic file statistics
1522	The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
1523
1524	USAGE:
1525	filestat(FN)
1526
1527	DESCRIPTION:
1528	Must compile Python with
1529	CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
1530	configure
1531	as given in section 8.1.1 Large File Support in the Libray Reference
1532	"""
1533
1534	import os, time
1535
1536	try:
1537	stats = os.stat(FN)
1538	size = stats[6]
1539	atime = stats[7]
1540	mtime = stats[8]
1541	ctime = stats[9]
1542	except:
1543
1544	# Hack to get the results anyway (works only on Unix at the moment)
1545	#
1546	print 'Hack to get os.stat when files are too large'
1547
1548	if unix:
1549	tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
1550	# Unique filename, FIXME: Use random number
1551
1552	# Get size and access time (atime)
1553	#
1554	exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
1555	(size,atime) = get_lsline(tmp)
1556
1557	# Get size and modification time (mtime)
1558	#
1559	exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
1560	(size,mtime) = get_lsline(tmp)
1561
1562	# Get size and ctime
1563	#
1564	exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
1565	(size,ctime) = get_lsline(tmp)
1566
1567	try:
1568	exitcode=os.system('rm '+tmp)
1569	# FIXME: Gives error if file doesn't exist
1570	except:
1571	pass
1572	else:
1573	pass
1574	raise Exception # FIXME: Windows case
1575
1576	return(long(size),atime,mtime,ctime)
1577
1578	# -----------------------------------------------------------------------------
1579
1580	def get_lsline(FN):
1581	"""get size and time for filename
1582
1583	USAGE:
1584	get_lsline(file_name)
1585
1586	DESCRIPTION:
1587	Read in one line 'ls -la' item from file (generated by filestat) and
1588	convert time to seconds since epoch. Return file size and time.
1589	"""
1590
1591	import string, time
1592
1593	f = open(FN,'r')
1594	info = f.read()
1595	info = string.split(info)
1596
1597	size = info[4]
1598	week = info[5]
1599	mon = info[6]
1600	day = info[7]
1601	hour = info[8]
1602	year = info[9]
1603
1604	str = week+' '+mon+' '+day+' '+hour+' '+year
1605	timetup = time.strptime(str)
1606	t = time.mktime(timetup)
1607	return(size, t)
1608
1609	# -----------------------------------------------------------------------------
1610
1611	def checkdir(CD,verbose=None):
1612	"""Check or create caching directory
1613
1614	USAGE:
1615	checkdir(CD,verbose):
1616
1617	ARGUMENTS:
1618	CD -- Directory
1619	verbose -- Flag verbose output (default: None)
1620
1621	DESCRIPTION:
1622	If CD does not exist it will be created if possible
1623	"""
1624
1625	import os
1626	import os.path
1627
1628	if CD[-1] != os.sep:
1629	CD = CD + os.sep # Add separator for directories
1630
1631	CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
1632	if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
1633	try:
1634	exitcode=os.mkdir(CD)
1635
1636	# Change access rights if possible
1637	#
1638	if unix:
1639	exitcode=os.system('chmod 777 '+CD)
1640	else:
1641	pass # FIXME: What about acces rights under Windows?
1642	if verbose: print 'MESSAGE: Directory', CD, 'created.'
1643	except:
1644	print 'WARNING: Directory', CD, 'could not be created.'
1645	if unix:
1646	CD = '/tmp/'
1647	else:
1648	CD = 'C:'
1649	print 'Using directory %s instead' %CD
1650
1651	return(CD)
1652
1653	#==============================================================================
1654	# Statistics
1655	#==============================================================================
1656
1657	def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
1658	compression):
1659	"""Add stats entry
1660
1661	USAGE:
1662	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
1663
1664	DESCRIPTION:
1665	Make one entry in the stats file about one cache hit recording time saved
1666	and other statistics. The data are used by the function cachestat.
1667	"""
1668
1669	import os, time
1670
1671	try:
1672	TimeTuple = time.localtime(time.time())
1673	extension = time.strftime('%b%Y',TimeTuple)
1674	SFN = CD+statsfile+'.'+extension
1675	#statfile = open(SFN,'a')
1676	(statfile, dummy) = myopen(SFN,'a',compression=0)
1677
1678	# Change access rights if possible
1679	#
1680	#if unix:
1681	# try:
1682	# exitcode=os.system('chmod 666 '+SFN)
1683	# except:
1684	# pass
1685	except:
1686	print 'Warning: Stat file could not be opened'
1687
1688	try:
1689	if os.environ.has_key('USER'):
1690	user = os.environ['USER']
1691	else:
1692	user = 'Nobody'
1693
1694	date = time.asctime(TimeTuple)
1695
1696	if Retrieved:
1697	hit = '1'
1698	else:
1699	hit = '0'
1700
1701	# Get size of result file
1702	#
1703	if compression:
1704	stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
1705	else:
1706	stats = os.stat(CD+FN+'_'+file_types[0])
1707
1708	if stats:
1709	size = stats[6]
1710	else:
1711	size = -1 # Error condition, but don't crash. This is just statistics
1712
1713	# Build entry
1714	#
1715	entry = date + ',' +\
1716	user + ',' +\
1717	FN + ',' +\
1718	str(int(size)) + ',' +\
1719	str(compression) + ',' +\
1720	hit + ',' +\
1721	str(reason) + ',' +\
1722	str(round(comptime,4)) + ',' +\
1723	str(round(loadtime,4)) +\
1724	CR
1725
1726	statfile.write(entry)
1727	statfile.close()
1728	except:
1729	print 'Warning: Writing of stat file failed'
1730
1731	# -----------------------------------------------------------------------------
1732
1733	# FIXME: should take cachedir as an optional arg
1734	#
1735	def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1736	""" List caching statistics.
1737
1738	USAGE:
1739	__cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
1740
1741	Generate statistics of caching efficiency.
1742	The parameter sortidx determines by what field lists are sorted.
1743	If the optional keyword period is set to -1,
1744	all available caching history is used.
1745	If it is 0 only the current month is used.
1746	Future versions will include more than one month....
1747	OMN 20/8/2000
1748	"""
1749
1750	import os
1751	import os.path
1752	from string import split, rstrip, find, atof, atoi
1753	from time import strptime, localtime, strftime, mktime, ctime
1754
1755	# sortidx = 4 # Index into Fields[1:]. What to sort by.
1756
1757	Fields = ['Name', 'Hits', 'Exec(s)', \
1758	'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
1759	Widths = [25,7,9,9,9,9,13]
1760	#Types = ['s','d','d','d','d','.2f','d']
1761	Types = ['s','d','.2f','.2f','.2f','.2f','d']
1762
1763	Dictnames = ['Function', 'User']
1764
1765	if not cachedir:
1766	cachedir = checkdir(options['cachedir'])
1767
1768	SD = os.path.expanduser(cachedir) # Expand ~ or ~user in pathname
1769
1770	if period == -1: # Take all available stats
1771	SFILENAME = statsfile
1772	else: # Only stats from current month
1773	# MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
1774	TimeTuple = localtime(time())
1775	extension = strftime('%b%Y',TimeTuple)
1776	SFILENAME = statsfile+'.'+extension
1777
1778	DIRLIST = os.listdir(SD)
1779	SF = []
1780	for FN in DIRLIST:
1781	if find(FN,SFILENAME) >= 0:
1782	SF.append(FN)
1783
1784	blocksize = 15000000
1785	total_read = 0
1786	total_hits = 0
1787	total_discarded = 0
1788	firstday = mktime(strptime('2030','%Y'))
1789	# FIXME: strptime don't exist in WINDOWS ?
1790	lastday = 0
1791
1792	FuncDict = {}
1793	UserDict = {}
1794	for FN in SF:
1795	input = open(SD+FN,'r')
1796	print 'Reading file ', SD+FN
1797
1798	while 1:
1799	A = input.readlines(blocksize)
1800	if len(A) == 0: break
1801	total_read = total_read + len(A)
1802	for record in A:
1803	record = tuple(split(rstrip(record),','))
1804	#print record
1805
1806	if len(record) in [8,9]:
1807	n = 0
1808	timestamp = record[n]; n=n+1
1809
1810	try:
1811	t = mktime(strptime(timestamp))
1812	except:
1813	total_discarded = total_discarded + 1
1814	continue
1815
1816	if t > lastday:
1817	lastday = t
1818	if t < firstday:
1819	firstday = t
1820
1821	user = record[n]; n=n+1
1822	func = record[n]; n=n+1
1823
1824	# Strip hash-stamp off
1825	#
1826	i = find(func,'[')
1827	func = func[:i]
1828
1829	size = atof(record[n]); n=n+1
1830	compression = atoi(record[n]); n=n+1
1831	hit = atoi(record[n]); n=n+1
1832	reason = atoi(record[n]); n=n+1 # Not used here
1833	cputime = atof(record[n]); n=n+1
1834	loadtime = atof(record[n]); n=n+1
1835
1836	if hit:
1837	total_hits = total_hits + 1
1838	saving = cputime-loadtime
1839
1840	if cputime != 0:
1841	rel_saving = round(100.0*saving/cputime,2)
1842	else:
1843	#rel_saving = round(1.0*saving,2)
1844	rel_saving = 100.0 - round(1.0*saving,2) # A bit of a hack
1845
1846	info = [1,cputime,loadtime,saving,rel_saving,size]
1847
1848	UpdateDict(UserDict,user,info)
1849	UpdateDict(FuncDict,func,info)
1850	else:
1851	pass #Stats on recomputations and their reasons could go in here
1852
1853	else:
1854	#print 'Record discarded'
1855	#print record
1856	total_discarded = total_discarded + 1
1857
1858	input.close()
1859
1860	# Compute averages of all sums and write list
1861	#
1862
1863	if total_read == 0:
1864	printline(Widths,'=')
1865	print 'CACHING STATISTICS: No valid records read'
1866	printline(Widths,'=')
1867	return
1868
1869	print
1870	printline(Widths,'=')
1871	print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
1872	printline(Widths,'=')
1873	#print ' Period:', ctime(firstday), 'to', ctime(lastday)
1874	print ' Total number of valid records', total_read
1875	print ' Total number of discarded records', total_discarded
1876	print ' Total number of hits', total_hits
1877	print
1878
1879	print ' Fields', Fields[2:], 'are averaged over number of hits'
1880	print ' Time is measured in seconds and size in bytes'
1881	print ' Tables are sorted by', Fields[1:][sortidx]
1882
1883	# printline(Widths,'-')
1884
1885	if showuser:
1886	Dictionaries = [FuncDict, UserDict]
1887	else:
1888	Dictionaries = [FuncDict]
1889
1890	i = 0
1891	for Dict in Dictionaries:
1892	for key in Dict.keys():
1893	rec = Dict[key]
1894	for n in range(len(rec)):
1895	if n > 0:
1896	rec[n] = round(1.0*rec[n]/rec[0],2)
1897	Dict[key] = rec
1898
1899	# Sort and output
1900	#
1901	keylist = SortDict(Dict,sortidx)
1902
1903	# Write Header
1904	#
1905	print
1906	#print Dictnames[i], 'statistics:'; i=i+1
1907	printline(Widths,'-')
1908	n = 0
1909	for s in Fields:
1910	if s == Fields[0]: # Left justify
1911	s = Dictnames[i] + ' ' + s; i=i+1
1912	exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
1913	else:
1914	exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
1915	print
1916	printline(Widths,'-')
1917
1918	# Output Values
1919	#
1920	for key in keylist:
1921	rec = Dict[key]
1922	n = 0
1923	if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
1924	exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
1925	for val in rec:
1926	exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
1927	print
1928	print
1929
1930	#==============================================================================
1931	# Auxiliary stats functions
1932	#==============================================================================
1933
1934	def UpdateDict(Dict,key,info):
1935	"""Update dictionary by adding new values to existing.
1936
1937	USAGE:
1938	UpdateDict(Dict,key,info)
1939	"""
1940
1941	if Dict.has_key(key):
1942	dinfo = Dict[key]
1943	for n in range(len(dinfo)):
1944	dinfo[n] = info[n] + dinfo[n]
1945	else:
1946	dinfo = info[:] # Make a copy of info list
1947
1948	Dict[key] = dinfo
1949	return Dict
1950
1951	# -----------------------------------------------------------------------------
1952
1953	def SortDict(Dict,sortidx=0):
1954	"""Sort dictionary
1955
1956	USAGE:
1957	SortDict(Dict,sortidx):
1958
1959	DESCRIPTION:
1960	Sort dictionary of lists according field number 'sortidx'
1961	"""
1962
1963	import types
1964
1965	sortlist = []
1966	keylist = Dict.keys()
1967	for key in keylist:
1968	rec = Dict[key]
1969	if not type(rec) in [types.ListType, types.TupleType]:
1970	rec = [rec]
1971
1972	if sortidx > len(rec)-1:
1973	if options['verbose']:
1974	print 'ERROR: Sorting index to large, sortidx = ', sortidx
1975	raise IndexError
1976
1977	val = rec[sortidx]
1978	sortlist.append(val)
1979
1980	A = map(None,sortlist,keylist)
1981	A.sort()
1982	keylist = map(lambda x: x[1], A) # keylist sorted by sortidx
1983
1984	return(keylist)
1985
1986	# -----------------------------------------------------------------------------
1987
1988	def printline(Widths,char):
1989	"""Print textline in fixed field.
1990
1991	USAGE:
1992	printline(Widths,char)
1993	"""
1994
1995	s = ''
1996	for n in range(len(Widths)):
1997	s = s+Widths[n]*char
1998	if n > 0:
1999	s = s+char
2000
2001	print s
2002
2003	#==============================================================================
2004	# Messages
2005	#==============================================================================
2006
2007	def msg1(funcname,args,kwargs,reason):
2008	"""Message 1
2009
2010	USAGE:
2011	msg1(funcname,args,kwargs,reason):
2012	"""
2013
2014	import string
2015	#print 'MESSAGE (caching.py): Evaluating function', funcname,
2016
2017	print_header_box('Evaluating function %s' %funcname)
2018
2019	msg7(args,kwargs)
2020	msg8(reason)
2021
2022	print_footer()
2023
2024	#
2025	# Old message
2026	#
2027	#args_present = 0
2028	#if args:
2029	# if len(args) == 1:
2030	# print 'with argument', mkargstr(args[0], textwidth2),
2031	# else:
2032	# print 'with arguments', mkargstr(args, textwidth2),
2033	# args_present = 1
2034	#
2035	#if kwargs:
2036	# if args_present:
2037	# word = 'and'
2038	# else:
2039	# word = 'with'
2040	#
2041	# if len(kwargs) == 1:
2042	# print word + ' keyword argument', mkargstr(kwargs, textwidth2)
2043	# else:
2044	# print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
2045	# args_present = 1
2046	#else:
2047	# print # Newline when no keyword args present
2048	#
2049	#if not args_present:
2050	# print '', # Default if no args or kwargs present
2051
2052
2053
2054	# -----------------------------------------------------------------------------
2055
2056	def msg2(funcname,args,kwargs,comptime,reason):
2057	"""Message 2
2058
2059	USAGE:
2060	msg2(funcname,args,kwargs,comptime,reason)
2061	"""
2062
2063	import string
2064
2065	#try:
2066	# R = Reason_msg[reason]
2067	#except:
2068	# R = 'Unknown reason'
2069
2070	#print_header_box('Caching statistics (storing) - %s' %R)
2071	print_header_box('Caching statistics (storing)')
2072
2073	msg6(funcname,args,kwargs)
2074	msg8(reason)
2075
2076	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2077
2078	# -----------------------------------------------------------------------------
2079
2080	def msg3(savetime, CD, FN, deps,compression):
2081	"""Message 3
2082
2083	USAGE:
2084	msg3(savetime, CD, FN, deps,compression)
2085	"""
2086
2087	import string
2088	print string.ljust('\| Loading time:', textwidth1) + str(round(savetime,2)) + \
2089	' seconds (estimated)'
2090	msg5(CD,FN,deps,compression)
2091
2092	# -----------------------------------------------------------------------------
2093
2094	def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
2095	"""Message 4
2096
2097	USAGE:
2098	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
2099	"""
2100
2101	import string
2102
2103	print_header_box('Caching statistics (retrieving)')
2104
2105	msg6(funcname,args,kwargs)
2106	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
2107	print string.ljust('\| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
2108	print string.ljust('\| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
2109	' seconds'
2110	msg5(CD,FN,deps,compression)
2111
2112	# -----------------------------------------------------------------------------
2113
2114	def msg5(CD,FN,deps,compression):
2115	"""Message 5
2116
2117	USAGE:
2118	msg5(CD,FN,deps,compression)
2119
2120	DESCRIPTION:
2121	Print dependency stats. Used by msg3 and msg4
2122	"""
2123
2124	import os, time, string
2125
2126	print '\|'
2127	print string.ljust('\| Caching dir: ', textwidth1) + CD
2128
2129	if compression:
2130	suffix = '.z'
2131	bytetext = 'bytes, compressed'
2132	else:
2133	suffix = ''
2134	bytetext = 'bytes'
2135
2136	for file_type in file_types:
2137	file_name = FN + '_' + file_type + suffix
2138	print string.ljust('\| ' + file_type + ' file: ', textwidth1) + file_name,
2139	stats = os.stat(CD+file_name)
2140	print '('+ str(stats[6]) + ' ' + bytetext + ')'
2141
2142	print '\|'
2143	if len(deps) > 0:
2144	print '\| Dependencies: '
2145	dependencies = deps.keys()
2146	dlist = []; maxd = 0
2147	tlist = []; maxt = 0
2148	slist = []; maxs = 0
2149	for d in dependencies:
2150	stats = deps[d]
2151	t = time.ctime(stats[1])
2152	s = str(stats[0])
2153	#if s[-1] == 'L':
2154	# s = s[:-1] # Strip rightmost 'long integer' L off.
2155	# # FIXME: Unnecessary in versions later than 1.5.2
2156
2157	if len(d) > maxd: maxd = len(d)
2158	if len(t) > maxt: maxt = len(t)
2159	if len(s) > maxs: maxs = len(s)
2160	dlist.append(d)
2161	tlist.append(t)
2162	slist.append(s)
2163
2164	for n in range(len(dlist)):
2165	d = string.ljust(dlist[n]+':', maxd+1)
2166	t = string.ljust(tlist[n], maxt)
2167	s = string.rjust(slist[n], maxs)
2168
2169	print '\| ', d, t, ' ', s, 'bytes'
2170	else:
2171	print '\| No dependencies'
2172	print_footer()
2173
2174	# -----------------------------------------------------------------------------
2175
2176	def msg6(funcname,args,kwargs):
2177	"""Message 6
2178
2179	USAGE:
2180	msg6(funcname,args,kwargs)
2181	"""
2182
2183	import string
2184	print string.ljust('\| Function:', textwidth1) + funcname
2185
2186	msg7(args,kwargs)
2187
2188	# -----------------------------------------------------------------------------
2189
2190	def msg7(args,kwargs):
2191	"""Message 7
2192
2193	USAGE:
2194	msg7(args,kwargs):
2195	"""
2196
2197	import string
2198
2199	args_present = 0
2200	if args:
2201	if len(args) == 1:
2202	print string.ljust('\| Argument:', textwidth1) + mkargstr(args[0], \
2203	textwidth2)
2204	else:
2205	print string.ljust('\| Arguments:', textwidth1) + \
2206	mkargstr(args, textwidth2)
2207	args_present = 1
2208
2209	if kwargs:
2210	if len(kwargs) == 1:
2211	print string.ljust('\| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
2212	textwidth2)
2213	else:
2214	print string.ljust('\| Keyword Args:', textwidth1) + \
2215	mkargstr(kwargs, textwidth2)
2216	args_present = 1
2217
2218	if not args_present:
2219	print '\| No arguments' # Default if no args or kwargs present
2220
2221	# -----------------------------------------------------------------------------
2222
2223	def msg8(reason):
2224	"""Message 8
2225
2226	USAGE:
2227	msg8(reason):
2228	"""
2229
2230	import string
2231
2232	try:
2233	R = Reason_msg[reason]
2234	except:
2235	R = 'Unknown'
2236
2237	print string.ljust('\| Reason:', textwidth1) + R
2238
2239	# -----------------------------------------------------------------------------
2240
2241	def print_header_box(line):
2242	"""Print line in a nice box.
2243
2244	USAGE:
2245	print_header_box(line)
2246
2247	"""
2248	global textwidth3
2249
2250	import time
2251
2252	time_stamp = time.ctime(time.time())
2253	line = time_stamp + '. ' + line
2254
2255	N = len(line) + 1
2256	s = '+' + '-'*N + CR
2257
2258	print s + '\| ' + line + CR + s,
2259
2260	textwidth3 = N
2261
2262	# -----------------------------------------------------------------------------
2263
2264	def print_footer():
2265	"""Print line same width as that of print_header_box.
2266	"""
2267
2268	N = textwidth3
2269	s = '+' + '-'*N + CR
2270
2271	print s
2272
2273	# -----------------------------------------------------------------------------
2274
2275	def mkargstr(args, textwidth, argstr = ''):
2276	""" Generate a string containing first textwidth characters of arguments.
2277
2278	USAGE:
2279	mkargstr(args, textwidth, argstr = '')
2280
2281	DESCRIPTION:
2282	Exactly the same as str(args) possibly followed by truncation,
2283	but faster if args is huge.
2284	"""
2285
2286	import types
2287
2288	WasTruncated = 0
2289
2290	if not type(args) in [types.TupleType, types.ListType, types.DictType]:
2291	if type(args) == types.StringType:
2292	argstr = argstr + "'"+str(args)+"'"
2293	else:
2294	#Truncate large Numeric arrays before using str()
2295	import Numeric
2296	if type(args) == Numeric.ArrayType:
2297	if len(args.flat) > textwidth:
2298	args = 'Array: ' + str(args.shape)
2299
2300	argstr = argstr + str(args)
2301	else:
2302	if type(args) == types.DictType:
2303	argstr = argstr + "{"
2304	for key in args.keys():
2305	argstr = argstr + mkargstr(key, textwidth) + ": " + \
2306	mkargstr(args[key], textwidth) + ", "
2307	if len(argstr) > textwidth:
2308	WasTruncated = 1
2309	break
2310	argstr = argstr[:-2] # Strip off trailing comma
2311	argstr = argstr + "}"
2312
2313	else:
2314	if type(args) == types.TupleType:
2315	lc = '('
2316	rc = ')'
2317	else:
2318	lc = '['
2319	rc = ']'
2320	argstr = argstr + lc
2321	for arg in args:
2322	argstr = argstr + mkargstr(arg, textwidth) + ', '
2323	if len(argstr) > textwidth:
2324	WasTruncated = 1
2325	break
2326
2327	# Strip off trailing comma and space unless singleton tuple
2328	#
2329	if type(args) == types.TupleType and len(args) == 1:
2330	argstr = argstr[:-1]
2331	else:
2332	argstr = argstr[:-2]
2333	argstr = argstr + rc
2334
2335	if len(argstr) > textwidth:
2336	WasTruncated = 1
2337
2338	if WasTruncated:
2339	argstr = argstr[:textwidth]+'...'
2340	return(argstr)
2341
2342	# -----------------------------------------------------------------------------
2343
2344	def test_OK(msg):
2345	"""Print OK msg if test is OK.
2346
2347	USAGE
2348	test_OK(message)
2349	"""
2350
2351	import string
2352
2353	print string.ljust(msg, textwidth4) + ' - OK'
2354
2355	#raise StandardError
2356
2357	# -----------------------------------------------------------------------------
2358
2359	def test_error(msg):
2360	"""Print error if test fails.
2361
2362	USAGE
2363	test_error(message)
2364	"""
2365
2366	print 'ERROR (caching.test): %s' %msg
2367	print 'Please send this code example and output to '
2368	print 'Ole.Nielsen@anu.edu.au'
2369	print
2370	print
2371
2372	#import sys
2373	#sys.exit()
2374	raise StandardError

Note: See TracBrowser for help on using the repository browser.

Download in other formats: