Context Navigation

source: anuga_core/source/anuga/caching/caching.py @ 5856

Last change on this file since 5856 was 5856, checked in by ole, 15 years ago

A bit more fiddling with caching and circular structures not
involving instances.

All relevant tests pass

File size: 72.7 KB

Rev	Line
[130]	1	# =============================================================================
	2	# caching.py - Supervised caching of function results.
	3	# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
[1831]	4	# Australian National University (1999-2003)
	5	# Geoscience Australia (2003-present)
[130]	6	#
	7	# This program is free software; you can redistribute it and/or modify
	8	# it under the terms of the GNU General Public License as published by
	9	# the Free Software Foundation; either version 2 of the License, or
	10	# (at your option) any later version.
	11	#
	12	# This program is distributed in the hope that it will be useful,
	13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	# GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
	16	# for more details.
	17	#
	18	# You should have received a copy of the GNU General Public License
	19	# along with this program; if not, write to the Free Software
	20	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
	21	#
	22	#
[1831]	23	# Contact address: Ole.Nielsen@ga.gov.au
[130]	24	#
	25	# Version 1.5.6 February 2002
	26	# =============================================================================
	27
	28	"""Module caching.py - Supervised caching of function results.
	29
	30	Public functions:
	31
	32	cache(func,args) -- Cache values returned from func given args.
	33	cachestat() -- Reports statistics about cache hits and time saved.
	34	test() -- Conducts a basic test of the caching functionality.
	35
	36	See doc strings of individual functions for detailed documentation.
	37	"""
	38
	39	# -----------------------------------------------------------------------------
	40	# Initialisation code
	41
	42	# Determine platform
	43	#
[5359]	44	from os import getenv
[5854]	45	import types
[5359]	46
[130]	47	import os
	48	if os.name in ['nt', 'dos', 'win32', 'what else?']:
	49	unix = 0
	50	else:
	51	unix = 1
	52
[5359]	53	cache_dir = '.python_cache'
[130]	54	# Make default caching directory name
	55	#
	56	if unix:
[5359]	57	homedir = getenv('INUNDATIONHOME')
	58	if homedir is None:
	59	homedir = '~'
	60	else:
	61	homedir = homedir + os.sep + '.cache'
	62	# Since homedir will be a group area, individually label the caches
	63	user = getenv('LOGNAME')
	64	if user is not None:
	65	cache_dir += '_' + user
	66
[130]	67	CR = '\n'
	68	else:
	69	homedir = 'c:'
	70	CR = '\r\n' #FIXME: Not tested under windows
	71
[5359]	72	cachedir = homedir + os.sep + cache_dir + os.sep
[130]	73
	74	# -----------------------------------------------------------------------------
	75	# Options directory with default values - to be set by user
	76	#
	77
	78	options = {
	79	'cachedir': cachedir, # Default cache directory
	80	'maxfiles': 1000000, # Maximum number of cached files
	81	'savestat': 1, # Log caching info to stats file
	82	'verbose': 1, # Write messages to standard output
	83	'bin': 1, # Use binary format (more efficient)
[1831]	84	'compression': 1, # Use zlib compression
[130]	85	'bytecode': 0, # Recompute if bytecode has changed
	86	'expire': 0 # Automatically remove files that have been accessed
	87	# least recently
	88	}
	89
	90	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	91
	92	def set_option(key, value):
	93	"""Function to set values in the options directory.
	94
	95	USAGE:
	96	set_option(key, value)
	97
	98	ARGUMENTS:
	99	key -- Key in options dictionary. (Required)
	100	value -- New value for key. (Required)
	101
	102	DESCRIPTION:
	103	Function to set values in the options directory.
	104	Raises an exception if key is not in options.
	105	"""
	106
	107	if options.has_key(key):
	108	options[key] = value
	109	else:
	110	raise KeyError(key) # Key not found, raise an exception
	111
	112	# -----------------------------------------------------------------------------
	113	# Function cache - the main routine
	114
	115	def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
	116	verbose=None, compression=None, evaluate=0, test=0, clear=0,
	117	return_filename=0):
	118	"""Supervised caching of function results.
	119
	120	USAGE:
	121	result = cache(func, args, kwargs, dependencies, cachedir, verbose,
	122	compression, evaluate, test, return_filename)
	123
	124	ARGUMENTS:
	125	func -- Function object (Required)
	126	args -- Arguments to func (Default: ())
	127	kwargs -- Keyword arguments to func (Default: {})
	128	dependencies -- Filenames that func depends on (Default: None)
	129	cachedir -- Directory for cache files (Default: options['cachedir'])
	130	verbose -- Flag verbose output to stdout
	131	(Default: options['verbose'])
	132	compression -- Flag zlib compression (Default: options['compression'])
	133	evaluate -- Flag forced evaluation of func (Default: 0)
	134	test -- Flag test for cached results (Default: 0)
	135	clear -- Flag delete cached results (Default: 0)
	136	return_filename -- Flag return of cache filename (Default: 0)
	137
	138	DESCRIPTION:
	139	A Python function call of the form
	140
	141	result = func(arg1,...,argn)
	142
	143	can be replaced by
	144
	145	from caching import cache
	146	result = cache(func,(arg1,...,argn))
	147
	148	The latter form returns the same output as the former but reuses cached
	149	results if the function has been computed previously in the same context.
	150	'result' and the arguments can be simple types, tuples, list, dictionaries or
	151	objects, but not unhashable types such as functions or open file objects.
	152	The function 'func' may be a member function of an object or a module.
	153
	154	This type of caching is particularly useful for computationally intensive
	155	functions with few frequently used combinations of input arguments. Note that
	156	if the inputs or output are very large caching might not save time because
	157	disc access may dominate the execution time.
	158
	159	If the function definition changes after a result has been cached it will be
	160	detected by examining the functions bytecode (co_code, co_consts,
	161	func_defualts, co_argcount) and it will be recomputed.
	162
	163	LIMITATIONS:
	164	1 Caching uses the apply function and will work with anything that can be
	165	pickled, so any limitation in apply or pickle extends to caching.
	166	2 A function to be cached should not depend on global variables
	167	as wrong results may occur if globals are changed after a result has
	168	been cached.
	169
	170	-----------------------------------------------------------------------------
	171	Additional functionality:
	172
	173	Keyword args
	174	Keyword arguments (kwargs) can be added as a dictionary of keyword: value
	175	pairs, following the syntax of the built-in function apply().
	176	A Python function call of the form
	177
	178	result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)
	179
	180	is then cached as follows
	181
	182	from caching import cache
	183	result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
	184
	185	The default value of kwargs is {}
	186
	187	Explicit dependencies:
	188	The call
	189	cache(func,(arg1,...,argn),dependencies = <list of filenames>)
	190	Checks the size, creation time and modification time of each listed file.
	191	If any file has changed the function is recomputed and the results stored
	192	again.
	193
	194	Specify caching directory:
	195	The call
	196	cache(func,(arg1,...,argn), cachedir = <cachedir>)
	197	designates <cachedir> where cached data are stored. Use ~ to indicate users
	198	home directory - not $HOME. The default is ~/.python_cache on a UNIX
	199	platform and c:/.python_cache on a Win platform.
	200
	201	Silent operation:
	202	The call
	203	cache(func,(arg1,...,argn),verbose=0)
	204	suppresses messages to standard output.
	205
	206	Compression:
	207	The call
	208	cache(func,(arg1,...,argn),compression=0)
	209	disables compression. (Default: compression=1). If the requested compressed
	210	or uncompressed file is not there, it'll try the other version.
	211
	212	Forced evaluation:
	213	The call
	214	cache(func,(arg1,...,argn),evaluate=1)
	215	forces the function to evaluate even though cached data may exist.
	216
	217	Testing for presence of cached result:
	218	The call
	219	cache(func,(arg1,...,argn),test=1)
	220	retrieves cached result if it exists, otherwise None. The function will not
	221	be evaluated. If both evaluate and test are switched on, evaluate takes
	222	precedence.
	223
	224	Obtain cache filenames:
	225	The call
	226	cache(func,(arg1,...,argn),return_filename=1)
	227	returns the hashed base filename under which this function and its
	228	arguments would be cached
	229
	230	Clearing cached results:
	231	The call
	232	cache(func,'clear')
	233	clears all cached data for 'func' and
	234	cache('clear')
	235	clears all cached data.
	236
	237	NOTE: The string 'clear' can be passed an argument to func using
	238	cache(func,('clear',)) or cache(func,tuple(['clear'])).
	239
	240	New form of clear:
	241	cache(func,(arg1,...,argn),clear=1)
	242	clears cached data for particular combination func and args
	243
	244	"""
	245
	246	# Imports and input checks
	247	#
	248	import types, time, string
	249
	250	if not cachedir:
	251	cachedir = options['cachedir']
	252
	253	if verbose == None: # Do NOT write 'if not verbose:', it could be zero.
	254	verbose = options['verbose']
	255
	256	if compression == None: # Do NOT write 'if not compression:',
	257	# it could be zero.
	258	compression = options['compression']
	259
	260	# Create cache directory if needed
	261	CD = checkdir(cachedir,verbose)
	262
	263	# Handle the case cache('clear')
	264	if type(func) == types.StringType:
	265	if string.lower(func) == 'clear':
	266	clear_cache(CD,verbose=verbose)
	267	return
	268
	269	# Handle the case cache(func, 'clear')
	270	if type(args) == types.StringType:
	271	if string.lower(args) == 'clear':
	272	clear_cache(CD,func,verbose=verbose)
	273	return
	274
	275	# Force singleton arg into a tuple
	276	if type(args) != types.TupleType:
	277	args = tuple([args])
	278
	279	# Check that kwargs is a dictionary
	280	if type(kwargs) != types.DictType:
	281	raise TypeError
	282
	283	# Hash arguments (and keyword args) to integer
[5855]	284	arghash = myhash((args, kwargs))
[5854]	285
[130]	286	# Get sizes and timestamps for files listed in dependencies.
	287	# Force singletons into a tuple.
	288	if dependencies and type(dependencies) != types.TupleType \
	289	and type(dependencies) != types.ListType:
	290	dependencies = tuple([dependencies])
	291	deps = get_depstats(dependencies)
	292
	293	# Extract function name from func object
	294	funcname = get_funcname(func)
	295
	296	# Create cache filename
	297	FN = funcname+'['+`arghash`+']' # The symbol '(' does not work under unix
	298
	299	if return_filename:
	300	return(FN)
	301
	302	if clear:
	303	for file_type in file_types:
	304	file_name = CD+FN+'_'+file_type
	305	for fn in [file_name, file_name + '.z']:
	306	if os.access(fn, os.F_OK):
	307	if unix:
	308	os.remove(fn)
	309	else:
	310	# FIXME: os.remove doesn't work under windows
	311	os.system('del '+fn)
[3715]	312	if verbose is True:
[130]	313	print 'MESSAGE (caching): File %s deleted' %fn
	314	##else:
	315	## print '%s was not accessed' %fn
	316	return None
	317
	318
	319	#-------------------------------------------------------------------
	320
	321	# Check if previous computation has been cached
	322	if evaluate:
	323	Retrieved = None # Force evaluation of func regardless of caching status.
[5854]	324	reason = 5
[130]	325	else:
[5855]	326	T, FN, Retrieved, reason, comptime, loadtime, compressed = \
	327	CacheLookup(CD, FN, func,
	328	args, kwargs,
	329	deps,
	330	verbose,
	331	compression,
	332	dependencies)
[130]	333
	334	if not Retrieved:
	335	if test: # Do not attempt to evaluate function
	336	T = None
	337	else: # Evaluate function and save to cache
	338	if verbose:
[4277]	339
[130]	340	msg1(funcname, args, kwargs,reason)
	341
	342	# Remove expired files automatically
	343	if options['expire']:
	344	DeleteOldFiles(CD,verbose)
	345
	346	# Save args before function is evaluated in case
	347	# they are modified by function
	348	save_args_to_cache(CD,FN,args,kwargs,compression)
	349
	350	# Execute and time function with supplied arguments
	351	t0 = time.time()
	352	T = apply(func,args,kwargs)
	353	#comptime = round(time.time()-t0)
	354	comptime = time.time()-t0
	355
	356	if verbose:
	357	msg2(funcname,args,kwargs,comptime,reason)
	358
	359	# Save results and estimated loading time to cache
	360	loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
	361	funcname, dependencies, compression)
	362	if verbose:
	363	msg3(loadtime, CD, FN, deps, compression)
	364	compressed = compression
	365
	366	if options['savestat'] and (not test or Retrieved):
	367	##if options['savestat']:
	368	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
	369
	370	return(T) # Return results in all cases
	371
	372	# -----------------------------------------------------------------------------
	373
	374	def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
	375	"""Generate statistics of caching efficiency.
	376
	377	USAGE:
	378	cachestat(sortidx, period, showuser, cachedir)
	379
	380	ARGUMENTS:
	381	sortidx -- Index of field by which lists are (default: 4)
	382	Legal values are
	383	0: 'Name'
	384	1: 'Hits'
	385	2: 'CPU'
	386	3: 'Time Saved'
	387	4: 'Gain(%)'
	388	5: 'Size'
	389	period -- If set to -1 all available caching history is used.
	390	If set 0 only the current month is used (default -1).
	391	showuser -- Flag for additional table showing user statistics
	392	(default: None).
	393	cachedir -- Directory for cache files (default: options['cachedir']).
	394
	395	DESCRIPTION:
	396	Logged caching statistics is converted into summaries of the form
	397	--------------------------------------------------------------------------
	398	Function Name Hits Exec(s) Cache(s) Saved(s) Gain(%) Size
	399	--------------------------------------------------------------------------
	400	"""
	401
	402	__cachestat(sortidx, period, showuser, cachedir)
	403	return
	404
	405	# -----------------------------------------------------------------------------
	406
[2043]	407	#Has mostly been moved to proper unit test
[130]	408	def test(cachedir=None,verbose=0,compression=None):
	409	"""Test the functionality of caching.
	410
	411	USAGE:
	412	test(verbose)
	413
	414	ARGUMENTS:
	415	verbose -- Flag whether caching will output its statistics (default=0)
	416	cachedir -- Directory for cache files (Default: options['cachedir'])
	417	compression -- Flag zlib compression (Default: options['compression'])
	418	"""
	419
	420	import string, time
	421
	422	# Initialise
	423	#
	424	import caching
	425	reload(caching)
	426
	427	if not cachedir:
	428	cachedir = options['cachedir']
	429
	430	if verbose is None: # Do NOT write 'if not verbose:', it could be zero.
	431	verbose = options['verbose']
	432
	433	if compression == None: # Do NOT write 'if not compression:',
	434	# it could be zero.
	435	compression = options['compression']
	436	else:
	437	try:
	438	set_option('compression', compression)
	439	except:
	440	test_error('Set option failed')
	441
	442	try:
	443	import zlib
	444	except:
	445	print
	446	print '* Could not find zlib, default to no-compression *'
	447	print '* Installing zlib will improve performance of caching *'
	448	print
	449	compression = 0
	450	set_option('compression', compression)
	451
	452	print
	453	print_header_box('Testing caching module - please stand by')
	454	print
	455
	456	# Define a test function to be cached
	457	#
	458	def f(a,b,c,N,x=0,y='abcdefg'):
	459	"""f(a,b,c,N)
	460	Do something time consuming and produce a complex result.
	461	"""
	462
	463	import string
	464
	465	B = []
	466	for n in range(N):
	467	s = str(n+2.0/(n + 4.0))+'.a'*10
	468	B.append((a,b,c,s,n,x,y))
	469	return(B)
	470
	471	# Check that default cachedir is OK
	472	#
	473	CD = checkdir(cachedir,verbose)
	474
	475
	476	# Make a dependency file
	477	#
	478	try:
	479	DepFN = CD + 'testfile.tmp'
	480	DepFN_wildcard = CD + 'test*.tmp'
	481	Depfile = open(DepFN,'w')
	482	Depfile.write('We are the knights who say NI!')
	483	Depfile.close()
	484	test_OK('Wrote file %s' %DepFN)
	485	except:
	486	test_error('Could not open file %s for writing - check your environment' \
	487	% DepFN)
	488
	489	# Check set_option (and switch stats off
	490	#
	491	try:
	492	set_option('savestat',0)
	493	assert(options['savestat'] == 0)
	494	test_OK('Set option')
	495	except:
	496	test_error('Set option failed')
	497
	498	# Make some test input arguments
	499	#
	500	N = 5000 #Make N fairly small here
	501
	502	a = [1,2]
	503	b = ('Thou shalt count the number three',4)
	504	c = {'Five is right out': 6, (7,8): 9}
	505	x = 3
	506	y = 'holy hand granate'
	507
	508	# Test caching
	509	#
	510	if compression:
	511	comprange = 2
	512	else:
	513	comprange = 1
	514
	515	for comp in range(comprange):
	516
	517	# Evaluate and store
	518	#
	519	try:
	520	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
	521	verbose=verbose, compression=comp)
	522	if comp:
	523	test_OK('Caching evaluation with compression')
	524	else:
	525	test_OK('Caching evaluation without compression')
	526	except:
	527	if comp:
	528	test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
	529	else:
	530	test_error('Caching evaluation failed - try caching.test(verbose=1)')
	531
	532	# Retrieve
	533	#
	534	try:
	535	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	536	compression=comp)
	537
	538	if comp:
	539	test_OK('Caching retrieval with compression')
	540	else:
	541	test_OK('Caching retrieval without compression')
	542	except:
	543	if comp:
	544	test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
	545	else:
	546	test_error('Caching retrieval failed - try caching.test(verbose=1)')
	547
	548	# Reference result
	549	#
	550	T3 = f(a,b,c,N,x=x,y=y) # Compute without caching
	551
	552	if T1 == T2 and T2 == T3:
	553	if comp:
	554	test_OK('Basic caching functionality (with compression)')
	555	else:
	556	test_OK('Basic caching functionality (without compression)')
	557	else:
	558	test_error('Cached result does not match computed result')
	559
	560
	561	# Test return_filename
	562	#
	563	try:
	564	FN = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	565	return_filename=1)
	566	assert(FN[:2] == 'f[')
	567	test_OK('Return of cache filename')
	568	except:
	569	test_error('Return of cache filename failed')
	570
	571	# Test existence of cachefiles
	572	#
	573	try:
	574	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
	575	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
	576	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
	577	test_OK('Presence of cache files')
	578	datafile.close()
	579	argsfile.close()
	580	admfile.close()
	581	except:
	582	test_error('Expected cache files did not exist')
	583
	584	# Test 'test' function when cache is present
	585	#
	586	try:
	587	#T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	588	# evaluate=1)
	589	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
	590	assert(T1 == T4)
	591
	592	test_OK("Option 'test' when cache file present")
	593	except:
	594	test_error("Option 'test' when cache file present failed")
	595
	596	# Test that 'clear' works
	597	#
	598	#try:
	599	# caching.cache(f,'clear',verbose=verbose)
	600	# test_OK('Clearing of cache files')
	601	#except:
	602	# test_error('Clear does not work')
	603	try:
	604	caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)
	605	test_OK('Clearing of cache files')
	606	except:
	607	test_error('Clear does not work')
	608
	609
	610
	611	# Test 'test' function when cache is absent
	612	#
	613	try:
	614	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
	615	assert(T4 is None)
	616	test_OK("Option 'test' when cache absent")
	617	except:
	618	test_error("Option 'test' when cache absent failed")
	619
	620	# Test dependencies
	621	#
	622	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	623	dependencies=DepFN)
	624	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	625	dependencies=DepFN)
	626
	627	if T1 == T2:
	628	test_OK('Basic dependencies functionality')
	629	else:
	630	test_error('Dependencies do not work')
	631
	632	# Test basic wildcard dependency
	633	#
	634	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	635	dependencies=DepFN_wildcard)
	636
	637	if T1 == T3:
	638	test_OK('Basic dependencies with wildcard functionality')
	639	else:
	640	test_error('Dependencies with wildcards do not work')
	641
	642
	643	# Test that changed timestamp in dependencies triggers recomputation
	644
	645	# Modify dependency file
	646	Depfile = open(DepFN,'a')
	647	Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
	648	Depfile.close()
	649
	650	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	651	dependencies=DepFN, test = 1)
	652
	653	if T3 is None:
	654	test_OK('Changed dependencies recognised')
	655	else:
	656	test_error('Changed dependencies not recognised')
	657
	658	# Test recomputation when dependencies have changed
	659	#
	660	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	661	dependencies=DepFN)
	662	if T1 == T3:
	663	test_OK('Recomputed value with changed dependencies')
	664	else:
	665	test_error('Recomputed value with changed dependencies failed')
[1831]	666
[130]	667	# Performance test (with statistics)
	668	# Don't really rely on this as it will depend on specific computer.
	669	#
	670
	671	set_option('savestat',1)
	672
	673	N = 20*N #Should be large on fast computers...
	674	tt = time.time()
	675	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
	676	t1 = time.time() - tt
	677
	678	tt = time.time()
	679	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
	680	t2 = time.time() - tt
	681
	682	if T1 == T2:
	683	if t1 > t2:
	684	test_OK('Performance test: relative time saved = %s pct' \
	685	%str(round((t1-t2)*100/t1,2)))
	686	#else:
	687	# print 'WARNING: Performance a bit low - this could be specific to current platform'
	688	else:
	689	test_error('Basic caching failed for new problem')
	690
	691	# Test presence of statistics file
	692	#
	693	try:
	694	DIRLIST = os.listdir(CD)
	695	SF = []
	696	for FN in DIRLIST:
	697	if string.find(FN,statsfile) >= 0:
	698	fid = open(CD+FN,'r')
	699	fid.close()
	700	test_OK('Statistics files present')
	701	except:
	702	test_OK('Statistics files cannot be opened')
	703
	704	print_header_box('Show sample output of the caching function:')
	705
	706	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
	707	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
	708	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
	709
	710	print_header_box('Show sample output of cachestat():')
	711	if unix:
	712	cachestat()
	713	else:
	714	try:
	715	import time
	716	t = time.strptime('2030','%Y')
	717	cachestat()
	718	except:
	719	print 'caching.cachestat() does not work here, because it'
	720	print 'relies on time.strptime() which is unavailable in Windows'
	721
	722	print
	723	test_OK('Caching self test completed')
	724
	725
	726	# Test setoption (not yet implemented)
	727	#
[1831]	728
[130]	729
	730	#==============================================================================
	731	# Auxiliary functions
	732	#==============================================================================
	733
	734	# Import pickler
	735	# cPickle is used by functions mysave, myload, and compare
	736	#
	737	import cPickle # 10 to 100 times faster than pickle
	738	pickler = cPickle
	739
	740	# Local immutable constants
	741	#
	742	comp_level = 1 # Compression level for zlib.
	743	# comp_level = 1 works well.
	744	textwidth1 = 16 # Text width of key fields in report forms.
[4277]	745	#textwidth2 = 132 # Maximal width of textual representation of
	746	textwidth2 = 300 # Maximal width of textual representation of
[130]	747	# arguments.
	748	textwidth3 = 16 # Initial width of separation lines. Is modified.
	749	textwidth4 = 50 # Text width in test_OK()
	750	statsfile = '.cache_stat' # Basefilename for cached statistics.
	751	# It will reside in the chosen cache directory.
	752
	753	file_types = ['Result', # File name extension for cached function results.
	754	'Args', # File name extension for stored function args.
	755	'Admin'] # File name extension for administrative info.
	756
	757	Reason_msg = ['OK', # Verbose reasons for recomputation
	758	'No cached result',
	759	'Dependencies have changed',
[5854]	760	'Arguments have changed',
	761	'Bytecode has changed',
[130]	762	'Recomputation was requested by caller',
	763	'Cached file was unreadable']
	764
	765	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	766
	767	def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression,
	768	dependencies):
	769	"""Determine whether cached result exists and return info.
	770
	771	USAGE:
	772	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
	773	CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
	774	dependencies)
	775
	776	INPUT ARGUMENTS:
	777	CD -- Cache Directory
	778	FN -- Suggested cache file name
	779	func -- Function object
	780	args -- Tuple of arguments
	781	kwargs -- Dictionary of keyword arguments
	782	deps -- Dependencies time stamps
	783	verbose -- Flag text output
	784	compression -- Flag zlib compression
	785	dependencies -- Given list of dependencies
	786
	787	OUTPUT ARGUMENTS:
	788	T -- Cached result if present otherwise None
	789	FN -- File name under which new results must be saved
	790	Retrieved -- True if a valid cached result was found
	791	reason -- 0: OK (if Retrieved),
	792	1: No cached result,
	793	2: Dependencies have changed,
[5854]	794	3: Arguments have changed
	795	4: Bytecode has changed
	796	5: Recomputation was forced
	797	6: Unreadable file
[130]	798	comptime -- Number of seconds it took to computed cachged result
	799	loadtime -- Number of seconds it took to load cached result
	800	compressed -- Flag (0,1) if cached results were compressed or not
	801
	802	DESCRIPTION:
	803	Determine if cached result exists as follows:
	804	Load in saved arguments and bytecode stored under hashed filename.
	805	If they are identical to current arguments and bytecode and if dependencies
	806	have not changed their time stamp, then return cached result.
	807
	808	Otherwise return filename under which new results should be cached.
	809	Hash collisions are handled recursively by calling CacheLookup again with a
	810	modified filename.
	811	"""
	812
	813	import time, string, types
	814
	815	# Assess whether cached result exists - compressed or not.
	816	#
	817	if verbose:
	818	print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
	819	%(CD+FN, file_types[0], file_types[1], file_types[2])
	820	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
	821	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
	822	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
	823
[4425]	824	if verbose is True and deps is not None:
	825	print 'Caching: Dependencies are', deps.keys()
	826
[130]	827	if not (argsfile and datafile and admfile) or \
	828	not (compressed0 == compressed1 and compressed0 == compressed2):
	829	# Cached result does not exist or files were compressed differently
	830	#
	831	# This will ensure that evaluation will take place unless all files are
	832	# present.
	833
	834	reason = 1
	835	return(None,FN,None,reason,None,None,None) #Recompute using same filename
	836
	837	compressed = compressed0 # Remember if compressed files were actually used
	838	datafile.close()
	839
	840	# Retrieve arguments and adm. info
	841	#
[5855]	842	R, reason = myload(argsfile, compressed) # The original arguments
[130]	843	argsfile.close()
	844
	845	if reason > 0:
[5855]	846	# Recompute using same filename
	847	return(None, FN, None, reason, None, None, None)
[130]	848	else:
[5855]	849	(argsref, kwargsref) = R
[130]	850
[5855]	851	R, reason = myload(admfile, compressed)
[130]	852	admfile.close()
[5855]	853
[130]	854	if reason > 0:
	855	return(None,FN,None,reason,None,None,None) #Recompute using same filename
	856
	857
	858	depsref = R[0] # Dependency statistics
	859	comptime = R[1] # The computation time
	860	coderef = R[2] # The byte code
	861	funcname = R[3] # The function name
	862
	863	# Check if dependencies have changed
	864	#
[5855]	865	if dependencies and not compare(depsref, deps):
[130]	866	if verbose:
	867	print 'MESSAGE (caching.py): Dependencies', dependencies, \
	868	'have changed - recomputing'
	869	# Don't use cached file - recompute
	870	reason = 2
[5855]	871	return(None, FN, None, reason, None, None, None)
[130]	872
	873	# Get bytecode from func
	874	#
	875	bytecode = get_bytecode(func)
	876
[5855]	877	#print compare(argsref,args),
[130]	878	#print compare(kwargsref,kwargs),
	879	#print compare(bytecode,coderef)
	880
	881	# Check if arguments or bytecode have changed
[5855]	882	if compare(argsref, args) and compare(kwargsref, kwargs) and \
	883	(not options['bytecode'] or compare(bytecode, coderef)):
[130]	884
	885	# Arguments and dependencies match. Get cached results
[5855]	886	T, loadtime, compressed, reason = load_from_cache(CD, FN, compressed)
[130]	887	if reason > 0:
[5855]	888	# Recompute using same FN
	889	return(None, FN, None, reason, None, None, None)
[130]	890
	891	Retrieved = 1
	892	reason = 0
	893
	894	if verbose:
	895	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
	896
	897	if loadtime >= comptime:
	898	print 'WARNING (caching.py): Caching did not yield any gain.'
	899	print ' Consider executing function ',
	900	print '('+funcname+') without caching.'
	901	else:
	902
	903	# Non matching arguments or bytecodes signify a hash-collision.
	904	# This is resolved by recursive search of cache filenames
	905	# until either a matching or an unused filename is found.
	906	#
[5855]	907	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
	908	CacheLookup(CD, FN+'x', func, args, kwargs, deps,
	909	verbose, compression, dependencies)
[130]	910
	911	# DEBUGGING
	912	# if not Retrieved:
	913	# print 'Arguments did not match'
	914	# else:
	915	# print 'Match found !'
[5854]	916
	917	# The real reason is that args or bytecodes have changed.
	918	# Not that the recursive seach has found an unused filename
[130]	919	if not Retrieved:
[5855]	920	if not compare(bytecode, coderef):
[5854]	921	reason = 4 # Bytecode has changed
	922	else:
	923	reason = 3 # Arguments have changed
	924
[130]	925
	926	return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
	927
	928	# -----------------------------------------------------------------------------
	929
	930	def clear_cache(CD,func=None, verbose=None):
	931	"""Clear cache for func.
	932
	933	USAGE:
	934	clear(CD, func, verbose)
	935
	936	ARGUMENTS:
	937	CD -- Caching directory (required)
	938	func -- Function object (default: None)
	939	verbose -- Flag verbose output (default: None)
	940
	941	DESCRIPTION:
	942
	943	If func == None, clear everything,
	944	otherwise clear only files pertaining to func.
	945	"""
	946
	947	import os, re
	948
	949	if CD[-1] != os.sep:
	950	CD = CD+os.sep
	951
	952	if verbose == None:
	953	verbose = options['verbose']
	954
	955	# FIXME: Windows version needs to be tested
	956
	957	if func:
	958	funcname = get_funcname(func)
	959	if verbose:
	960	print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
	961
	962	file_names = os.listdir(CD)
	963	for file_name in file_names:
	964	#RE = re.search('^' + funcname,file_name) #Inefficient
	965	#if RE:
	966	if file_name[:len(funcname)] == funcname:
	967	if unix:
	968	os.remove(CD+file_name)
	969	else:
	970	os.system('del '+CD+file_name)
	971	# FIXME: os.remove doesn't work under windows
	972	else:
	973	file_names = os.listdir(CD)
	974	if len(file_names) > 0:
	975	if verbose:
	976	print 'MESSAGE (caching.py): Remove the following files:'
	977	for file_name in file_names:
	978	print file_name
	979
	980	A = raw_input('Delete (Y/N)[N] ?')
	981	else:
	982	A = 'Y'
	983
	984	if A == 'Y' or A == 'y':
	985	for file_name in file_names:
	986	if unix:
	987	os.remove(CD+file_name)
	988	else:
	989	os.system('del '+CD+file_name)
	990	# FIXME: os.remove doesn't work under windows
	991	#exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
	992
	993	# -----------------------------------------------------------------------------
	994
	995	def DeleteOldFiles(CD,verbose=None):
	996	"""Remove expired files
	997
	998	USAGE:
	999	DeleteOldFiles(CD,verbose=None)
	1000	"""
	1001
	1002	if verbose == None:
	1003	verbose = options['verbose']
	1004
	1005	maxfiles = options['maxfiles']
	1006
	1007	# FIXME: Windows version
	1008
	1009	import os
	1010	block = 1000 # How many files to delete per invokation
	1011	Files = os.listdir(CD)
	1012	numfiles = len(Files)
	1013	if not unix: return # FIXME: Windows case ?
	1014
	1015	if numfiles > maxfiles:
	1016	delfiles = numfiles-maxfiles+block
	1017	if verbose:
	1018	print 'Deleting '+`delfiles`+' expired files:'
	1019	os.system('ls -lur '+CD+'* \| head -' + `delfiles`) # List them
	1020	os.system('ls -ur '+CD+'* \| head -' + `delfiles` + ' \| xargs /bin/rm')
	1021	# Delete them
	1022	# FIXME: Replace this with os.listdir and os.remove
	1023
	1024	# -----------------------------------------------------------------------------
	1025
[5791]	1026	def save_args_to_cache(CD, FN, args, kwargs, compression):
[130]	1027	"""Save arguments to cache
	1028
	1029	USAGE:
	1030	save_args_to_cache(CD,FN,args,kwargs,compression)
	1031	"""
	1032
	1033	import time, os, sys, types
	1034
	1035	(argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
	1036
[5791]	1037	if argsfile is None:
	1038	msg = 'ERROR (caching): Could not open argsfile for writing: %s' %FN
	1039	raise IOError, msg
[130]	1040
	1041	mysave((args,kwargs),argsfile,compression) # Save args and kwargs to cache
	1042	argsfile.close()
	1043
	1044	# Change access rights if possible
	1045	#
	1046	#if unix:
	1047	# try:
	1048	# exitcode=os.system('chmod 666 '+argsfile.name)
	1049	# except:
	1050	# pass
	1051	#else:
	1052	# pass # FIXME: Take care of access rights under Windows
	1053
	1054	return
	1055
	1056	# -----------------------------------------------------------------------------
	1057
	1058	def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
	1059	dependencies, compression):
	1060	"""Save computed results T and admin info to cache
	1061
	1062	USAGE:
	1063	save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
	1064	dependencies, compression)
	1065	"""
	1066
	1067	import time, os, sys, types
	1068
	1069	(datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
	1070	(admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
	1071
	1072	if not datafile:
	1073	if verbose:
	1074	print 'ERROR (caching): Could not open %s' %datafile.name
	1075	raise IOError
	1076
	1077	if not admfile:
	1078	if verbose:
	1079	print 'ERROR (caching): Could not open %s' %admfile.name
	1080	raise IOError
	1081
	1082	t0 = time.time()
	1083
	1084	mysave(T,datafile,compression) # Save data to cache
	1085	datafile.close()
	1086	#savetime = round(time.time()-t0,2)
	1087	savetime = time.time()-t0
	1088
	1089	bytecode = get_bytecode(func) # Get bytecode from function object
	1090	admtup = (deps, comptime, bytecode, funcname) # Gather admin info
	1091
	1092	mysave(admtup,admfile,compression) # Save admin info to cache
	1093	admfile.close()
	1094
	1095	# Change access rights if possible
	1096	#
	1097	#if unix:
	1098	# try:
	1099	# exitcode=os.system('chmod 666 '+datafile.name)
	1100	# exitcode=os.system('chmod 666 '+admfile.name)
	1101	# except:
	1102	# pass
	1103	#else:
	1104	# pass # FIXME: Take care of access rights under Windows
	1105
	1106	return(savetime)
	1107
	1108	# -----------------------------------------------------------------------------
	1109
	1110	def load_from_cache(CD,FN,compression):
	1111	"""Load previously cached data from file FN
	1112
	1113	USAGE:
	1114	load_from_cache(CD,FN,compression)
	1115	"""
	1116
	1117	import time
	1118
	1119	(datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
	1120	t0 = time.time()
	1121	T, reason = myload(datafile,compressed)
[5854]	1122
[130]	1123	loadtime = time.time()-t0
	1124	datafile.close()
	1125
	1126	return T, loadtime, compressed, reason
	1127
	1128	# -----------------------------------------------------------------------------
	1129
	1130	def myopen(FN,mode,compression=1):
	1131	"""Open file FN using given mode
	1132
	1133	USAGE:
	1134	myopen(FN,mode,compression=1)
	1135
	1136	ARGUMENTS:
	1137	FN -- File name to be opened
	1138	mode -- Open mode (as in open)
	1139	compression -- Flag zlib compression
	1140
	1141	DESCRIPTION:
	1142	if compression
	1143	Attempt first to open FN + '.z'
	1144	If this fails try to open FN
	1145	else do the opposite
	1146	Return file handle plus info about whether it was compressed or not.
	1147	"""
	1148
	1149	import string
	1150
	1151	# Determine if file exists already (if writing was requested)
	1152	# This info is only used to determine if access modes should be set
	1153	#
	1154	if 'w' in mode or 'a' in mode:
	1155	try:
	1156	file = open(FN+'.z','r')
	1157	file.close()
	1158	new_file = 0
	1159	except:
	1160	try:
	1161	file = open(FN,'r')
	1162	file.close()
	1163	new_file = 0
	1164	except:
	1165	new_file = 1
	1166	else:
	1167	new_file = 0 #Assume it exists if mode was not 'w'
	1168
	1169
	1170	compressed = 0
	1171	if compression:
	1172	try:
	1173	file = open(FN+'.z',mode)
	1174	compressed = 1
	1175	except:
	1176	try:
	1177	file = open(FN,mode)
	1178	except:
	1179	file = None
	1180	else:
	1181	try:
	1182	file = open(FN,mode)
	1183	except:
	1184	try:
	1185	file = open(FN+'.z',mode)
	1186	compressed = 1
	1187	except:
	1188	file = None
	1189
	1190	# Now set access rights if it is a new file
	1191	#
	1192	if file and new_file:
	1193	if unix:
	1194	exitcode=os.system('chmod 666 '+file.name)
	1195	else:
	1196	pass # FIXME: Take care of access rights under Windows
	1197
	1198	return(file,compressed)
	1199
	1200	# -----------------------------------------------------------------------------
	1201
	1202	def myload(file, compressed):
	1203	"""Load data from file
	1204
	1205	USAGE:
	1206	myload(file, compressed)
	1207	"""
	1208
	1209	reason = 0
	1210	try:
	1211	if compressed:
	1212	import zlib
	1213
	1214	RsC = file.read()
	1215	try:
	1216	Rs = zlib.decompress(RsC)
	1217	except:
	1218	# File "./caching.py", line 1032, in load_from_cache
	1219	# T = myload(datafile,compressed)
	1220	# File "./caching.py", line 1124, in myload
	1221	# Rs = zlib.decompress(RsC)
	1222	# zlib.error: Error -5 while decompressing data
	1223	#print 'ERROR (caching): Could not decompress ', file.name
	1224	#raise Exception
[5854]	1225	reason = 6 # Unreadable file
[130]	1226	return None, reason
	1227
	1228
	1229	del RsC # Free up some space
[5855]	1230	R = pickler.loads(Rs)
[130]	1231	else:
[2043]	1232	try:
	1233	R = pickler.load(file)
	1234	#except EOFError, e:
	1235	except:
	1236	#Catch e.g., file with 0 length or corrupted
[5854]	1237	reason = 6 # Unreadable file
[2043]	1238	return None, reason
	1239
[130]	1240	except MemoryError:
	1241	import sys
	1242	if options['verbose']:
	1243	print 'ERROR (caching): Out of memory while loading %s, aborting' \
	1244	%(file.name)
	1245
	1246	# Raise the error again for now
	1247	#
	1248	raise MemoryError
	1249
	1250	return R, reason
	1251
	1252	# -----------------------------------------------------------------------------
	1253
	1254	def mysave(T,file,compression):
	1255	"""Save data T to file
	1256
	1257	USAGE:
	1258	mysave(T,file,compression)
	1259
	1260	"""
	1261
	1262	bin = options['bin']
	1263
	1264	if compression:
	1265	try:
	1266	import zlib
	1267	except:
	1268	print
	1269	print '* Could not find zlib *'
	1270	print '* Try to run caching with compression off *'
	1271	print "* caching.set_option('compression', 0) *"
	1272	raise Exception
	1273
	1274
[1895]	1275	try:
[2043]	1276	Ts = pickler.dumps(T, bin)
[1895]	1277	except MemoryError:
	1278	msg = '****WARNING (caching.py): Could not pickle data for compression.'
[3514]	1279	msg += ' Try using compression = False'
	1280	raise MemoryError, msg
[1895]	1281	else:
[5854]	1282	# Compressed pickling
[2043]	1283	TsC = zlib.compress(Ts, comp_level)
[1895]	1284	file.write(TsC)
[130]	1285	else:
[5854]	1286	# Uncompressed pickling
[2043]	1287	pickler.dump(T, file, bin)
[130]	1288
[2043]	1289	# FIXME: This may not work on Windoze network drives.
	1290	# The error msg is IOError: [Errno 22] Invalid argument
	1291	# Testing with small files was OK, though.
	1292	# I think this is an OS problem.
	1293
	1294	# Excerpt from http://www.ultraseek.com/support/faqs/4173.html
[1895]	1295
[2043]	1296	# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
	1297	#
	1298	#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
	1299	#
	1300	#The following extract is taken from the site http://www.python.org:
	1301	#
	1302	#---------------------------------------------------------------------------------------------
	1303	#exception IOError
	1304	#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
	1305	#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
	1306	#---------------------------------------------------------------------------------------------
	1307	#
	1308	#The error code(s) that accompany exceptions are described at:
	1309	#http://www.python.org/dev/doc/devel//lib/module-errno.html
	1310	#
	1311	#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
	1312	#
	1313	#exceptions.IOError invalid argument Errno 22
	1314
	1315	#try:
	1316	# pickler.dump(T,file,bin)
	1317	#except IOError, e:
	1318	# print e
	1319	# msg = 'Could not store to %s, bin=%s' %(file, bin)
	1320	# raise msg
	1321
[1895]	1322
[130]	1323	# -----------------------------------------------------------------------------
	1324
[5855]	1325
[5854]	1326	def myhash(T, ids=None):
	1327	"""Compute hashed integer from a range of inputs.
	1328	If T is not hashable being e.g. a tuple T, myhash will recursively
	1329	hash the values individually
[130]	1330
	1331	USAGE:
	1332	myhash(T)
	1333
	1334	ARGUMENTS:
[5854]	1335	T -- Anything
[130]	1336	"""
	1337
[5854]	1338	from types import TupleType, ListType, DictType, InstanceType
[5855]	1339	from Numeric import ArrayType, average
[5856]	1340
	1341
	1342	if type(T) in [TupleType, ListType, DictType, InstanceType]:
[5854]	1343	# Keep track of unique id's to protect against infinite recursion
	1344	if ids is None: ids = []
[130]	1345
[5854]	1346	# Check if T has already been encountered
	1347	i = id(T)
	1348
	1349	if i in ids:
[5856]	1350	return 0 # T has been hashed already
[5854]	1351	else:
	1352	ids.append(i)
	1353
	1354
[5856]	1355
[5854]	1356	# Start hashing
	1357
[4270]	1358	# On some architectures None, False and True gets different hash values
[4198]	1359	if T is None:
[5854]	1360	return(-1)
[4270]	1361	if T is False:
[5854]	1362	return(0)
[4270]	1363	if T is True:
[5854]	1364	return(1)
[4198]	1365
[5854]	1366	# Get hash values for hashable entries
	1367	if type(T) in [TupleType, ListType]:
	1368	hvals = []
	1369	for t in T:
	1370	h = myhash(t, ids)
	1371	hvals.append(h)
	1372	val = hash(tuple(hvals))
	1373	elif type(T) == DictType:
[5855]	1374	# Make dictionary ordering unique
	1375	I = T.items()
	1376	I.sort()
	1377	val = myhash(I, ids)
[5854]	1378	elif type(T) == ArrayType:
[5856]	1379	# Use mean value for efficiency
	1380	val = hash(average(T.flat))
[5854]	1381	elif type(T) == InstanceType:
	1382	val = myhash(T.__dict__, ids)
[130]	1383	else:
	1384	try:
[5854]	1385	val = hash(T)
	1386	except:
	1387	val = 1
[130]	1388
	1389	return(val)
	1390
	1391	# -----------------------------------------------------------------------------
	1392
[5855]	1393
[5854]	1394	def compare(A, B, ids=None):
	1395	"""Safe comparison of general objects
[130]	1396
[5854]	1397	USAGE:
	1398	compare(A,B)
[130]	1399
[5854]	1400	DESCRIPTION:
	1401	Return 1 if A and B they are identical, 0 otherwise
	1402	"""
[130]	1403
[5856]	1404	try:
	1405	identical = (A == B)
	1406	except:
	1407	# E.g. if A and B are circular or otherwise can't be compared.
	1408	identical = False
	1409
	1410
	1411	if identical is False:
[5855]	1412	# Use pickle to compare data
	1413	# The native pickler must be used
	1414	# since the faster cPickle does not
	1415	# guarantee a unique translation
	1416
	1417	import pickle as pickler # Use non-C version here
[5856]	1418	#import cPickle as pickler
[5855]	1419	identical = (pickler.dumps(A,0) == pickler.dumps(B,0))
	1420
	1421	return(identical)
	1422
	1423
[5856]	1424	def old_compare(A, B, ids=None):
[5855]	1425	"""Safe comparison of general objects
	1426
	1427	USAGE:
	1428	compare(A,B)
	1429
	1430	DESCRIPTION:
	1431	Return 1 if A and B they are identical, 0 otherwise
	1432	"""
	1433
[5856]	1434	# FIXME (Ole): This is probably obsolete now
	1435
[5854]	1436	from types import TupleType, ListType, DictType, InstanceType
	1437
	1438
	1439	# Keep track of unique id's to protect against infinite recursion
	1440	if ids is None: ids = {}
[130]	1441
	1442
[5854]	1443	# Check if T has already been encountered
	1444	iA = id(A)
	1445	iB = id(B)
	1446
	1447	if ids.has_key((iA, iB)):
	1448	# A and B have been compared already
	1449	#print 'Found', (iA, iB), A, B
	1450	return ids[(iA, iB)]
	1451	else:
	1452	ids[(iA, iB)] = True
	1453
	1454
	1455	#print 'Comparing', A, B, (iA, iB)
	1456	#print ids
	1457	#raw_input()
	1458
	1459	if type(A) in [TupleType, ListType] and type(B) in [TupleType, ListType]:
	1460	N = len(A)
	1461	if len(B) != N:
	1462	identical = False
	1463	else:
	1464	identical = True
	1465	for i in range(N):
	1466	if not compare(A[i], B[i], ids):
	1467	identical = False; break
	1468
	1469	elif type(A) == DictType and type(B) == DictType:
	1470	if len(A) != len(B):
	1471	identical = False
	1472	else:
	1473	identical = True
	1474	for key in A.keys():
	1475	if not B.has_key(key):
	1476	identical = False; break
	1477
	1478	if not compare(A[key], B[key], ids):
	1479	identical = False; break
	1480
	1481	elif type(A) == type(B) == types.InstanceType:
	1482	# Take care of special case where elements are instances
	1483	# Base comparison on attributes
	1484
	1485	a = A.__dict__
	1486	b = B.__dict__
	1487
	1488	identical = compare(a, b, ids)
[130]	1489
[5854]	1490
	1491
	1492	else:
	1493	# Fall back to general code
	1494	try:
	1495	identical = (A == B)
	1496	except:
[5855]	1497	import pickle # Use non-C version here
[5854]	1498	try:
[5855]	1499	identical = (pickle.dumps(A,0) == pickle.dumps(B,0))
[5854]	1500	except:
	1501	identical = 0
[130]	1502
[5854]	1503	# Record result of comparison and return
	1504	ids[(iA, iB)] = identical
	1505
	1506	return(identical)
[130]	1507
	1508	# -----------------------------------------------------------------------------
	1509
	1510	def nospace(s):
	1511	"""Replace spaces in string s with underscores
	1512
	1513	USAGE:
	1514	nospace(s)
	1515
	1516	ARGUMENTS:
	1517	s -- string
	1518	"""
	1519
	1520	import string
	1521
	1522	newstr = ''
	1523	for i in range(len(s)):
	1524	if s[i] == ' ':
	1525	newstr = newstr+'_'
	1526	else:
	1527	newstr = newstr+s[i]
	1528
	1529	return(newstr)
	1530
	1531	# -----------------------------------------------------------------------------
	1532
	1533	def get_funcname(func):
	1534	"""Retrieve name of function object func (depending on its type)
	1535
	1536	USAGE:
	1537	get_funcname(func)
	1538	"""
	1539
	1540	import types, string
	1541
	1542	if type(func) == types.FunctionType:
	1543	funcname = func.func_name
	1544	elif type(func) == types.BuiltinFunctionType:
	1545	funcname = func.__name__
	1546	else:
	1547	tab = string.maketrans("<>'"," ")
	1548	tmp = string.translate(`func`,tab)
	1549	tmp = string.split(tmp)
	1550	funcname = string.join(tmp)
	1551
[4333]	1552	# Truncate memory address as in
	1553	# class __main__.Dummy at 0x00A915D0
	1554	index = funcname.find('at 0x')
	1555	if index >= 0:
	1556	funcname = funcname[:index+5] # Keep info that there is an address
	1557
[130]	1558	funcname = nospace(funcname)
	1559	return(funcname)
	1560
	1561	# -----------------------------------------------------------------------------
	1562
	1563	def get_bytecode(func):
	1564	""" Get bytecode from function object.
	1565
	1566	USAGE:
	1567	get_bytecode(func)
	1568	"""
	1569
	1570	import types
	1571
	1572	if type(func) == types.FunctionType:
	1573	bytecode = func.func_code.co_code
	1574	consts = func.func_code.co_consts
	1575	argcount = func.func_code.co_argcount
	1576	defaults = func.func_defaults
	1577	elif type(func) == types.MethodType:
	1578	bytecode = func.im_func.func_code.co_code
	1579	consts = func.im_func.func_code.co_consts
	1580	argcount = func.im_func.func_code.co_argcount
	1581	defaults = func.im_func.func_defaults
	1582	else:
	1583	#raise Exception #Test only
	1584	bytecode = None #Built-in functions are assumed not to change
	1585	consts = 0
	1586	argcount = 0
	1587	defaults = 0
	1588
	1589	return (bytecode, consts, argcount, defaults)
	1590
	1591	# -----------------------------------------------------------------------------
	1592
	1593	def get_depstats(dependencies):
	1594	""" Build dictionary of dependency files and their size, mod. time and ctime.
	1595
	1596	USAGE:
	1597	get_depstats(dependencies):
	1598	"""
	1599
	1600	import types
	1601
[4425]	1602	#print 'Caching DEBUG: Dependencies are', dependencies
[130]	1603	d = {}
	1604	if dependencies:
	1605
	1606	#Expand any wildcards
	1607	import glob
	1608	expanded_dependencies = []
	1609	for FN in dependencies:
	1610	expanded_FN = glob.glob(FN)
[4425]	1611
	1612	if expanded_FN == []:
	1613	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
	1614	raise Exception, errmsg
	1615
[130]	1616	expanded_dependencies += expanded_FN
	1617
	1618
	1619	for FN in expanded_dependencies:
	1620	if not type(FN) == types.StringType:
	1621	errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
[4425]	1622	errmsg += ' Dependency given: %s' %FN
[130]	1623	raise Exception, errmsg
	1624	if not os.access(FN,os.F_OK):
	1625	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
	1626	raise Exception, errmsg
	1627	(size,atime,mtime,ctime) = filestat(FN)
	1628
	1629	# We don't use atime because that would cause recomputation every time.
	1630	# We don't use ctime because that is irrelevant and confusing for users.
	1631	d.update({FN : (size,mtime)})
	1632
	1633	return(d)
	1634
	1635	# -----------------------------------------------------------------------------
	1636
	1637	def filestat(FN):
	1638	"""A safe wrapper using os.stat to get basic file statistics
	1639	The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
	1640
	1641	USAGE:
	1642	filestat(FN)
	1643
	1644	DESCRIPTION:
	1645	Must compile Python with
	1646	CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
	1647	configure
	1648	as given in section 8.1.1 Large File Support in the Libray Reference
	1649	"""
	1650
	1651	import os, time
	1652
	1653	try:
	1654	stats = os.stat(FN)
	1655	size = stats[6]
	1656	atime = stats[7]
	1657	mtime = stats[8]
	1658	ctime = stats[9]
	1659	except:
	1660
	1661	# Hack to get the results anyway (works only on Unix at the moment)
	1662	#
	1663	print 'Hack to get os.stat when files are too large'
	1664
	1665	if unix:
	1666	tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
	1667	# Unique filename, FIXME: Use random number
	1668
	1669	# Get size and access time (atime)
	1670	#
	1671	exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
	1672	(size,atime) = get_lsline(tmp)
	1673
	1674	# Get size and modification time (mtime)
	1675	#
	1676	exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
	1677	(size,mtime) = get_lsline(tmp)
	1678
	1679	# Get size and ctime
	1680	#
	1681	exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
	1682	(size,ctime) = get_lsline(tmp)
	1683
	1684	try:
	1685	exitcode=os.system('rm '+tmp)
	1686	# FIXME: Gives error if file doesn't exist
	1687	except:
	1688	pass
	1689	else:
	1690	pass
	1691	raise Exception # FIXME: Windows case
	1692
	1693	return(long(size),atime,mtime,ctime)
	1694
	1695	# -----------------------------------------------------------------------------
	1696
	1697	def get_lsline(FN):
	1698	"""get size and time for filename
	1699
	1700	USAGE:
	1701	get_lsline(file_name)
	1702
	1703	DESCRIPTION:
	1704	Read in one line 'ls -la' item from file (generated by filestat) and
	1705	convert time to seconds since epoch. Return file size and time.
	1706	"""
	1707
	1708	import string, time
	1709
	1710	f = open(FN,'r')
	1711	info = f.read()
	1712	info = string.split(info)
	1713
	1714	size = info[4]
	1715	week = info[5]
	1716	mon = info[6]
	1717	day = info[7]
	1718	hour = info[8]
	1719	year = info[9]
	1720
	1721	str = week+' '+mon+' '+day+' '+hour+' '+year
	1722	timetup = time.strptime(str)
	1723	t = time.mktime(timetup)
	1724	return(size, t)
	1725
	1726	# -----------------------------------------------------------------------------
	1727
	1728	def checkdir(CD,verbose=None):
	1729	"""Check or create caching directory
	1730
	1731	USAGE:
	1732	checkdir(CD,verbose):
	1733
	1734	ARGUMENTS:
	1735	CD -- Directory
	1736	verbose -- Flag verbose output (default: None)
	1737
	1738	DESCRIPTION:
	1739	If CD does not exist it will be created if possible
	1740	"""
	1741
	1742	import os
	1743	import os.path
	1744
	1745	if CD[-1] != os.sep:
	1746	CD = CD + os.sep # Add separator for directories
	1747
	1748	CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
	1749	if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
	1750	try:
	1751	exitcode=os.mkdir(CD)
	1752
	1753	# Change access rights if possible
	1754	#
	1755	if unix:
	1756	exitcode=os.system('chmod 777 '+CD)
	1757	else:
	1758	pass # FIXME: What about acces rights under Windows?
	1759	if verbose: print 'MESSAGE: Directory', CD, 'created.'
	1760	except:
	1761	print 'WARNING: Directory', CD, 'could not be created.'
	1762	if unix:
	1763	CD = '/tmp/'
	1764	else:
	1765	CD = 'C:'
	1766	print 'Using directory %s instead' %CD
	1767
	1768	return(CD)
	1769
	1770	#==============================================================================
	1771	# Statistics
	1772	#==============================================================================
	1773
	1774	def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
	1775	compression):
	1776	"""Add stats entry
	1777
	1778	USAGE:
	1779	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
	1780
	1781	DESCRIPTION:
	1782	Make one entry in the stats file about one cache hit recording time saved
	1783	and other statistics. The data are used by the function cachestat.
	1784	"""
	1785
	1786	import os, time
	1787
	1788	try:
	1789	TimeTuple = time.localtime(time.time())
	1790	extension = time.strftime('%b%Y',TimeTuple)
	1791	SFN = CD+statsfile+'.'+extension
	1792	#statfile = open(SFN,'a')
	1793	(statfile, dummy) = myopen(SFN,'a',compression=0)
	1794
	1795	# Change access rights if possible
	1796	#
	1797	#if unix:
	1798	# try:
	1799	# exitcode=os.system('chmod 666 '+SFN)
	1800	# except:
	1801	# pass
	1802	except:
	1803	print 'Warning: Stat file could not be opened'
	1804
	1805	try:
	1806	if os.environ.has_key('USER'):
	1807	user = os.environ['USER']
	1808	else:
	1809	user = 'Nobody'
	1810
	1811	date = time.asctime(TimeTuple)
	1812
	1813	if Retrieved:
	1814	hit = '1'
	1815	else:
	1816	hit = '0'
	1817
	1818	# Get size of result file
	1819	#
	1820	if compression:
	1821	stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
	1822	else:
	1823	stats = os.stat(CD+FN+'_'+file_types[0])
	1824
	1825	if stats:
	1826	size = stats[6]
	1827	else:
	1828	size = -1 # Error condition, but don't crash. This is just statistics
	1829
	1830	# Build entry
	1831	#
	1832	entry = date + ',' +\
	1833	user + ',' +\
	1834	FN + ',' +\
	1835	str(int(size)) + ',' +\
	1836	str(compression) + ',' +\
	1837	hit + ',' +\
	1838	str(reason) + ',' +\
	1839	str(round(comptime,4)) + ',' +\
	1840	str(round(loadtime,4)) +\
	1841	CR
	1842
	1843	statfile.write(entry)
	1844	statfile.close()
	1845	except:
	1846	print 'Warning: Writing of stat file failed'
	1847
	1848	# -----------------------------------------------------------------------------
	1849
	1850	# FIXME: should take cachedir as an optional arg
	1851	#
	1852	def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
	1853	""" List caching statistics.
	1854
	1855	USAGE:
	1856	__cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
	1857
	1858	Generate statistics of caching efficiency.
	1859	The parameter sortidx determines by what field lists are sorted.
	1860	If the optional keyword period is set to -1,
	1861	all available caching history is used.
	1862	If it is 0 only the current month is used.
	1863	Future versions will include more than one month....
	1864	OMN 20/8/2000
	1865	"""
	1866
	1867	import os
	1868	import os.path
[4817]	1869	from string import split, rstrip, find
[130]	1870	from time import strptime, localtime, strftime, mktime, ctime
	1871
	1872	# sortidx = 4 # Index into Fields[1:]. What to sort by.
	1873
	1874	Fields = ['Name', 'Hits', 'Exec(s)', \
	1875	'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
	1876	Widths = [25,7,9,9,9,9,13]
	1877	#Types = ['s','d','d','d','d','.2f','d']
	1878	Types = ['s','d','.2f','.2f','.2f','.2f','d']
	1879
	1880	Dictnames = ['Function', 'User']
	1881
	1882	if not cachedir:
	1883	cachedir = checkdir(options['cachedir'])
	1884
	1885	SD = os.path.expanduser(cachedir) # Expand ~ or ~user in pathname
	1886
	1887	if period == -1: # Take all available stats
	1888	SFILENAME = statsfile
	1889	else: # Only stats from current month
	1890	# MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
	1891	TimeTuple = localtime(time())
	1892	extension = strftime('%b%Y',TimeTuple)
	1893	SFILENAME = statsfile+'.'+extension
	1894
	1895	DIRLIST = os.listdir(SD)
	1896	SF = []
	1897	for FN in DIRLIST:
	1898	if find(FN,SFILENAME) >= 0:
	1899	SF.append(FN)
	1900
	1901	blocksize = 15000000
	1902	total_read = 0
	1903	total_hits = 0
	1904	total_discarded = 0
	1905	firstday = mktime(strptime('2030','%Y'))
	1906	# FIXME: strptime don't exist in WINDOWS ?
	1907	lastday = 0
	1908
	1909	FuncDict = {}
	1910	UserDict = {}
	1911	for FN in SF:
	1912	input = open(SD+FN,'r')
	1913	print 'Reading file ', SD+FN
	1914
	1915	while 1:
	1916	A = input.readlines(blocksize)
	1917	if len(A) == 0: break
	1918	total_read = total_read + len(A)
	1919	for record in A:
	1920	record = tuple(split(rstrip(record),','))
[4817]	1921	#print record, len(record)
[130]	1922
[4817]	1923	if len(record) == 9:
	1924	timestamp = record[0]
[130]	1925
	1926	try:
	1927	t = mktime(strptime(timestamp))
	1928	except:
	1929	total_discarded = total_discarded + 1
	1930	continue
	1931
	1932	if t > lastday:
	1933	lastday = t
	1934	if t < firstday:
	1935	firstday = t
	1936
[4817]	1937	user = record[1]
	1938	func = record[2]
[130]	1939
	1940	# Strip hash-stamp off
	1941	#
	1942	i = find(func,'[')
	1943	func = func[:i]
	1944
[4817]	1945	size = float(record[3])
[130]	1946
[4817]	1947	# Compression kepword can be Boolean
	1948	if record[4] in ['True', '1']:
	1949	compression = 1
	1950	elif record[4] in ['False', '0']:
	1951	compression = 0
	1952	else:
	1953	print 'Unknown value of compression', record[4]
	1954	print record
	1955	total_discarded = total_discarded + 1
	1956	continue
	1957
	1958	#compression = int(record[4]) # Can be Boolean
	1959	hit = int(record[5])
	1960	reason = int(record[6]) # Not used here
	1961	cputime = float(record[7])
	1962	loadtime = float(record[8])
	1963
[130]	1964	if hit:
	1965	total_hits = total_hits + 1
	1966	saving = cputime-loadtime
	1967
	1968	if cputime != 0:
	1969	rel_saving = round(100.0*saving/cputime,2)
	1970	else:
	1971	#rel_saving = round(1.0*saving,2)
	1972	rel_saving = 100.0 - round(1.0*saving,2) # A bit of a hack
	1973
	1974	info = [1,cputime,loadtime,saving,rel_saving,size]
	1975
	1976	UpdateDict(UserDict,user,info)
	1977	UpdateDict(FuncDict,func,info)
	1978	else:
	1979	pass #Stats on recomputations and their reasons could go in here
	1980
	1981	else:
	1982	#print 'Record discarded'
	1983	#print record
	1984	total_discarded = total_discarded + 1
	1985
	1986	input.close()
	1987
	1988	# Compute averages of all sums and write list
	1989	#
	1990
	1991	if total_read == 0:
	1992	printline(Widths,'=')
	1993	print 'CACHING STATISTICS: No valid records read'
	1994	printline(Widths,'=')
	1995	return
	1996
	1997	print
	1998	printline(Widths,'=')
	1999	print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
	2000	printline(Widths,'=')
	2001	#print ' Period:', ctime(firstday), 'to', ctime(lastday)
	2002	print ' Total number of valid records', total_read
	2003	print ' Total number of discarded records', total_discarded
	2004	print ' Total number of hits', total_hits
	2005	print
	2006
	2007	print ' Fields', Fields[2:], 'are averaged over number of hits'
	2008	print ' Time is measured in seconds and size in bytes'
	2009	print ' Tables are sorted by', Fields[1:][sortidx]
	2010
	2011	# printline(Widths,'-')
	2012
	2013	if showuser:
	2014	Dictionaries = [FuncDict, UserDict]
	2015	else:
	2016	Dictionaries = [FuncDict]
	2017
	2018	i = 0
	2019	for Dict in Dictionaries:
	2020	for key in Dict.keys():
	2021	rec = Dict[key]
	2022	for n in range(len(rec)):
	2023	if n > 0:
	2024	rec[n] = round(1.0*rec[n]/rec[0],2)
	2025	Dict[key] = rec
	2026
	2027	# Sort and output
	2028	#
	2029	keylist = SortDict(Dict,sortidx)
	2030
	2031	# Write Header
	2032	#
	2033	print
	2034	#print Dictnames[i], 'statistics:'; i=i+1
	2035	printline(Widths,'-')
	2036	n = 0
	2037	for s in Fields:
	2038	if s == Fields[0]: # Left justify
	2039	s = Dictnames[i] + ' ' + s; i=i+1
	2040	exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
	2041	else:
	2042	exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
	2043	print
	2044	printline(Widths,'-')
	2045
	2046	# Output Values
	2047	#
	2048	for key in keylist:
	2049	rec = Dict[key]
	2050	n = 0
	2051	if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
	2052	exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
	2053	for val in rec:
	2054	exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
	2055	print
	2056	print
	2057
	2058	#==============================================================================
	2059	# Auxiliary stats functions
	2060	#==============================================================================
	2061
	2062	def UpdateDict(Dict,key,info):
	2063	"""Update dictionary by adding new values to existing.
	2064
	2065	USAGE:
	2066	UpdateDict(Dict,key,info)
	2067	"""
	2068
	2069	if Dict.has_key(key):
	2070	dinfo = Dict[key]
	2071	for n in range(len(dinfo)):
	2072	dinfo[n] = info[n] + dinfo[n]
	2073	else:
	2074	dinfo = info[:] # Make a copy of info list
	2075
	2076	Dict[key] = dinfo
	2077	return Dict
	2078
	2079	# -----------------------------------------------------------------------------
	2080
	2081	def SortDict(Dict,sortidx=0):
	2082	"""Sort dictionary
	2083
	2084	USAGE:
	2085	SortDict(Dict,sortidx):
	2086
	2087	DESCRIPTION:
	2088	Sort dictionary of lists according field number 'sortidx'
	2089	"""
	2090
	2091	import types
	2092
	2093	sortlist = []
	2094	keylist = Dict.keys()
	2095	for key in keylist:
	2096	rec = Dict[key]
	2097	if not type(rec) in [types.ListType, types.TupleType]:
	2098	rec = [rec]
	2099
	2100	if sortidx > len(rec)-1:
	2101	if options['verbose']:
	2102	print 'ERROR: Sorting index to large, sortidx = ', sortidx
	2103	raise IndexError
	2104
	2105	val = rec[sortidx]
	2106	sortlist.append(val)
	2107
	2108	A = map(None,sortlist,keylist)
	2109	A.sort()
	2110	keylist = map(lambda x: x[1], A) # keylist sorted by sortidx
	2111
	2112	return(keylist)
	2113
	2114	# -----------------------------------------------------------------------------
	2115
	2116	def printline(Widths,char):
	2117	"""Print textline in fixed field.
	2118
	2119	USAGE:
	2120	printline(Widths,char)
	2121	"""
	2122
	2123	s = ''
	2124	for n in range(len(Widths)):
	2125	s = s+Widths[n]*char
	2126	if n > 0:
	2127	s = s+char
	2128
	2129	print s
	2130
	2131	#==============================================================================
	2132	# Messages
	2133	#==============================================================================
	2134
	2135	def msg1(funcname,args,kwargs,reason):
	2136	"""Message 1
	2137
	2138	USAGE:
	2139	msg1(funcname,args,kwargs,reason):
	2140	"""
	2141
	2142	import string
	2143	#print 'MESSAGE (caching.py): Evaluating function', funcname,
	2144
	2145	print_header_box('Evaluating function %s' %funcname)
	2146
[5856]	2147	msg7(args, kwargs)
[130]	2148	msg8(reason)
	2149
	2150	print_footer()
	2151
	2152	#
	2153	# Old message
	2154	#
	2155	#args_present = 0
	2156	#if args:
	2157	# if len(args) == 1:
	2158	# print 'with argument', mkargstr(args[0], textwidth2),
	2159	# else:
	2160	# print 'with arguments', mkargstr(args, textwidth2),
	2161	# args_present = 1
	2162	#
	2163	#if kwargs:
	2164	# if args_present:
	2165	# word = 'and'
	2166	# else:
	2167	# word = 'with'
	2168	#
	2169	# if len(kwargs) == 1:
	2170	# print word + ' keyword argument', mkargstr(kwargs, textwidth2)
	2171	# else:
	2172	# print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
	2173	# args_present = 1
	2174	#else:
	2175	# print # Newline when no keyword args present
	2176	#
	2177	#if not args_present:
	2178	# print '', # Default if no args or kwargs present
	2179
	2180
	2181
	2182	# -----------------------------------------------------------------------------
	2183
	2184	def msg2(funcname,args,kwargs,comptime,reason):
	2185	"""Message 2
	2186
	2187	USAGE:
	2188	msg2(funcname,args,kwargs,comptime,reason)
	2189	"""
	2190
	2191	import string
	2192
	2193	#try:
	2194	# R = Reason_msg[reason]
	2195	#except:
	2196	# R = 'Unknown reason'
	2197
	2198	#print_header_box('Caching statistics (storing) - %s' %R)
	2199	print_header_box('Caching statistics (storing)')
	2200
	2201	msg6(funcname,args,kwargs)
	2202	msg8(reason)
	2203
	2204	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
	2205
	2206	# -----------------------------------------------------------------------------
	2207
	2208	def msg3(savetime, CD, FN, deps,compression):
	2209	"""Message 3
	2210
	2211	USAGE:
	2212	msg3(savetime, CD, FN, deps,compression)
	2213	"""
	2214
	2215	import string
	2216	print string.ljust('\| Loading time:', textwidth1) + str(round(savetime,2)) + \
	2217	' seconds (estimated)'
	2218	msg5(CD,FN,deps,compression)
	2219
	2220	# -----------------------------------------------------------------------------
	2221
	2222	def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
	2223	"""Message 4
	2224
	2225	USAGE:
	2226	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
	2227	"""
	2228
	2229	import string
	2230
	2231	print_header_box('Caching statistics (retrieving)')
	2232
	2233	msg6(funcname,args,kwargs)
	2234	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
	2235	print string.ljust('\| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
	2236	print string.ljust('\| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
	2237	' seconds'
	2238	msg5(CD,FN,deps,compression)
	2239
	2240	# -----------------------------------------------------------------------------
	2241
	2242	def msg5(CD,FN,deps,compression):
	2243	"""Message 5
	2244
	2245	USAGE:
	2246	msg5(CD,FN,deps,compression)
	2247
	2248	DESCRIPTION:
	2249	Print dependency stats. Used by msg3 and msg4
	2250	"""
	2251
	2252	import os, time, string
	2253
	2254	print '\|'
	2255	print string.ljust('\| Caching dir: ', textwidth1) + CD
	2256
	2257	if compression:
	2258	suffix = '.z'
	2259	bytetext = 'bytes, compressed'
	2260	else:
	2261	suffix = ''
	2262	bytetext = 'bytes'
	2263
	2264	for file_type in file_types:
	2265	file_name = FN + '_' + file_type + suffix
	2266	print string.ljust('\| ' + file_type + ' file: ', textwidth1) + file_name,
	2267	stats = os.stat(CD+file_name)
	2268	print '('+ str(stats[6]) + ' ' + bytetext + ')'
	2269
	2270	print '\|'
	2271	if len(deps) > 0:
	2272	print '\| Dependencies: '
	2273	dependencies = deps.keys()
	2274	dlist = []; maxd = 0
	2275	tlist = []; maxt = 0
	2276	slist = []; maxs = 0
	2277	for d in dependencies:
	2278	stats = deps[d]
	2279	t = time.ctime(stats[1])
	2280	s = str(stats[0])
	2281	#if s[-1] == 'L':
	2282	# s = s[:-1] # Strip rightmost 'long integer' L off.
	2283	# # FIXME: Unnecessary in versions later than 1.5.2
	2284
	2285	if len(d) > maxd: maxd = len(d)
	2286	if len(t) > maxt: maxt = len(t)
	2287	if len(s) > maxs: maxs = len(s)
	2288	dlist.append(d)
	2289	tlist.append(t)
	2290	slist.append(s)
	2291
	2292	for n in range(len(dlist)):
	2293	d = string.ljust(dlist[n]+':', maxd+1)
	2294	t = string.ljust(tlist[n], maxt)
	2295	s = string.rjust(slist[n], maxs)
	2296
	2297	print '\| ', d, t, ' ', s, 'bytes'
	2298	else:
	2299	print '\| No dependencies'
	2300	print_footer()
	2301
	2302	# -----------------------------------------------------------------------------
	2303
	2304	def msg6(funcname,args,kwargs):
	2305	"""Message 6
	2306
	2307	USAGE:
	2308	msg6(funcname,args,kwargs)
	2309	"""
	2310
	2311	import string
	2312	print string.ljust('\| Function:', textwidth1) + funcname
	2313
[5856]	2314	msg7(args, kwargs)
[130]	2315
	2316	# -----------------------------------------------------------------------------
	2317
[5856]	2318	def msg7(args, kwargs):
[130]	2319	"""Message 7
	2320
	2321	USAGE:
[5856]	2322	msg7(args, kwargs):
[130]	2323	"""
	2324
	2325	import string
	2326
	2327	args_present = 0
	2328	if args:
	2329	if len(args) == 1:
	2330	print string.ljust('\| Argument:', textwidth1) + mkargstr(args[0], \
	2331	textwidth2)
	2332	else:
	2333	print string.ljust('\| Arguments:', textwidth1) + \
	2334	mkargstr(args, textwidth2)
	2335	args_present = 1
	2336
	2337	if kwargs:
	2338	if len(kwargs) == 1:
	2339	print string.ljust('\| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
	2340	textwidth2)
	2341	else:
	2342	print string.ljust('\| Keyword Args:', textwidth1) + \
	2343	mkargstr(kwargs, textwidth2)
	2344	args_present = 1
	2345
	2346	if not args_present:
	2347	print '\| No arguments' # Default if no args or kwargs present
	2348
	2349	# -----------------------------------------------------------------------------
	2350
	2351	def msg8(reason):
	2352	"""Message 8
	2353
	2354	USAGE:
	2355	msg8(reason):
	2356	"""
	2357
	2358	import string
	2359
	2360	try:
	2361	R = Reason_msg[reason]
	2362	except:
	2363	R = 'Unknown'
	2364
	2365	print string.ljust('\| Reason:', textwidth1) + R
	2366
	2367	# -----------------------------------------------------------------------------
	2368
	2369	def print_header_box(line):
	2370	"""Print line in a nice box.
	2371
	2372	USAGE:
	2373	print_header_box(line)
	2374
	2375	"""
	2376	global textwidth3
	2377
	2378	import time
	2379
	2380	time_stamp = time.ctime(time.time())
	2381	line = time_stamp + '. ' + line
	2382
	2383	N = len(line) + 1
	2384	s = '+' + '-'*N + CR
	2385
	2386	print s + '\| ' + line + CR + s,
	2387
	2388	textwidth3 = N
	2389
	2390	# -----------------------------------------------------------------------------
	2391
	2392	def print_footer():
	2393	"""Print line same width as that of print_header_box.
	2394	"""
	2395
	2396	N = textwidth3
	2397	s = '+' + '-'*N + CR
	2398
	2399	print s
	2400
	2401	# -----------------------------------------------------------------------------
	2402
[5856]	2403	def mkargstr(args, textwidth, argstr = '', level=0):
[130]	2404	""" Generate a string containing first textwidth characters of arguments.
	2405
	2406	USAGE:
[5856]	2407	mkargstr(args, textwidth, argstr = '', level=0)
[130]	2408
	2409	DESCRIPTION:
	2410	Exactly the same as str(args) possibly followed by truncation,
	2411	but faster if args is huge.
	2412	"""
	2413
	2414	import types
	2415
[5856]	2416	if level > 10:
	2417	# Protect against circular structures
	2418	return '...'
	2419
[130]	2420	WasTruncated = 0
	2421
	2422	if not type(args) in [types.TupleType, types.ListType, types.DictType]:
	2423	if type(args) == types.StringType:
	2424	argstr = argstr + "'"+str(args)+"'"
	2425	else:
[5854]	2426	# Truncate large Numeric arrays before using str()
[1745]	2427	import Numeric
	2428	if type(args) == Numeric.ArrayType:
[4277]	2429	# if len(args.flat) > textwidth:
	2430	# Changed by Duncan and Nick 21/2/07 .flat has problems with
	2431	# non-contigous arrays and ravel is equal to .flat except it
[4279]	2432	# can work with non-contiguous arrays
[4277]	2433	if len(Numeric.ravel(args)) > textwidth:
[1746]	2434	args = 'Array: ' + str(args.shape)
[1745]	2435
[130]	2436	argstr = argstr + str(args)
	2437	else:
	2438	if type(args) == types.DictType:
	2439	argstr = argstr + "{"
	2440	for key in args.keys():
[5856]	2441	argstr = argstr + mkargstr(key, textwidth, level=level+1) + ": " + \
	2442	mkargstr(args[key], textwidth, level=level+1) + ", "
[130]	2443	if len(argstr) > textwidth:
	2444	WasTruncated = 1
	2445	break
	2446	argstr = argstr[:-2] # Strip off trailing comma
	2447	argstr = argstr + "}"
	2448
	2449	else:
	2450	if type(args) == types.TupleType:
	2451	lc = '('
	2452	rc = ')'
	2453	else:
	2454	lc = '['
	2455	rc = ']'
	2456	argstr = argstr + lc
	2457	for arg in args:
[5856]	2458	argstr = argstr + mkargstr(arg, textwidth, level=level+1) + ', '
[130]	2459	if len(argstr) > textwidth:
	2460	WasTruncated = 1
	2461	break
	2462
	2463	# Strip off trailing comma and space unless singleton tuple
	2464	#
	2465	if type(args) == types.TupleType and len(args) == 1:
	2466	argstr = argstr[:-1]
	2467	else:
	2468	argstr = argstr[:-2]
	2469	argstr = argstr + rc
	2470
	2471	if len(argstr) > textwidth:
	2472	WasTruncated = 1
	2473
	2474	if WasTruncated:
	2475	argstr = argstr[:textwidth]+'...'
	2476	return(argstr)
	2477
	2478	# -----------------------------------------------------------------------------
	2479
	2480	def test_OK(msg):
	2481	"""Print OK msg if test is OK.
	2482
	2483	USAGE
	2484	test_OK(message)
	2485	"""
	2486
	2487	import string
	2488
	2489	print string.ljust(msg, textwidth4) + ' - OK'
	2490
	2491	#raise StandardError
	2492
	2493	# -----------------------------------------------------------------------------
	2494
	2495	def test_error(msg):
	2496	"""Print error if test fails.
	2497
	2498	USAGE
	2499	test_error(message)
	2500	"""
	2501
	2502	print 'ERROR (caching.test): %s' %msg
	2503	print 'Please send this code example and output to '
	2504	print 'Ole.Nielsen@anu.edu.au'
	2505	print
	2506	print
	2507
	2508	#import sys
	2509	#sys.exit()
	2510	raise StandardError
[5359]	2511
	2512	#-------------------------------------------------------------
	2513	if __name__ == "__main__":
	2514	pass

Note: See TracBrowser for help on using the repository browser.

Download in other formats: