Context Navigation

source: inundation/caching/caching.py @ 3372

Last change on this file since 3372 was 2043, checked in by ole, 19 years ago
Investigated file write error on networked drives
File size: 68.0 KB

Rev	Line
[130]	1	# =============================================================================
	2	# caching.py - Supervised caching of function results.
	3	# Copyright (C) 1999, 2000, 2001, 2002 Ole Moller Nielsen
[1831]	4	# Australian National University (1999-2003)
	5	# Geoscience Australia (2003-present)
[130]	6	#
	7	# This program is free software; you can redistribute it and/or modify
	8	# it under the terms of the GNU General Public License as published by
	9	# the Free Software Foundation; either version 2 of the License, or
	10	# (at your option) any later version.
	11	#
	12	# This program is distributed in the hope that it will be useful,
	13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	# GNU General Public License (http://www.gnu.org/copyleft/gpl.html)
	16	# for more details.
	17	#
	18	# You should have received a copy of the GNU General Public License
	19	# along with this program; if not, write to the Free Software
	20	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
	21	#
	22	#
[1831]	23	# Contact address: Ole.Nielsen@ga.gov.au
[130]	24	#
	25	# Version 1.5.6 February 2002
	26	# =============================================================================
	27
	28	"""Module caching.py - Supervised caching of function results.
	29
	30	Public functions:
	31
	32	cache(func,args) -- Cache values returned from func given args.
	33	cachestat() -- Reports statistics about cache hits and time saved.
	34	test() -- Conducts a basic test of the caching functionality.
	35
	36	See doc strings of individual functions for detailed documentation.
	37	"""
	38
	39	# -----------------------------------------------------------------------------
	40	# Initialisation code
	41
	42	# Determine platform
	43	#
	44	import os
	45	if os.name in ['nt', 'dos', 'win32', 'what else?']:
	46	unix = 0
	47	else:
	48	unix = 1
	49
	50	# Make default caching directory name
	51	#
	52	if unix:
	53	homedir = '~'
	54	CR = '\n'
	55	else:
	56	homedir = 'c:'
	57	CR = '\r\n' #FIXME: Not tested under windows
	58
	59	cachedir = homedir + os.sep + '.python_cache' + os.sep
	60
	61	# -----------------------------------------------------------------------------
	62	# Options directory with default values - to be set by user
	63	#
	64
	65	options = {
	66	'cachedir': cachedir, # Default cache directory
	67	'maxfiles': 1000000, # Maximum number of cached files
	68	'savestat': 1, # Log caching info to stats file
	69	'verbose': 1, # Write messages to standard output
	70	'bin': 1, # Use binary format (more efficient)
[1831]	71	'compression': 1, # Use zlib compression
[130]	72	'bytecode': 0, # Recompute if bytecode has changed
	73	'expire': 0 # Automatically remove files that have been accessed
	74	# least recently
	75	}
	76
	77	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	78
	79	def set_option(key, value):
	80	"""Function to set values in the options directory.
	81
	82	USAGE:
	83	set_option(key, value)
	84
	85	ARGUMENTS:
	86	key -- Key in options dictionary. (Required)
	87	value -- New value for key. (Required)
	88
	89	DESCRIPTION:
	90	Function to set values in the options directory.
	91	Raises an exception if key is not in options.
	92	"""
	93
	94	if options.has_key(key):
	95	options[key] = value
	96	else:
	97	raise KeyError(key) # Key not found, raise an exception
	98
	99	# -----------------------------------------------------------------------------
	100	# Function cache - the main routine
	101
	102	def cache(func, args=(), kwargs = {}, dependencies=None , cachedir=None,
	103	verbose=None, compression=None, evaluate=0, test=0, clear=0,
	104	return_filename=0):
	105	"""Supervised caching of function results.
	106
	107	USAGE:
	108	result = cache(func, args, kwargs, dependencies, cachedir, verbose,
	109	compression, evaluate, test, return_filename)
	110
	111	ARGUMENTS:
	112	func -- Function object (Required)
	113	args -- Arguments to func (Default: ())
	114	kwargs -- Keyword arguments to func (Default: {})
	115	dependencies -- Filenames that func depends on (Default: None)
	116	cachedir -- Directory for cache files (Default: options['cachedir'])
	117	verbose -- Flag verbose output to stdout
	118	(Default: options['verbose'])
	119	compression -- Flag zlib compression (Default: options['compression'])
	120	evaluate -- Flag forced evaluation of func (Default: 0)
	121	test -- Flag test for cached results (Default: 0)
	122	clear -- Flag delete cached results (Default: 0)
	123	return_filename -- Flag return of cache filename (Default: 0)
	124
	125	DESCRIPTION:
	126	A Python function call of the form
	127
	128	result = func(arg1,...,argn)
	129
	130	can be replaced by
	131
	132	from caching import cache
	133	result = cache(func,(arg1,...,argn))
	134
	135	The latter form returns the same output as the former but reuses cached
	136	results if the function has been computed previously in the same context.
	137	'result' and the arguments can be simple types, tuples, list, dictionaries or
	138	objects, but not unhashable types such as functions or open file objects.
	139	The function 'func' may be a member function of an object or a module.
	140
	141	This type of caching is particularly useful for computationally intensive
	142	functions with few frequently used combinations of input arguments. Note that
	143	if the inputs or output are very large caching might not save time because
	144	disc access may dominate the execution time.
	145
	146	If the function definition changes after a result has been cached it will be
	147	detected by examining the functions bytecode (co_code, co_consts,
	148	func_defualts, co_argcount) and it will be recomputed.
	149
	150	LIMITATIONS:
	151	1 Caching uses the apply function and will work with anything that can be
	152	pickled, so any limitation in apply or pickle extends to caching.
	153	2 A function to be cached should not depend on global variables
	154	as wrong results may occur if globals are changed after a result has
	155	been cached.
	156
	157	-----------------------------------------------------------------------------
	158	Additional functionality:
	159
	160	Keyword args
	161	Keyword arguments (kwargs) can be added as a dictionary of keyword: value
	162	pairs, following the syntax of the built-in function apply().
	163	A Python function call of the form
	164
	165	result = func(arg1,...,argn, kwarg1=val1,...,kwargm=valm)
	166
	167	is then cached as follows
	168
	169	from caching import cache
	170	result = cache(func,(arg1,...,argn), {kwarg1:val1,...,kwargm:valm})
	171
	172	The default value of kwargs is {}
	173
	174	Explicit dependencies:
	175	The call
	176	cache(func,(arg1,...,argn),dependencies = <list of filenames>)
	177	Checks the size, creation time and modification time of each listed file.
	178	If any file has changed the function is recomputed and the results stored
	179	again.
	180
	181	Specify caching directory:
	182	The call
	183	cache(func,(arg1,...,argn), cachedir = <cachedir>)
	184	designates <cachedir> where cached data are stored. Use ~ to indicate users
	185	home directory - not $HOME. The default is ~/.python_cache on a UNIX
	186	platform and c:/.python_cache on a Win platform.
	187
	188	Silent operation:
	189	The call
	190	cache(func,(arg1,...,argn),verbose=0)
	191	suppresses messages to standard output.
	192
	193	Compression:
	194	The call
	195	cache(func,(arg1,...,argn),compression=0)
	196	disables compression. (Default: compression=1). If the requested compressed
	197	or uncompressed file is not there, it'll try the other version.
	198
	199	Forced evaluation:
	200	The call
	201	cache(func,(arg1,...,argn),evaluate=1)
	202	forces the function to evaluate even though cached data may exist.
	203
	204	Testing for presence of cached result:
	205	The call
	206	cache(func,(arg1,...,argn),test=1)
	207	retrieves cached result if it exists, otherwise None. The function will not
	208	be evaluated. If both evaluate and test are switched on, evaluate takes
	209	precedence.
	210
	211	Obtain cache filenames:
	212	The call
	213	cache(func,(arg1,...,argn),return_filename=1)
	214	returns the hashed base filename under which this function and its
	215	arguments would be cached
	216
	217	Clearing cached results:
	218	The call
	219	cache(func,'clear')
	220	clears all cached data for 'func' and
	221	cache('clear')
	222	clears all cached data.
	223
	224	NOTE: The string 'clear' can be passed an argument to func using
	225	cache(func,('clear',)) or cache(func,tuple(['clear'])).
	226
	227	New form of clear:
	228	cache(func,(arg1,...,argn),clear=1)
	229	clears cached data for particular combination func and args
	230
	231	"""
	232
	233	# Imports and input checks
	234	#
	235	import types, time, string
	236
	237	if not cachedir:
	238	cachedir = options['cachedir']
	239
	240	if verbose == None: # Do NOT write 'if not verbose:', it could be zero.
	241	verbose = options['verbose']
	242
	243	if compression == None: # Do NOT write 'if not compression:',
	244	# it could be zero.
	245	compression = options['compression']
	246
	247	# Create cache directory if needed
	248	#
	249	CD = checkdir(cachedir,verbose)
	250
	251	# Handle the case cache('clear')
	252	#
	253	if type(func) == types.StringType:
	254	if string.lower(func) == 'clear':
	255	clear_cache(CD,verbose=verbose)
	256	return
	257
	258	# Handle the case cache(func, 'clear')
	259	#
	260	if type(args) == types.StringType:
	261	if string.lower(args) == 'clear':
	262	clear_cache(CD,func,verbose=verbose)
	263	return
	264
	265	# Force singleton arg into a tuple
	266	#
	267	if type(args) != types.TupleType:
	268	args = tuple([args])
	269
	270	# Check that kwargs is a dictionary
	271	#
	272	if type(kwargs) != types.DictType:
	273	raise TypeError
	274
	275	#print 'hashing' #FIXME: make faster hashing function
	276
	277	# Hash arguments (and keyword args) to integer
	278	#
	279	arghash = myhash((args,kwargs))
	280
	281	# Get sizes and timestamps for files listed in dependencies.
	282	# Force singletons into a tuple.
	283	#
	284	if dependencies and type(dependencies) != types.TupleType \
	285	and type(dependencies) != types.ListType:
	286	dependencies = tuple([dependencies])
	287	deps = get_depstats(dependencies)
	288
	289	# Extract function name from func object
	290	#
	291	funcname = get_funcname(func)
	292
	293	# Create cache filename
	294	#
	295	FN = funcname+'['+`arghash`+']' # The symbol '(' does not work under unix
	296
	297	if return_filename:
	298	return(FN)
	299
	300	if clear:
	301	for file_type in file_types:
	302	file_name = CD+FN+'_'+file_type
	303	for fn in [file_name, file_name + '.z']:
	304	if os.access(fn, os.F_OK):
	305	if unix:
	306	os.remove(fn)
	307	else:
	308	# FIXME: os.remove doesn't work under windows
	309	os.system('del '+fn)
	310	if verbose:
	311	print 'MESSAGE (caching): File %s deleted' %fn
	312	##else:
	313	## print '%s was not accessed' %fn
	314	return None
	315
	316
	317	#-------------------------------------------------------------------
	318
	319	# Check if previous computation has been cached
	320	#
	321	if evaluate:
	322	Retrieved = None # Force evaluation of func regardless of caching status.
	323	reason = 4
	324	else:
	325	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
	326	CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
	327	dependencies)
	328
	329	if not Retrieved:
	330	if test: # Do not attempt to evaluate function
	331	T = None
	332	else: # Evaluate function and save to cache
	333	if verbose:
	334	msg1(funcname, args, kwargs,reason)
	335
	336	# Remove expired files automatically
	337	#
	338	if options['expire']:
	339	DeleteOldFiles(CD,verbose)
	340
	341	# Save args before function is evaluated in case
	342	# they are modified by function
	343	#
	344	save_args_to_cache(CD,FN,args,kwargs,compression)
	345
	346	# Execute and time function with supplied arguments
	347	#
	348	t0 = time.time()
	349	T = apply(func,args,kwargs)
	350	#comptime = round(time.time()-t0)
	351	comptime = time.time()-t0
	352
	353	if verbose:
	354	msg2(funcname,args,kwargs,comptime,reason)
	355
	356	# Save results and estimated loading time to cache
	357	#
	358	loadtime = save_results_to_cache(T, CD, FN, func, deps, comptime, \
	359	funcname, dependencies, compression)
	360	if verbose:
	361	msg3(loadtime, CD, FN, deps, compression)
	362	compressed = compression
	363
	364	if options['savestat'] and (not test or Retrieved):
	365	##if options['savestat']:
	366	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compressed)
	367
	368	return(T) # Return results in all cases
	369
	370	# -----------------------------------------------------------------------------
	371
	372	def cachestat(sortidx=4, period=-1, showuser=None, cachedir=None):
	373	"""Generate statistics of caching efficiency.
	374
	375	USAGE:
	376	cachestat(sortidx, period, showuser, cachedir)
	377
	378	ARGUMENTS:
	379	sortidx -- Index of field by which lists are (default: 4)
	380	Legal values are
	381	0: 'Name'
	382	1: 'Hits'
	383	2: 'CPU'
	384	3: 'Time Saved'
	385	4: 'Gain(%)'
	386	5: 'Size'
	387	period -- If set to -1 all available caching history is used.
	388	If set 0 only the current month is used (default -1).
	389	showuser -- Flag for additional table showing user statistics
	390	(default: None).
	391	cachedir -- Directory for cache files (default: options['cachedir']).
	392
	393	DESCRIPTION:
	394	Logged caching statistics is converted into summaries of the form
	395	--------------------------------------------------------------------------
	396	Function Name Hits Exec(s) Cache(s) Saved(s) Gain(%) Size
	397	--------------------------------------------------------------------------
	398	"""
	399
	400	__cachestat(sortidx, period, showuser, cachedir)
	401	return
	402
	403	# -----------------------------------------------------------------------------
	404
[2043]	405	#Has mostly been moved to proper unit test
[130]	406	def test(cachedir=None,verbose=0,compression=None):
	407	"""Test the functionality of caching.
	408
	409	USAGE:
	410	test(verbose)
	411
	412	ARGUMENTS:
	413	verbose -- Flag whether caching will output its statistics (default=0)
	414	cachedir -- Directory for cache files (Default: options['cachedir'])
	415	compression -- Flag zlib compression (Default: options['compression'])
	416	"""
	417
	418	import string, time
	419
	420	# Initialise
	421	#
	422	import caching
	423	reload(caching)
	424
	425	if not cachedir:
	426	cachedir = options['cachedir']
	427
	428	if verbose is None: # Do NOT write 'if not verbose:', it could be zero.
	429	verbose = options['verbose']
	430
	431	if compression == None: # Do NOT write 'if not compression:',
	432	# it could be zero.
	433	compression = options['compression']
	434	else:
	435	try:
	436	set_option('compression', compression)
	437	except:
	438	test_error('Set option failed')
	439
	440	try:
	441	import zlib
	442	except:
	443	print
	444	print '* Could not find zlib, default to no-compression *'
	445	print '* Installing zlib will improve performance of caching *'
	446	print
	447	compression = 0
	448	set_option('compression', compression)
	449
	450	print
	451	print_header_box('Testing caching module - please stand by')
	452	print
	453
	454	# Define a test function to be cached
	455	#
	456	def f(a,b,c,N,x=0,y='abcdefg'):
	457	"""f(a,b,c,N)
	458	Do something time consuming and produce a complex result.
	459	"""
	460
	461	import string
	462
	463	B = []
	464	for n in range(N):
	465	s = str(n+2.0/(n + 4.0))+'.a'*10
	466	B.append((a,b,c,s,n,x,y))
	467	return(B)
	468
	469	# Check that default cachedir is OK
	470	#
	471	CD = checkdir(cachedir,verbose)
	472
	473
	474	# Make a dependency file
	475	#
	476	try:
	477	DepFN = CD + 'testfile.tmp'
	478	DepFN_wildcard = CD + 'test*.tmp'
	479	Depfile = open(DepFN,'w')
	480	Depfile.write('We are the knights who say NI!')
	481	Depfile.close()
	482	test_OK('Wrote file %s' %DepFN)
	483	except:
	484	test_error('Could not open file %s for writing - check your environment' \
	485	% DepFN)
	486
	487	# Check set_option (and switch stats off
	488	#
	489	try:
	490	set_option('savestat',0)
	491	assert(options['savestat'] == 0)
	492	test_OK('Set option')
	493	except:
	494	test_error('Set option failed')
	495
	496	# Make some test input arguments
	497	#
	498	N = 5000 #Make N fairly small here
	499
	500	a = [1,2]
	501	b = ('Thou shalt count the number three',4)
	502	c = {'Five is right out': 6, (7,8): 9}
	503	x = 3
	504	y = 'holy hand granate'
	505
	506	# Test caching
	507	#
	508	if compression:
	509	comprange = 2
	510	else:
	511	comprange = 1
	512
	513	for comp in range(comprange):
	514
	515	# Evaluate and store
	516	#
	517	try:
	518	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, evaluate=1, \
	519	verbose=verbose, compression=comp)
	520	if comp:
	521	test_OK('Caching evaluation with compression')
	522	else:
	523	test_OK('Caching evaluation without compression')
	524	except:
	525	if comp:
	526	test_error('Caching evaluation with compression failed - try caching.test(compression=0)')
	527	else:
	528	test_error('Caching evaluation failed - try caching.test(verbose=1)')
	529
	530	# Retrieve
	531	#
	532	try:
	533	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	534	compression=comp)
	535
	536	if comp:
	537	test_OK('Caching retrieval with compression')
	538	else:
	539	test_OK('Caching retrieval without compression')
	540	except:
	541	if comp:
	542	test_error('Caching retrieval with compression failed - try caching.test(compression=0)')
	543	else:
	544	test_error('Caching retrieval failed - try caching.test(verbose=1)')
	545
	546	# Reference result
	547	#
	548	T3 = f(a,b,c,N,x=x,y=y) # Compute without caching
	549
	550	if T1 == T2 and T2 == T3:
	551	if comp:
	552	test_OK('Basic caching functionality (with compression)')
	553	else:
	554	test_OK('Basic caching functionality (without compression)')
	555	else:
	556	test_error('Cached result does not match computed result')
	557
	558
	559	# Test return_filename
	560	#
	561	try:
	562	FN = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	563	return_filename=1)
	564	assert(FN[:2] == 'f[')
	565	test_OK('Return of cache filename')
	566	except:
	567	test_error('Return of cache filename failed')
	568
	569	# Test existence of cachefiles
	570	#
	571	try:
	572	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
	573	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
	574	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
	575	test_OK('Presence of cache files')
	576	datafile.close()
	577	argsfile.close()
	578	admfile.close()
	579	except:
	580	test_error('Expected cache files did not exist')
	581
	582	# Test 'test' function when cache is present
	583	#
	584	try:
	585	#T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	586	# evaluate=1)
	587	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
	588	assert(T1 == T4)
	589
	590	test_OK("Option 'test' when cache file present")
	591	except:
	592	test_error("Option 'test' when cache file present failed")
	593
	594	# Test that 'clear' works
	595	#
	596	#try:
	597	# caching.cache(f,'clear',verbose=verbose)
	598	# test_OK('Clearing of cache files')
	599	#except:
	600	# test_error('Clear does not work')
	601	try:
	602	caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, clear=1)
	603	test_OK('Clearing of cache files')
	604	except:
	605	test_error('Clear does not work')
	606
	607
	608
	609	# Test 'test' function when cache is absent
	610	#
	611	try:
	612	T4 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, test=1)
	613	assert(T4 is None)
	614	test_OK("Option 'test' when cache absent")
	615	except:
	616	test_error("Option 'test' when cache absent failed")
	617
	618	# Test dependencies
	619	#
	620	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	621	dependencies=DepFN)
	622	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	623	dependencies=DepFN)
	624
	625	if T1 == T2:
	626	test_OK('Basic dependencies functionality')
	627	else:
	628	test_error('Dependencies do not work')
	629
	630	# Test basic wildcard dependency
	631	#
	632	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	633	dependencies=DepFN_wildcard)
	634
	635	if T1 == T3:
	636	test_OK('Basic dependencies with wildcard functionality')
	637	else:
	638	test_error('Dependencies with wildcards do not work')
	639
	640
	641	# Test that changed timestamp in dependencies triggers recomputation
	642
	643	# Modify dependency file
	644	Depfile = open(DepFN,'a')
	645	Depfile.write('You must cut down the mightiest tree in the forest with a Herring')
	646	Depfile.close()
	647
	648	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	649	dependencies=DepFN, test = 1)
	650
	651	if T3 is None:
	652	test_OK('Changed dependencies recognised')
	653	else:
	654	test_error('Changed dependencies not recognised')
	655
	656	# Test recomputation when dependencies have changed
	657	#
	658	T3 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose, \
	659	dependencies=DepFN)
	660	if T1 == T3:
	661	test_OK('Recomputed value with changed dependencies')
	662	else:
	663	test_error('Recomputed value with changed dependencies failed')
[1831]	664
[130]	665	# Performance test (with statistics)
	666	# Don't really rely on this as it will depend on specific computer.
	667	#
	668
	669	set_option('savestat',1)
	670
	671	N = 20*N #Should be large on fast computers...
	672	tt = time.time()
	673	T1 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
	674	t1 = time.time() - tt
	675
	676	tt = time.time()
	677	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=verbose)
	678	t2 = time.time() - tt
	679
	680	if T1 == T2:
	681	if t1 > t2:
	682	test_OK('Performance test: relative time saved = %s pct' \
	683	%str(round((t1-t2)*100/t1,2)))
	684	#else:
	685	# print 'WARNING: Performance a bit low - this could be specific to current platform'
	686	else:
	687	test_error('Basic caching failed for new problem')
	688
	689	# Test presence of statistics file
	690	#
	691	try:
	692	DIRLIST = os.listdir(CD)
	693	SF = []
	694	for FN in DIRLIST:
	695	if string.find(FN,statsfile) >= 0:
	696	fid = open(CD+FN,'r')
	697	fid.close()
	698	test_OK('Statistics files present')
	699	except:
	700	test_OK('Statistics files cannot be opened')
	701
	702	print_header_box('Show sample output of the caching function:')
	703
	704	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
	705	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=0)
	706	T2 = caching.cache(f,(a,b,c,N), {'x':x, 'y':y}, verbose=1)
	707
	708	print_header_box('Show sample output of cachestat():')
	709	if unix:
	710	cachestat()
	711	else:
	712	try:
	713	import time
	714	t = time.strptime('2030','%Y')
	715	cachestat()
	716	except:
	717	print 'caching.cachestat() does not work here, because it'
	718	print 'relies on time.strptime() which is unavailable in Windows'
	719
	720	print
	721	test_OK('Caching self test completed')
	722
	723
	724	# Test setoption (not yet implemented)
	725	#
[1831]	726
[130]	727
	728	#==============================================================================
	729	# Auxiliary functions
	730	#==============================================================================
	731
	732	# Import pickler
	733	# cPickle is used by functions mysave, myload, and compare
	734	#
	735	import cPickle # 10 to 100 times faster than pickle
	736	pickler = cPickle
	737
	738	# Local immutable constants
	739	#
	740	comp_level = 1 # Compression level for zlib.
	741	# comp_level = 1 works well.
	742	textwidth1 = 16 # Text width of key fields in report forms.
	743	textwidth2 = 132 # Maximal width of textual representation of
	744	# arguments.
	745	textwidth3 = 16 # Initial width of separation lines. Is modified.
	746	textwidth4 = 50 # Text width in test_OK()
	747	statsfile = '.cache_stat' # Basefilename for cached statistics.
	748	# It will reside in the chosen cache directory.
	749
	750	file_types = ['Result', # File name extension for cached function results.
	751	'Args', # File name extension for stored function args.
	752	'Admin'] # File name extension for administrative info.
	753
	754	Reason_msg = ['OK', # Verbose reasons for recomputation
	755	'No cached result',
	756	'Dependencies have changed',
	757	'Byte code or arguments have changed',
	758	'Recomputation was requested by caller',
	759	'Cached file was unreadable']
	760
	761	# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
	762
	763	def CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression,
	764	dependencies):
	765	"""Determine whether cached result exists and return info.
	766
	767	USAGE:
	768	(T, FN, Retrieved, reason, comptime, loadtime, compressed) = \
	769	CacheLookup(CD, FN, func, args, kwargs, deps, verbose, compression, \
	770	dependencies)
	771
	772	INPUT ARGUMENTS:
	773	CD -- Cache Directory
	774	FN -- Suggested cache file name
	775	func -- Function object
	776	args -- Tuple of arguments
	777	kwargs -- Dictionary of keyword arguments
	778	deps -- Dependencies time stamps
	779	verbose -- Flag text output
	780	compression -- Flag zlib compression
	781	dependencies -- Given list of dependencies
	782
	783	OUTPUT ARGUMENTS:
	784	T -- Cached result if present otherwise None
	785	FN -- File name under which new results must be saved
	786	Retrieved -- True if a valid cached result was found
	787	reason -- 0: OK (if Retrieved),
	788	1: No cached result,
	789	2: Dependencies have changed,
	790	3: Arguments or Bytecode have changed
	791	4: Recomputation was forced
	792	comptime -- Number of seconds it took to computed cachged result
	793	loadtime -- Number of seconds it took to load cached result
	794	compressed -- Flag (0,1) if cached results were compressed or not
	795
	796	DESCRIPTION:
	797	Determine if cached result exists as follows:
	798	Load in saved arguments and bytecode stored under hashed filename.
	799	If they are identical to current arguments and bytecode and if dependencies
	800	have not changed their time stamp, then return cached result.
	801
	802	Otherwise return filename under which new results should be cached.
	803	Hash collisions are handled recursively by calling CacheLookup again with a
	804	modified filename.
	805	"""
	806
	807	import time, string, types
	808
	809	# Assess whether cached result exists - compressed or not.
	810	#
	811	if verbose:
	812	print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\
	813	%(CD+FN, file_types[0], file_types[1], file_types[2])
	814	(datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
	815	(argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression)
	816	(admfile,compressed2) = myopen(CD+FN+'_'+file_types[2],"rb",compression)
	817
	818	if not (argsfile and datafile and admfile) or \
	819	not (compressed0 == compressed1 and compressed0 == compressed2):
	820	# Cached result does not exist or files were compressed differently
	821	#
	822	# This will ensure that evaluation will take place unless all files are
	823	# present.
	824
	825	reason = 1
	826	return(None,FN,None,reason,None,None,None) #Recompute using same filename
	827
	828	compressed = compressed0 # Remember if compressed files were actually used
	829	datafile.close()
	830
	831	# Retrieve arguments and adm. info
	832	#
	833	R, reason = myload(argsfile,compressed) # The original arguments
	834	argsfile.close()
	835
	836	##if R == None and reason > 0:
	837	if reason > 0:
	838	return(None,FN,None,reason,None,None,None) #Recompute using same filename
	839	else:
	840	(argsref, kwargsref) = R
	841
	842	R, reason = myload(admfile,compressed)
	843	admfile.close()
	844	##if R == None and reason > 0:
	845	if reason > 0:
	846	return(None,FN,None,reason,None,None,None) #Recompute using same filename
	847
	848
	849	depsref = R[0] # Dependency statistics
	850	comptime = R[1] # The computation time
	851	coderef = R[2] # The byte code
	852	funcname = R[3] # The function name
	853
	854	# Check if dependencies have changed
	855	#
	856	if dependencies and not compare(depsref,deps):
	857	if verbose:
	858	print 'MESSAGE (caching.py): Dependencies', dependencies, \
	859	'have changed - recomputing'
	860	# Don't use cached file - recompute
	861	reason = 2
	862	return(None,FN,None,reason,None,None,None)
	863
	864	# Get bytecode from func
	865	#
	866	bytecode = get_bytecode(func)
	867
	868	#print compare(argsref,args),
	869	#print compare(kwargsref,kwargs),
	870	#print compare(bytecode,coderef)
	871
	872	# Check if arguments or bytecode have changed
	873	#
	874	if compare(argsref,args) and compare(kwargsref,kwargs) and \
	875	(not options['bytecode'] or compare(bytecode,coderef)):
	876
	877	# Arguments and dependencies match. Get cached results
	878	#
	879	T, loadtime, compressed, reason = load_from_cache(CD,FN,compressed)
	880	###if T == None and reason > 0: #This doesn't work if T is a numeric array
	881	if reason > 0:
	882	return(None,FN,None,reason,None,None,None) #Recompute using same FN
	883
	884	Retrieved = 1
	885	reason = 0
	886
	887	if verbose:
	888	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compressed)
	889
	890	if loadtime >= comptime:
	891	print 'WARNING (caching.py): Caching did not yield any gain.'
	892	print ' Consider executing function ',
	893	print '('+funcname+') without caching.'
	894	else:
	895
	896	# Non matching arguments or bytecodes signify a hash-collision.
	897	# This is resolved by recursive search of cache filenames
	898	# until either a matching or an unused filename is found.
	899	#
	900	(T,FN,Retrieved,reason,comptime,loadtime,compressed) = \
	901	CacheLookup(CD,FN+'x',func,args,kwargs,deps,verbose,compression, \
	902	dependencies)
	903
	904	# DEBUGGING
	905	# if not Retrieved:
	906	# print 'Arguments did not match'
	907	# else:
	908	# print 'Match found !'
	909	if not Retrieved:
	910	reason = 3 #The real reason is that args or bytecodes have changed.
	911	#Not that the recursive seach has found an unused filename
	912
	913	return((T, FN, Retrieved, reason, comptime, loadtime, compressed))
	914
	915	# -----------------------------------------------------------------------------
	916
	917	def clear_cache(CD,func=None, verbose=None):
	918	"""Clear cache for func.
	919
	920	USAGE:
	921	clear(CD, func, verbose)
	922
	923	ARGUMENTS:
	924	CD -- Caching directory (required)
	925	func -- Function object (default: None)
	926	verbose -- Flag verbose output (default: None)
	927
	928	DESCRIPTION:
	929
	930	If func == None, clear everything,
	931	otherwise clear only files pertaining to func.
	932	"""
	933
	934	import os, re
	935
	936	if CD[-1] != os.sep:
	937	CD = CD+os.sep
	938
	939	if verbose == None:
	940	verbose = options['verbose']
	941
	942	# FIXME: Windows version needs to be tested
	943
	944	if func:
	945	funcname = get_funcname(func)
	946	if verbose:
	947	print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'
	948
	949	file_names = os.listdir(CD)
	950	for file_name in file_names:
	951	#RE = re.search('^' + funcname,file_name) #Inefficient
	952	#if RE:
	953	if file_name[:len(funcname)] == funcname:
	954	if unix:
	955	os.remove(CD+file_name)
	956	else:
	957	os.system('del '+CD+file_name)
	958	# FIXME: os.remove doesn't work under windows
	959	else:
	960	file_names = os.listdir(CD)
	961	if len(file_names) > 0:
	962	if verbose:
	963	print 'MESSAGE (caching.py): Remove the following files:'
	964	for file_name in file_names:
	965	print file_name
	966
	967	A = raw_input('Delete (Y/N)[N] ?')
	968	else:
	969	A = 'Y'
	970
	971	if A == 'Y' or A == 'y':
	972	for file_name in file_names:
	973	if unix:
	974	os.remove(CD+file_name)
	975	else:
	976	os.system('del '+CD+file_name)
	977	# FIXME: os.remove doesn't work under windows
	978	#exitcode=os.system('/bin/rm '+CD+'* 2> /dev/null')
	979
	980	# -----------------------------------------------------------------------------
	981
	982	def DeleteOldFiles(CD,verbose=None):
	983	"""Remove expired files
	984
	985	USAGE:
	986	DeleteOldFiles(CD,verbose=None)
	987	"""
	988
	989	if verbose == None:
	990	verbose = options['verbose']
	991
	992	maxfiles = options['maxfiles']
	993
	994	# FIXME: Windows version
	995
	996	import os
	997	block = 1000 # How many files to delete per invokation
	998	Files = os.listdir(CD)
	999	numfiles = len(Files)
	1000	if not unix: return # FIXME: Windows case ?
	1001
	1002	if numfiles > maxfiles:
	1003	delfiles = numfiles-maxfiles+block
	1004	if verbose:
	1005	print 'Deleting '+`delfiles`+' expired files:'
	1006	os.system('ls -lur '+CD+'* \| head -' + `delfiles`) # List them
	1007	os.system('ls -ur '+CD+'* \| head -' + `delfiles` + ' \| xargs /bin/rm')
	1008	# Delete them
	1009	# FIXME: Replace this with os.listdir and os.remove
	1010
	1011	# -----------------------------------------------------------------------------
	1012
	1013	def save_args_to_cache(CD,FN,args,kwargs,compression):
	1014	"""Save arguments to cache
	1015
	1016	USAGE:
	1017	save_args_to_cache(CD,FN,args,kwargs,compression)
	1018	"""
	1019
	1020	import time, os, sys, types
	1021
	1022	(argsfile, compressed) = myopen(CD+FN+'_'+file_types[1], 'wb', compression)
	1023
	1024	if not argsfile:
	1025	if verbose:
	1026	print 'ERROR (caching): Could not open %s' %argsfile.name
	1027	raise IOError
	1028
	1029	mysave((args,kwargs),argsfile,compression) # Save args and kwargs to cache
	1030	argsfile.close()
	1031
	1032	# Change access rights if possible
	1033	#
	1034	#if unix:
	1035	# try:
	1036	# exitcode=os.system('chmod 666 '+argsfile.name)
	1037	# except:
	1038	# pass
	1039	#else:
	1040	# pass # FIXME: Take care of access rights under Windows
	1041
	1042	return
	1043
	1044	# -----------------------------------------------------------------------------
	1045
	1046	def save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
	1047	dependencies, compression):
	1048	"""Save computed results T and admin info to cache
	1049
	1050	USAGE:
	1051	save_results_to_cache(T, CD, FN, func, deps, comptime, funcname,
	1052	dependencies, compression)
	1053	"""
	1054
	1055	import time, os, sys, types
	1056
	1057	(datafile, compressed1) = myopen(CD+FN+'_'+file_types[0],'wb',compression)
	1058	(admfile, compressed2) = myopen(CD+FN+'_'+file_types[2],'wb',compression)
	1059
	1060	if not datafile:
	1061	if verbose:
	1062	print 'ERROR (caching): Could not open %s' %datafile.name
	1063	raise IOError
	1064
	1065	if not admfile:
	1066	if verbose:
	1067	print 'ERROR (caching): Could not open %s' %admfile.name
	1068	raise IOError
	1069
	1070	t0 = time.time()
	1071
	1072	mysave(T,datafile,compression) # Save data to cache
	1073	datafile.close()
	1074	#savetime = round(time.time()-t0,2)
	1075	savetime = time.time()-t0
	1076
	1077	bytecode = get_bytecode(func) # Get bytecode from function object
	1078	admtup = (deps, comptime, bytecode, funcname) # Gather admin info
	1079
	1080	mysave(admtup,admfile,compression) # Save admin info to cache
	1081	admfile.close()
	1082
	1083	# Change access rights if possible
	1084	#
	1085	#if unix:
	1086	# try:
	1087	# exitcode=os.system('chmod 666 '+datafile.name)
	1088	# exitcode=os.system('chmod 666 '+admfile.name)
	1089	# except:
	1090	# pass
	1091	#else:
	1092	# pass # FIXME: Take care of access rights under Windows
	1093
	1094	return(savetime)
	1095
	1096	# -----------------------------------------------------------------------------
	1097
	1098	def load_from_cache(CD,FN,compression):
	1099	"""Load previously cached data from file FN
	1100
	1101	USAGE:
	1102	load_from_cache(CD,FN,compression)
	1103	"""
	1104
	1105	import time
	1106
	1107	(datafile, compressed) = myopen(CD+FN+'_'+file_types[0],"rb",compression)
	1108	t0 = time.time()
	1109	T, reason = myload(datafile,compressed)
	1110	#loadtime = round(time.time()-t0,2)
	1111	loadtime = time.time()-t0
	1112	datafile.close()
	1113
	1114	return T, loadtime, compressed, reason
	1115
	1116	# -----------------------------------------------------------------------------
	1117
	1118	def myopen(FN,mode,compression=1):
	1119	"""Open file FN using given mode
	1120
	1121	USAGE:
	1122	myopen(FN,mode,compression=1)
	1123
	1124	ARGUMENTS:
	1125	FN -- File name to be opened
	1126	mode -- Open mode (as in open)
	1127	compression -- Flag zlib compression
	1128
	1129	DESCRIPTION:
	1130	if compression
	1131	Attempt first to open FN + '.z'
	1132	If this fails try to open FN
	1133	else do the opposite
	1134	Return file handle plus info about whether it was compressed or not.
	1135	"""
	1136
	1137	import string
	1138
	1139	# Determine if file exists already (if writing was requested)
	1140	# This info is only used to determine if access modes should be set
	1141	#
	1142	if 'w' in mode or 'a' in mode:
	1143	try:
	1144	file = open(FN+'.z','r')
	1145	file.close()
	1146	new_file = 0
	1147	except:
	1148	try:
	1149	file = open(FN,'r')
	1150	file.close()
	1151	new_file = 0
	1152	except:
	1153	new_file = 1
	1154	else:
	1155	new_file = 0 #Assume it exists if mode was not 'w'
	1156
	1157
	1158	compressed = 0
	1159	if compression:
	1160	try:
	1161	file = open(FN+'.z',mode)
	1162	compressed = 1
	1163	except:
	1164	try:
	1165	file = open(FN,mode)
	1166	except:
	1167	file = None
	1168	else:
	1169	try:
	1170	file = open(FN,mode)
	1171	except:
	1172	try:
	1173	file = open(FN+'.z',mode)
	1174	compressed = 1
	1175	except:
	1176	file = None
	1177
	1178	# Now set access rights if it is a new file
	1179	#
	1180	if file and new_file:
	1181	if unix:
	1182	exitcode=os.system('chmod 666 '+file.name)
	1183	else:
	1184	pass # FIXME: Take care of access rights under Windows
	1185
	1186	return(file,compressed)
	1187
	1188	# -----------------------------------------------------------------------------
	1189
	1190	def myload(file, compressed):
	1191	"""Load data from file
	1192
	1193	USAGE:
	1194	myload(file, compressed)
	1195	"""
	1196
	1197	reason = 0
	1198	try:
	1199	if compressed:
	1200	import zlib
	1201
	1202	RsC = file.read()
	1203	try:
	1204	Rs = zlib.decompress(RsC)
	1205	except:
	1206	# File "./caching.py", line 1032, in load_from_cache
	1207	# T = myload(datafile,compressed)
	1208	# File "./caching.py", line 1124, in myload
	1209	# Rs = zlib.decompress(RsC)
	1210	# zlib.error: Error -5 while decompressing data
	1211	#print 'ERROR (caching): Could not decompress ', file.name
	1212	#raise Exception
	1213	reason = 5 #(Unreadable file)
	1214	return None, reason
	1215
	1216
	1217	del RsC # Free up some space
	1218	R = pickler.loads(Rs)
	1219	else:
[2043]	1220	try:
	1221	R = pickler.load(file)
	1222	#except EOFError, e:
	1223	except:
	1224	#Catch e.g., file with 0 length or corrupted
	1225	reason = 5 #(Unreadable file)
	1226	return None, reason
	1227
[130]	1228	except MemoryError:
	1229	import sys
	1230	if options['verbose']:
	1231	print 'ERROR (caching): Out of memory while loading %s, aborting' \
	1232	%(file.name)
	1233
	1234	# Raise the error again for now
	1235	#
	1236	raise MemoryError
	1237
	1238	return R, reason
	1239
	1240	# -----------------------------------------------------------------------------
	1241
	1242	def mysave(T,file,compression):
	1243	"""Save data T to file
	1244
	1245	USAGE:
	1246	mysave(T,file,compression)
	1247
	1248	"""
	1249
	1250	bin = options['bin']
	1251
	1252	if compression:
	1253	try:
	1254	import zlib
	1255	except:
	1256	print
	1257	print '* Could not find zlib *'
	1258	print '* Try to run caching with compression off *'
	1259	print "* caching.set_option('compression', 0) *"
	1260	raise Exception
	1261
	1262
[1895]	1263	try:
[2043]	1264	Ts = pickler.dumps(T, bin)
[1895]	1265	except MemoryError:
	1266	msg = '****WARNING (caching.py): Could not pickle data for compression.'
	1267	msg += ' Try using compression = False'
	1268	raise MemoryError, msg
	1269	else:
	1270	#Compressed pickling
[2043]	1271	TsC = zlib.compress(Ts, comp_level)
[1895]	1272	file.write(TsC)
[130]	1273	else:
[1895]	1274	#Uncompressed pickling
[2043]	1275	pickler.dump(T, file, bin)
[130]	1276
[2043]	1277	# FIXME: This may not work on Windoze network drives.
	1278	# The error msg is IOError: [Errno 22] Invalid argument
	1279	# Testing with small files was OK, though.
	1280	# I think this is an OS problem.
	1281
	1282	# Excerpt from http://www.ultraseek.com/support/faqs/4173.html
[1895]	1283
[2043]	1284	# The error is caused when there is a problem with server disk access (I/0). This happens at the OS level, and there is no controlling these errors through the Ultraseek application.
	1285	#
	1286	#Ultraseek contains an embedded Python interpreter. The exception "exceptions.IOError: [Errno 22] Invalid argument" is generated by the Python interpreter. The exception is thrown when a disk access operation fails due to an I/O-related reason.
	1287	#
	1288	#The following extract is taken from the site http://www.python.org:
	1289	#
	1290	#---------------------------------------------------------------------------------------------
	1291	#exception IOError
	1292	#Raised when an I/O operation (such as a print statement, the built-in open() function or a method of a file object) fails for an I/O-related reason, e.g., ``file not found'' or ``disk full''.
	1293	#This class is derived from EnvironmentError. See the discussion above for more information on exception instance attributes.
	1294	#---------------------------------------------------------------------------------------------
	1295	#
	1296	#The error code(s) that accompany exceptions are described at:
	1297	#http://www.python.org/dev/doc/devel//lib/module-errno.html
	1298	#
	1299	#You can view several postings on this error message by going to http://www.python.org, and typing the below into the search box:
	1300	#
	1301	#exceptions.IOError invalid argument Errno 22
	1302
	1303	#try:
	1304	# pickler.dump(T,file,bin)
	1305	#except IOError, e:
	1306	# print e
	1307	# msg = 'Could not store to %s, bin=%s' %(file, bin)
	1308	# raise msg
	1309
[1895]	1310
[130]	1311	# -----------------------------------------------------------------------------
	1312
	1313	def myhash(T):
	1314	"""Compute hashed integer from hashable values of tuple T
	1315
	1316	USAGE:
	1317	myhash(T)
	1318
	1319	ARGUMENTS:
	1320	T -- Tuple
	1321	"""
	1322
	1323	import types
	1324
	1325	# Get hash vals for hashable entries
	1326	#
	1327	if type(T) == types.TupleType or type(T) == types.ListType:
	1328	hvals = []
	1329	for k in range(len(T)):
	1330	h = myhash(T[k])
	1331	hvals.append(h)
	1332	val = hash(tuple(hvals))
	1333	elif type(T) == types.DictType:
	1334	val = dicthash(T)
	1335	else:
	1336	try:
	1337	val = hash(T)
	1338	except:
	1339	val = 1
	1340	try:
	1341	import Numeric
	1342	if type(T) == Numeric.ArrayType:
	1343	hvals = []
	1344	for e in T:
	1345	h = myhash(e)
	1346	hvals.append(h)
	1347	val = hash(tuple(hvals))
	1348	else:
	1349	val = 1 #Could implement other Numeric types here
	1350	except:
	1351	pass
	1352
	1353	return(val)
	1354
	1355	# -----------------------------------------------------------------------------
	1356
	1357	def dicthash(D):
	1358	"""Compute hashed integer from hashable values of dictionary D
	1359
	1360	USAGE:
	1361	dicthash(D)
	1362	"""
	1363
	1364	keys = D.keys()
	1365
	1366	# Get hash values for hashable entries
	1367	#
	1368	hvals = []
	1369	for k in range(len(keys)):
	1370	try:
	1371	h = hash(D[keys[k]])
	1372	hvals.append(h)
	1373	except:
	1374	pass
	1375
	1376	# Hash obtained values into one value
	1377	#
	1378	return(hash(tuple(hvals)))
	1379
	1380	# -----------------------------------------------------------------------------
	1381
	1382	def compare(A,B):
	1383	"""Safe comparison of general objects
	1384
	1385	USAGE:
	1386	compare(A,B)
	1387
	1388	DESCRIPTION:
	1389	Return 1 if A and B they are identical, 0 otherwise
	1390	"""
	1391
	1392	try:
	1393	identical = (A == B)
	1394	except:
	1395	try:
	1396	identical = (pickler.dumps(A) == pickler.dumps(B))
	1397	except:
	1398	identical = 0
	1399
	1400	return(identical)
	1401
	1402	# -----------------------------------------------------------------------------
	1403
	1404	def nospace(s):
	1405	"""Replace spaces in string s with underscores
	1406
	1407	USAGE:
	1408	nospace(s)
	1409
	1410	ARGUMENTS:
	1411	s -- string
	1412	"""
	1413
	1414	import string
	1415
	1416	newstr = ''
	1417	for i in range(len(s)):
	1418	if s[i] == ' ':
	1419	newstr = newstr+'_'
	1420	else:
	1421	newstr = newstr+s[i]
	1422
	1423	return(newstr)
	1424
	1425	# -----------------------------------------------------------------------------
	1426
	1427	def get_funcname(func):
	1428	"""Retrieve name of function object func (depending on its type)
	1429
	1430	USAGE:
	1431	get_funcname(func)
	1432	"""
	1433
	1434	import types, string
	1435
	1436	if type(func) == types.FunctionType:
	1437	funcname = func.func_name
	1438	elif type(func) == types.BuiltinFunctionType:
	1439	funcname = func.__name__
	1440	else:
	1441	tab = string.maketrans("<>'"," ")
	1442	tmp = string.translate(`func`,tab)
	1443	tmp = string.split(tmp)
	1444	funcname = string.join(tmp)
	1445
	1446	funcname = nospace(funcname)
	1447	return(funcname)
	1448
	1449	# -----------------------------------------------------------------------------
	1450
	1451	def get_bytecode(func):
	1452	""" Get bytecode from function object.
	1453
	1454	USAGE:
	1455	get_bytecode(func)
	1456	"""
	1457
	1458	import types
	1459
	1460	if type(func) == types.FunctionType:
	1461	bytecode = func.func_code.co_code
	1462	consts = func.func_code.co_consts
	1463	argcount = func.func_code.co_argcount
	1464	defaults = func.func_defaults
	1465	elif type(func) == types.MethodType:
	1466	bytecode = func.im_func.func_code.co_code
	1467	consts = func.im_func.func_code.co_consts
	1468	argcount = func.im_func.func_code.co_argcount
	1469	defaults = func.im_func.func_defaults
	1470	else:
	1471	#raise Exception #Test only
	1472	bytecode = None #Built-in functions are assumed not to change
	1473	consts = 0
	1474	argcount = 0
	1475	defaults = 0
	1476
	1477	return (bytecode, consts, argcount, defaults)
	1478
	1479	# -----------------------------------------------------------------------------
	1480
	1481	def get_depstats(dependencies):
	1482	""" Build dictionary of dependency files and their size, mod. time and ctime.
	1483
	1484	USAGE:
	1485	get_depstats(dependencies):
	1486	"""
	1487
	1488	import types
	1489
	1490	d = {}
	1491	if dependencies:
	1492
	1493	#Expand any wildcards
	1494	import glob
	1495	expanded_dependencies = []
	1496	for FN in dependencies:
	1497	expanded_FN = glob.glob(FN)
	1498
	1499	expanded_dependencies += expanded_FN
	1500
	1501
	1502	for FN in expanded_dependencies:
	1503	if not type(FN) == types.StringType:
	1504	errmsg = 'ERROR (caching.py): Dependency must be a string.\n'
	1505	errmsg += ' Dependency given: %s' %FN
	1506	raise Exception, errmsg
	1507	if not os.access(FN,os.F_OK):
	1508	errmsg = 'ERROR (caching.py): Dependency '+FN+' does not exist.'
	1509	raise Exception, errmsg
	1510	(size,atime,mtime,ctime) = filestat(FN)
	1511
	1512	# We don't use atime because that would cause recomputation every time.
	1513	# We don't use ctime because that is irrelevant and confusing for users.
	1514	d.update({FN : (size,mtime)})
	1515
	1516	return(d)
	1517
	1518	# -----------------------------------------------------------------------------
	1519
	1520	def filestat(FN):
	1521	"""A safe wrapper using os.stat to get basic file statistics
	1522	The built-in os.stat breaks down if file sizes are too large (> 2GB ?)
	1523
	1524	USAGE:
	1525	filestat(FN)
	1526
	1527	DESCRIPTION:
	1528	Must compile Python with
	1529	CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \
	1530	configure
	1531	as given in section 8.1.1 Large File Support in the Libray Reference
	1532	"""
	1533
	1534	import os, time
	1535
	1536	try:
	1537	stats = os.stat(FN)
	1538	size = stats[6]
	1539	atime = stats[7]
	1540	mtime = stats[8]
	1541	ctime = stats[9]
	1542	except:
	1543
	1544	# Hack to get the results anyway (works only on Unix at the moment)
	1545	#
	1546	print 'Hack to get os.stat when files are too large'
	1547
	1548	if unix:
	1549	tmp = '/tmp/cach.tmp.'+`time.time()`+`os.getpid()`
	1550	# Unique filename, FIXME: Use random number
	1551
	1552	# Get size and access time (atime)
	1553	#
	1554	exitcode=os.system('ls -l --full-time --time=atime '+FN+' > '+tmp)
	1555	(size,atime) = get_lsline(tmp)
	1556
	1557	# Get size and modification time (mtime)
	1558	#
	1559	exitcode=os.system('ls -l --full-time '+FN+' > '+tmp)
	1560	(size,mtime) = get_lsline(tmp)
	1561
	1562	# Get size and ctime
	1563	#
	1564	exitcode=os.system('ls -l --full-time --time=ctime '+FN+' > '+tmp)
	1565	(size,ctime) = get_lsline(tmp)
	1566
	1567	try:
	1568	exitcode=os.system('rm '+tmp)
	1569	# FIXME: Gives error if file doesn't exist
	1570	except:
	1571	pass
	1572	else:
	1573	pass
	1574	raise Exception # FIXME: Windows case
	1575
	1576	return(long(size),atime,mtime,ctime)
	1577
	1578	# -----------------------------------------------------------------------------
	1579
	1580	def get_lsline(FN):
	1581	"""get size and time for filename
	1582
	1583	USAGE:
	1584	get_lsline(file_name)
	1585
	1586	DESCRIPTION:
	1587	Read in one line 'ls -la' item from file (generated by filestat) and
	1588	convert time to seconds since epoch. Return file size and time.
	1589	"""
	1590
	1591	import string, time
	1592
	1593	f = open(FN,'r')
	1594	info = f.read()
	1595	info = string.split(info)
	1596
	1597	size = info[4]
	1598	week = info[5]
	1599	mon = info[6]
	1600	day = info[7]
	1601	hour = info[8]
	1602	year = info[9]
	1603
	1604	str = week+' '+mon+' '+day+' '+hour+' '+year
	1605	timetup = time.strptime(str)
	1606	t = time.mktime(timetup)
	1607	return(size, t)
	1608
	1609	# -----------------------------------------------------------------------------
	1610
	1611	def checkdir(CD,verbose=None):
	1612	"""Check or create caching directory
	1613
	1614	USAGE:
	1615	checkdir(CD,verbose):
	1616
	1617	ARGUMENTS:
	1618	CD -- Directory
	1619	verbose -- Flag verbose output (default: None)
	1620
	1621	DESCRIPTION:
	1622	If CD does not exist it will be created if possible
	1623	"""
	1624
	1625	import os
	1626	import os.path
	1627
	1628	if CD[-1] != os.sep:
	1629	CD = CD + os.sep # Add separator for directories
	1630
	1631	CD = os.path.expanduser(CD) # Expand ~ or ~user in pathname
	1632	if not (os.access(CD,os.R_OK and os.W_OK) or CD == ''):
	1633	try:
	1634	exitcode=os.mkdir(CD)
	1635
	1636	# Change access rights if possible
	1637	#
	1638	if unix:
	1639	exitcode=os.system('chmod 777 '+CD)
	1640	else:
	1641	pass # FIXME: What about acces rights under Windows?
	1642	if verbose: print 'MESSAGE: Directory', CD, 'created.'
	1643	except:
	1644	print 'WARNING: Directory', CD, 'could not be created.'
	1645	if unix:
	1646	CD = '/tmp/'
	1647	else:
	1648	CD = 'C:'
	1649	print 'Using directory %s instead' %CD
	1650
	1651	return(CD)
	1652
	1653	#==============================================================================
	1654	# Statistics
	1655	#==============================================================================
	1656
	1657	def addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,
	1658	compression):
	1659	"""Add stats entry
	1660
	1661	USAGE:
	1662	addstatsline(CD,funcname,FN,Retrieved,reason,comptime,loadtime,compression)
	1663
	1664	DESCRIPTION:
	1665	Make one entry in the stats file about one cache hit recording time saved
	1666	and other statistics. The data are used by the function cachestat.
	1667	"""
	1668
	1669	import os, time
	1670
	1671	try:
	1672	TimeTuple = time.localtime(time.time())
	1673	extension = time.strftime('%b%Y',TimeTuple)
	1674	SFN = CD+statsfile+'.'+extension
	1675	#statfile = open(SFN,'a')
	1676	(statfile, dummy) = myopen(SFN,'a',compression=0)
	1677
	1678	# Change access rights if possible
	1679	#
	1680	#if unix:
	1681	# try:
	1682	# exitcode=os.system('chmod 666 '+SFN)
	1683	# except:
	1684	# pass
	1685	except:
	1686	print 'Warning: Stat file could not be opened'
	1687
	1688	try:
	1689	if os.environ.has_key('USER'):
	1690	user = os.environ['USER']
	1691	else:
	1692	user = 'Nobody'
	1693
	1694	date = time.asctime(TimeTuple)
	1695
	1696	if Retrieved:
	1697	hit = '1'
	1698	else:
	1699	hit = '0'
	1700
	1701	# Get size of result file
	1702	#
	1703	if compression:
	1704	stats = os.stat(CD+FN+'_'+file_types[0]+'.z')
	1705	else:
	1706	stats = os.stat(CD+FN+'_'+file_types[0])
	1707
	1708	if stats:
	1709	size = stats[6]
	1710	else:
	1711	size = -1 # Error condition, but don't crash. This is just statistics
	1712
	1713	# Build entry
	1714	#
	1715	entry = date + ',' +\
	1716	user + ',' +\
	1717	FN + ',' +\
	1718	str(int(size)) + ',' +\
	1719	str(compression) + ',' +\
	1720	hit + ',' +\
	1721	str(reason) + ',' +\
	1722	str(round(comptime,4)) + ',' +\
	1723	str(round(loadtime,4)) +\
	1724	CR
	1725
	1726	statfile.write(entry)
	1727	statfile.close()
	1728	except:
	1729	print 'Warning: Writing of stat file failed'
	1730
	1731	# -----------------------------------------------------------------------------
	1732
	1733	# FIXME: should take cachedir as an optional arg
	1734	#
	1735	def __cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
	1736	""" List caching statistics.
	1737
	1738	USAGE:
	1739	__cachestat(sortidx=4,period=-1,showuser=None,cachedir=None):
	1740
	1741	Generate statistics of caching efficiency.
	1742	The parameter sortidx determines by what field lists are sorted.
	1743	If the optional keyword period is set to -1,
	1744	all available caching history is used.
	1745	If it is 0 only the current month is used.
	1746	Future versions will include more than one month....
	1747	OMN 20/8/2000
	1748	"""
	1749
	1750	import os
	1751	import os.path
	1752	from string import split, rstrip, find, atof, atoi
	1753	from time import strptime, localtime, strftime, mktime, ctime
	1754
	1755	# sortidx = 4 # Index into Fields[1:]. What to sort by.
	1756
	1757	Fields = ['Name', 'Hits', 'Exec(s)', \
	1758	'Cache(s)', 'Saved(s)', 'Gain(%)', 'Size']
	1759	Widths = [25,7,9,9,9,9,13]
	1760	#Types = ['s','d','d','d','d','.2f','d']
	1761	Types = ['s','d','.2f','.2f','.2f','.2f','d']
	1762
	1763	Dictnames = ['Function', 'User']
	1764
	1765	if not cachedir:
	1766	cachedir = checkdir(options['cachedir'])
	1767
	1768	SD = os.path.expanduser(cachedir) # Expand ~ or ~user in pathname
	1769
	1770	if period == -1: # Take all available stats
	1771	SFILENAME = statsfile
	1772	else: # Only stats from current month
	1773	# MAKE THIS MORE GENERAL SO period > 0 counts several months backwards!
	1774	TimeTuple = localtime(time())
	1775	extension = strftime('%b%Y',TimeTuple)
	1776	SFILENAME = statsfile+'.'+extension
	1777
	1778	DIRLIST = os.listdir(SD)
	1779	SF = []
	1780	for FN in DIRLIST:
	1781	if find(FN,SFILENAME) >= 0:
	1782	SF.append(FN)
	1783
	1784	blocksize = 15000000
	1785	total_read = 0
	1786	total_hits = 0
	1787	total_discarded = 0
	1788	firstday = mktime(strptime('2030','%Y'))
	1789	# FIXME: strptime don't exist in WINDOWS ?
	1790	lastday = 0
	1791
	1792	FuncDict = {}
	1793	UserDict = {}
	1794	for FN in SF:
	1795	input = open(SD+FN,'r')
	1796	print 'Reading file ', SD+FN
	1797
	1798	while 1:
	1799	A = input.readlines(blocksize)
	1800	if len(A) == 0: break
	1801	total_read = total_read + len(A)
	1802	for record in A:
	1803	record = tuple(split(rstrip(record),','))
	1804	#print record
	1805
	1806	if len(record) in [8,9]:
	1807	n = 0
	1808	timestamp = record[n]; n=n+1
	1809
	1810	try:
	1811	t = mktime(strptime(timestamp))
	1812	except:
	1813	total_discarded = total_discarded + 1
	1814	continue
	1815
	1816	if t > lastday:
	1817	lastday = t
	1818	if t < firstday:
	1819	firstday = t
	1820
	1821	user = record[n]; n=n+1
	1822	func = record[n]; n=n+1
	1823
	1824	# Strip hash-stamp off
	1825	#
	1826	i = find(func,'[')
	1827	func = func[:i]
	1828
	1829	size = atof(record[n]); n=n+1
	1830	compression = atoi(record[n]); n=n+1
	1831	hit = atoi(record[n]); n=n+1
	1832	reason = atoi(record[n]); n=n+1 # Not used here
	1833	cputime = atof(record[n]); n=n+1
	1834	loadtime = atof(record[n]); n=n+1
	1835
	1836	if hit:
	1837	total_hits = total_hits + 1
	1838	saving = cputime-loadtime
	1839
	1840	if cputime != 0:
	1841	rel_saving = round(100.0*saving/cputime,2)
	1842	else:
	1843	#rel_saving = round(1.0*saving,2)
	1844	rel_saving = 100.0 - round(1.0*saving,2) # A bit of a hack
	1845
	1846	info = [1,cputime,loadtime,saving,rel_saving,size]
	1847
	1848	UpdateDict(UserDict,user,info)
	1849	UpdateDict(FuncDict,func,info)
	1850	else:
	1851	pass #Stats on recomputations and their reasons could go in here
	1852
	1853	else:
	1854	#print 'Record discarded'
	1855	#print record
	1856	total_discarded = total_discarded + 1
	1857
	1858	input.close()
	1859
	1860	# Compute averages of all sums and write list
	1861	#
	1862
	1863	if total_read == 0:
	1864	printline(Widths,'=')
	1865	print 'CACHING STATISTICS: No valid records read'
	1866	printline(Widths,'=')
	1867	return
	1868
	1869	print
	1870	printline(Widths,'=')
	1871	print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)
	1872	printline(Widths,'=')
	1873	#print ' Period:', ctime(firstday), 'to', ctime(lastday)
	1874	print ' Total number of valid records', total_read
	1875	print ' Total number of discarded records', total_discarded
	1876	print ' Total number of hits', total_hits
	1877	print
	1878
	1879	print ' Fields', Fields[2:], 'are averaged over number of hits'
	1880	print ' Time is measured in seconds and size in bytes'
	1881	print ' Tables are sorted by', Fields[1:][sortidx]
	1882
	1883	# printline(Widths,'-')
	1884
	1885	if showuser:
	1886	Dictionaries = [FuncDict, UserDict]
	1887	else:
	1888	Dictionaries = [FuncDict]
	1889
	1890	i = 0
	1891	for Dict in Dictionaries:
	1892	for key in Dict.keys():
	1893	rec = Dict[key]
	1894	for n in range(len(rec)):
	1895	if n > 0:
	1896	rec[n] = round(1.0*rec[n]/rec[0],2)
	1897	Dict[key] = rec
	1898
	1899	# Sort and output
	1900	#
	1901	keylist = SortDict(Dict,sortidx)
	1902
	1903	# Write Header
	1904	#
	1905	print
	1906	#print Dictnames[i], 'statistics:'; i=i+1
	1907	printline(Widths,'-')
	1908	n = 0
	1909	for s in Fields:
	1910	if s == Fields[0]: # Left justify
	1911	s = Dictnames[i] + ' ' + s; i=i+1
	1912	exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1
	1913	else:
	1914	exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1
	1915	print
	1916	printline(Widths,'-')
	1917
	1918	# Output Values
	1919	#
	1920	for key in keylist:
	1921	rec = Dict[key]
	1922	n = 0
	1923	if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...'
	1924	exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1
	1925	for val in rec:
	1926	exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1
	1927	print
	1928	print
	1929
	1930	#==============================================================================
	1931	# Auxiliary stats functions
	1932	#==============================================================================
	1933
	1934	def UpdateDict(Dict,key,info):
	1935	"""Update dictionary by adding new values to existing.
	1936
	1937	USAGE:
	1938	UpdateDict(Dict,key,info)
	1939	"""
	1940
	1941	if Dict.has_key(key):
	1942	dinfo = Dict[key]
	1943	for n in range(len(dinfo)):
	1944	dinfo[n] = info[n] + dinfo[n]
	1945	else:
	1946	dinfo = info[:] # Make a copy of info list
	1947
	1948	Dict[key] = dinfo
	1949	return Dict
	1950
	1951	# -----------------------------------------------------------------------------
	1952
	1953	def SortDict(Dict,sortidx=0):
	1954	"""Sort dictionary
	1955
	1956	USAGE:
	1957	SortDict(Dict,sortidx):
	1958
	1959	DESCRIPTION:
	1960	Sort dictionary of lists according field number 'sortidx'
	1961	"""
	1962
	1963	import types
	1964
	1965	sortlist = []
	1966	keylist = Dict.keys()
	1967	for key in keylist:
	1968	rec = Dict[key]
	1969	if not type(rec) in [types.ListType, types.TupleType]:
	1970	rec = [rec]
	1971
	1972	if sortidx > len(rec)-1:
	1973	if options['verbose']:
	1974	print 'ERROR: Sorting index to large, sortidx = ', sortidx
	1975	raise IndexError
	1976
	1977	val = rec[sortidx]
	1978	sortlist.append(val)
	1979
	1980	A = map(None,sortlist,keylist)
	1981	A.sort()
	1982	keylist = map(lambda x: x[1], A) # keylist sorted by sortidx
	1983
	1984	return(keylist)
	1985
	1986	# -----------------------------------------------------------------------------
	1987
	1988	def printline(Widths,char):
	1989	"""Print textline in fixed field.
	1990
	1991	USAGE:
	1992	printline(Widths,char)
	1993	"""
	1994
	1995	s = ''
	1996	for n in range(len(Widths)):
	1997	s = s+Widths[n]*char
	1998	if n > 0:
	1999	s = s+char
	2000
	2001	print s
	2002
	2003	#==============================================================================
	2004	# Messages
	2005	#==============================================================================
	2006
	2007	def msg1(funcname,args,kwargs,reason):
	2008	"""Message 1
	2009
	2010	USAGE:
	2011	msg1(funcname,args,kwargs,reason):
	2012	"""
	2013
	2014	import string
	2015	#print 'MESSAGE (caching.py): Evaluating function', funcname,
	2016
	2017	print_header_box('Evaluating function %s' %funcname)
	2018
	2019	msg7(args,kwargs)
	2020	msg8(reason)
	2021
	2022	print_footer()
	2023
	2024	#
	2025	# Old message
	2026	#
	2027	#args_present = 0
	2028	#if args:
	2029	# if len(args) == 1:
	2030	# print 'with argument', mkargstr(args[0], textwidth2),
	2031	# else:
	2032	# print 'with arguments', mkargstr(args, textwidth2),
	2033	# args_present = 1
	2034	#
	2035	#if kwargs:
	2036	# if args_present:
	2037	# word = 'and'
	2038	# else:
	2039	# word = 'with'
	2040	#
	2041	# if len(kwargs) == 1:
	2042	# print word + ' keyword argument', mkargstr(kwargs, textwidth2)
	2043	# else:
	2044	# print word + ' keyword arguments', mkargstr(kwargs, textwidth2)
	2045	# args_present = 1
	2046	#else:
	2047	# print # Newline when no keyword args present
	2048	#
	2049	#if not args_present:
	2050	# print '', # Default if no args or kwargs present
	2051
	2052
	2053
	2054	# -----------------------------------------------------------------------------
	2055
	2056	def msg2(funcname,args,kwargs,comptime,reason):
	2057	"""Message 2
	2058
	2059	USAGE:
	2060	msg2(funcname,args,kwargs,comptime,reason)
	2061	"""
	2062
	2063	import string
	2064
	2065	#try:
	2066	# R = Reason_msg[reason]
	2067	#except:
	2068	# R = 'Unknown reason'
	2069
	2070	#print_header_box('Caching statistics (storing) - %s' %R)
	2071	print_header_box('Caching statistics (storing)')
	2072
	2073	msg6(funcname,args,kwargs)
	2074	msg8(reason)
	2075
	2076	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
	2077
	2078	# -----------------------------------------------------------------------------
	2079
	2080	def msg3(savetime, CD, FN, deps,compression):
	2081	"""Message 3
	2082
	2083	USAGE:
	2084	msg3(savetime, CD, FN, deps,compression)
	2085	"""
	2086
	2087	import string
	2088	print string.ljust('\| Loading time:', textwidth1) + str(round(savetime,2)) + \
	2089	' seconds (estimated)'
	2090	msg5(CD,FN,deps,compression)
	2091
	2092	# -----------------------------------------------------------------------------
	2093
	2094	def msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression):
	2095	"""Message 4
	2096
	2097	USAGE:
	2098	msg4(funcname,args,kwargs,deps,comptime,loadtime,CD,FN,compression)
	2099	"""
	2100
	2101	import string
	2102
	2103	print_header_box('Caching statistics (retrieving)')
	2104
	2105	msg6(funcname,args,kwargs)
	2106	print string.ljust('\| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds'
	2107	print string.ljust('\| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds'
	2108	print string.ljust('\| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \
	2109	' seconds'
	2110	msg5(CD,FN,deps,compression)
	2111
	2112	# -----------------------------------------------------------------------------
	2113
	2114	def msg5(CD,FN,deps,compression):
	2115	"""Message 5
	2116
	2117	USAGE:
	2118	msg5(CD,FN,deps,compression)
	2119
	2120	DESCRIPTION:
	2121	Print dependency stats. Used by msg3 and msg4
	2122	"""
	2123
	2124	import os, time, string
	2125
	2126	print '\|'
	2127	print string.ljust('\| Caching dir: ', textwidth1) + CD
	2128
	2129	if compression:
	2130	suffix = '.z'
	2131	bytetext = 'bytes, compressed'
	2132	else:
	2133	suffix = ''
	2134	bytetext = 'bytes'
	2135
	2136	for file_type in file_types:
	2137	file_name = FN + '_' + file_type + suffix
	2138	print string.ljust('\| ' + file_type + ' file: ', textwidth1) + file_name,
	2139	stats = os.stat(CD+file_name)
	2140	print '('+ str(stats[6]) + ' ' + bytetext + ')'
	2141
	2142	print '\|'
	2143	if len(deps) > 0:
	2144	print '\| Dependencies: '
	2145	dependencies = deps.keys()
	2146	dlist = []; maxd = 0
	2147	tlist = []; maxt = 0
	2148	slist = []; maxs = 0
	2149	for d in dependencies:
	2150	stats = deps[d]
	2151	t = time.ctime(stats[1])
	2152	s = str(stats[0])
	2153	#if s[-1] == 'L':
	2154	# s = s[:-1] # Strip rightmost 'long integer' L off.
	2155	# # FIXME: Unnecessary in versions later than 1.5.2
	2156
	2157	if len(d) > maxd: maxd = len(d)
	2158	if len(t) > maxt: maxt = len(t)
	2159	if len(s) > maxs: maxs = len(s)
	2160	dlist.append(d)
	2161	tlist.append(t)
	2162	slist.append(s)
	2163
	2164	for n in range(len(dlist)):
	2165	d = string.ljust(dlist[n]+':', maxd+1)
	2166	t = string.ljust(tlist[n], maxt)
	2167	s = string.rjust(slist[n], maxs)
	2168
	2169	print '\| ', d, t, ' ', s, 'bytes'
	2170	else:
	2171	print '\| No dependencies'
	2172	print_footer()
	2173
	2174	# -----------------------------------------------------------------------------
	2175
	2176	def msg6(funcname,args,kwargs):
	2177	"""Message 6
	2178
	2179	USAGE:
	2180	msg6(funcname,args,kwargs)
	2181	"""
	2182
	2183	import string
	2184	print string.ljust('\| Function:', textwidth1) + funcname
	2185
	2186	msg7(args,kwargs)
	2187
	2188	# -----------------------------------------------------------------------------
	2189
	2190	def msg7(args,kwargs):
	2191	"""Message 7
	2192
	2193	USAGE:
	2194	msg7(args,kwargs):
	2195	"""
	2196
	2197	import string
	2198
	2199	args_present = 0
	2200	if args:
	2201	if len(args) == 1:
	2202	print string.ljust('\| Argument:', textwidth1) + mkargstr(args[0], \
	2203	textwidth2)
	2204	else:
	2205	print string.ljust('\| Arguments:', textwidth1) + \
	2206	mkargstr(args, textwidth2)
	2207	args_present = 1
	2208
	2209	if kwargs:
	2210	if len(kwargs) == 1:
	2211	print string.ljust('\| Keyword Arg:', textwidth1) + mkargstr(kwargs, \
	2212	textwidth2)
	2213	else:
	2214	print string.ljust('\| Keyword Args:', textwidth1) + \
	2215	mkargstr(kwargs, textwidth2)
	2216	args_present = 1
	2217
	2218	if not args_present:
	2219	print '\| No arguments' # Default if no args or kwargs present
	2220
	2221	# -----------------------------------------------------------------------------
	2222
	2223	def msg8(reason):
	2224	"""Message 8
	2225
	2226	USAGE:
	2227	msg8(reason):
	2228	"""
	2229
	2230	import string
	2231
	2232	try:
	2233	R = Reason_msg[reason]
	2234	except:
	2235	R = 'Unknown'
	2236
	2237	print string.ljust('\| Reason:', textwidth1) + R
	2238
	2239	# -----------------------------------------------------------------------------
	2240
	2241	def print_header_box(line):
	2242	"""Print line in a nice box.
	2243
	2244	USAGE:
	2245	print_header_box(line)
	2246
	2247	"""
	2248	global textwidth3
	2249
	2250	import time
	2251
	2252	time_stamp = time.ctime(time.time())
	2253	line = time_stamp + '. ' + line
	2254
	2255	N = len(line) + 1
	2256	s = '+' + '-'*N + CR
	2257
	2258	print s + '\| ' + line + CR + s,
	2259
	2260	textwidth3 = N
	2261
	2262	# -----------------------------------------------------------------------------
	2263
	2264	def print_footer():
	2265	"""Print line same width as that of print_header_box.
	2266	"""
	2267
	2268	N = textwidth3
	2269	s = '+' + '-'*N + CR
	2270
	2271	print s
	2272
	2273	# -----------------------------------------------------------------------------
	2274
	2275	def mkargstr(args, textwidth, argstr = ''):
	2276	""" Generate a string containing first textwidth characters of arguments.
	2277
	2278	USAGE:
	2279	mkargstr(args, textwidth, argstr = '')
	2280
	2281	DESCRIPTION:
	2282	Exactly the same as str(args) possibly followed by truncation,
	2283	but faster if args is huge.
	2284	"""
	2285
	2286	import types
	2287
	2288	WasTruncated = 0
	2289
	2290	if not type(args) in [types.TupleType, types.ListType, types.DictType]:
	2291	if type(args) == types.StringType:
	2292	argstr = argstr + "'"+str(args)+"'"
	2293	else:
[1746]	2294	#Truncate large Numeric arrays before using str()
[1745]	2295	import Numeric
	2296	if type(args) == Numeric.ArrayType:
	2297	if len(args.flat) > textwidth:
[1746]	2298	args = 'Array: ' + str(args.shape)
[1745]	2299
[130]	2300	argstr = argstr + str(args)
	2301	else:
	2302	if type(args) == types.DictType:
	2303	argstr = argstr + "{"
	2304	for key in args.keys():
	2305	argstr = argstr + mkargstr(key, textwidth) + ": " + \
	2306	mkargstr(args[key], textwidth) + ", "
	2307	if len(argstr) > textwidth:
	2308	WasTruncated = 1
	2309	break
	2310	argstr = argstr[:-2] # Strip off trailing comma
	2311	argstr = argstr + "}"
	2312
	2313	else:
	2314	if type(args) == types.TupleType:
	2315	lc = '('
	2316	rc = ')'
	2317	else:
	2318	lc = '['
	2319	rc = ']'
	2320	argstr = argstr + lc
	2321	for arg in args:
	2322	argstr = argstr + mkargstr(arg, textwidth) + ', '
	2323	if len(argstr) > textwidth:
	2324	WasTruncated = 1
	2325	break
	2326
	2327	# Strip off trailing comma and space unless singleton tuple
	2328	#
	2329	if type(args) == types.TupleType and len(args) == 1:
	2330	argstr = argstr[:-1]
	2331	else:
	2332	argstr = argstr[:-2]
	2333	argstr = argstr + rc
	2334
	2335	if len(argstr) > textwidth:
	2336	WasTruncated = 1
	2337
	2338	if WasTruncated:
	2339	argstr = argstr[:textwidth]+'...'
	2340	return(argstr)
	2341
	2342	# -----------------------------------------------------------------------------
	2343
	2344	def test_OK(msg):
	2345	"""Print OK msg if test is OK.
	2346
	2347	USAGE
	2348	test_OK(message)
	2349	"""
	2350
	2351	import string
	2352
	2353	print string.ljust(msg, textwidth4) + ' - OK'
	2354
	2355	#raise StandardError
	2356
	2357	# -----------------------------------------------------------------------------
	2358
	2359	def test_error(msg):
	2360	"""Print error if test fails.
	2361
	2362	USAGE
	2363	test_error(message)
	2364	"""
	2365
	2366	print 'ERROR (caching.test): %s' %msg
	2367	print 'Please send this code example and output to '
	2368	print 'Ole.Nielsen@anu.edu.au'
	2369	print
	2370	print
	2371
	2372	#import sys
	2373	#sys.exit()
	2374	raise StandardError

Note: See TracBrowser for help on using the repository browser.

Download in other formats: