Changeset 7317 for anuga_core/source/anuga/caching/caching.py
- Timestamp:
- Jul 22, 2009, 9:22:11 AM (15 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
anuga_core/source/anuga/caching/caching.py
r7309 r7317 51 51 unix = True 52 52 53 import anuga.utilities.log as log 54 53 55 import numpy as num 54 56 … … 332 334 os.system('del '+fn) 333 335 if verbose is True: 334 print 'MESSAGE (caching): File %s deleted' %fn336 log.critical('MESSAGE (caching): File %s deleted' % fn) 335 337 ##else: 336 ## print '%s was not accessed' %fn338 ## log.critical('%s was not accessed' % fn) 337 339 return None 338 340 … … 468 470 import zlib 469 471 except: 470 print471 print '*** Could not find zlib, default to no-compression ***'472 print '*** Installing zlib will improve performance of caching ***'473 print472 log.critical() 473 log.critical('*** Could not find zlib, default to no-compression ***') 474 log.critical('*** Installing zlib will improve performance of caching ***') 475 log.critical() 474 476 compression = 0 475 477 set_option('compression', compression) 476 478 477 print 478 print_header_box('Testing caching module - please stand by') 479 print 479 log.critical('\nTesting caching module - please stand by\n') 480 480 481 481 # Define a test function to be cached … … 709 709 test_OK('Performance test: relative time saved = %s pct' \ 710 710 %str(round((t1-t2)*100/t1,2))) 711 #else:712 # print 'WARNING: Performance a bit low - this could be specific to current platform'713 711 else: 714 712 test_error('Basic caching failed for new problem') … … 742 740 cachestat() 743 741 except: 744 print 'cachestat() does not work here, because it'745 print 'relies on time.strptime() which is unavailable in Windows'742 log.critical('cachestat() does not work here, because it relies on ' 743 'time.strptime() which is unavailable in Windows') 746 744 747 print748 745 test_OK('Caching self test completed') 749 746 … … 841 838 # 842 839 if verbose: 843 print 'Caching: looking for cached files %s_{%s,%s,%s}.z'\844 %(CD+FN, file_types[0], file_types[1], file_types[2])840 log.critical('Caching: looking for cached files %s_{%s,%s,%s}.z' 841 % (CD+FN, file_types[0], file_types[1], file_types[2])) 845 842 (datafile,compressed0) = myopen(CD+FN+'_'+file_types[0],"rb",compression) 846 843 (argsfile,compressed1) = myopen(CD+FN+'_'+file_types[1],"rb",compression) … … 848 845 849 846 if verbose is True and deps is not None: 850 print 'Caching: Dependencies are', deps.keys()847 log.critical('Caching: Dependencies are %s' % deps.keys()) 851 848 852 849 if not (argsfile and datafile and admfile) or \ … … 890 887 if dependencies and not compare(depsref, deps): 891 888 if verbose: 892 print 'MESSAGE (caching.py): Dependencies', dependencies, \893 'have changed - recomputing' 889 log.critical('Dependencies %s have changed - recomputing' % dependencies) 890 894 891 # Don't use cached file - recompute 895 892 reason = 2 … … 899 896 # 900 897 bytecode = get_bytecode(my_F) 901 902 #print compare(argsref,args),903 #print compare(kwargsref,kwargs),904 #print compare(bytecode,coderef)905 898 906 899 # Check if arguments or bytecode have changed … … 921 914 922 915 if loadtime >= comptime: 923 print 'WARNING (caching.py): Caching did not yield any gain.'924 print ' Consider executing function ',925 print '('+funcname+') without caching.'916 log.critical('Caching did not yield any gain.') 917 log.critical('Consider executing function %s without caching.' 918 % funcname) 926 919 else: 927 920 … … 934 927 verbose, compression, dependencies) 935 928 936 # DEBUGGING937 # if not Retrieved:938 # print 'Arguments did not match'939 # else:940 # print 'Match found !'941 942 929 # The real reason is that args or bytecodes have changed. 943 930 # Not that the recursive seach has found an unused filename … … 983 970 funcname = get_funcname(my_F) 984 971 if verbose: 985 print 'MESSAGE (caching.py): Clearing', CD+funcname+'*'972 log.critical('Clearing %s' % CD+funcname+'*') 986 973 987 974 file_names = os.listdir(CD) … … 999 986 if len(file_names) > 0: 1000 987 if verbose: 1001 print 'MESSAGE (caching.py): Remove the following files:'988 log.critical('Remove the following files:') 1002 989 for file_name in file_names: 1003 print file_name990 log.critical(' ' + file_name) 1004 991 1005 992 A = raw_input('Delete (Y/N)[N] ?') … … 1041 1028 delfiles = numfiles-maxfiles+block 1042 1029 if verbose: 1043 print 'Deleting '+`delfiles`+' expired files:'1030 log.critical('Deleting %d expired files:' % delfiles) 1044 1031 os.system('ls -lur '+CD+'* | head -' + `delfiles`) # List them 1045 1032 os.system('ls -ur '+CD+'* | head -' + `delfiles` + ' | xargs /bin/rm') … … 1097 1084 if not datafile: 1098 1085 if verbose: 1099 print 'ERROR (caching): Could not open %s' %datafile.name1086 log.critical('ERROR: Could not open %s' % datafile.name) 1100 1087 raise IOError 1101 1088 1102 1089 if not admfile: 1103 1090 if verbose: 1104 print 'ERROR (caching): Could not open %s' %admfile.name1091 log.critical('ERROR: Could not open %s' % admfile.name) 1105 1092 raise IOError 1106 1093 … … 1246 1233 # Rs = zlib.decompress(RsC) 1247 1234 # zlib.error: Error -5 while decompressing data 1248 #print 'ERROR (caching): Could not decompress ', file.name1249 1235 #raise Exception 1250 1236 reason = 6 # Unreadable file … … 1266 1252 import sys 1267 1253 if options['verbose']: 1268 print 'ERROR (caching): Out of memory while loading %s, aborting' \1269 %(file.name)1254 log.critical('ERROR: Out of memory while loading %s, aborting' 1255 % file.name) 1270 1256 1271 1257 # Raise the error again for now … … 1291 1277 import zlib 1292 1278 except: 1293 print1294 print '*** Could not find zlib ***'1295 print '*** Try to run caching with compression off ***'1296 print "*** caching.set_option('compression', 0) ***"1279 log.critical() 1280 log.critical('*** Could not find zlib ***') 1281 log.critical('*** Try to run caching with compression off ***') 1282 log.critical("*** caching.set_option('compression', 0) ***") 1297 1283 raise Exception 1298 1284 … … 1440 1426 if (iA, iB) in ids: 1441 1427 # A and B have been compared already 1442 #print 'Found', (iA, iB), A, B1443 1428 return ids[(iA, iB)] 1444 1429 else: … … 1618 1603 import types 1619 1604 1620 #print 'Caching DEBUG: Dependencies are', dependencies1621 1605 d = {} 1622 1606 if dependencies: … … 1679 1663 # Hack to get the results anyway (works only on Unix at the moment) 1680 1664 # 1681 print 'Hack to get os.stat when files are too large'1665 log.critical('Hack to get os.stat when files are too large') 1682 1666 1683 1667 if unix: … … 1775 1759 else: 1776 1760 pass # FIXME: What about acces rights under Windows? 1777 if verbose: print 'MESSAGE: Directory', CD, 'created.'1761 if verbose: log.critical('MESSAGE: Directory %s created.' % CD) 1778 1762 except: 1779 1763 if warn is True: 1780 print 'WARNING: Directory', CD, 'could not be created.'1764 log.critical('WARNING: Directory %s could not be created.' % CD) 1781 1765 if unix: 1782 1766 CD = '/tmp/' … … 1784 1768 CD = 'C:' 1785 1769 if warn is True: 1786 print 'Using directory %s instead' %CD1770 log.critical('Using directory %s instead' % CD) 1787 1771 1788 1772 return(CD) … … 1823 1807 # pass 1824 1808 except: 1825 print 'Warning: Stat file could not be opened'1809 log.critical('Warning: Stat file could not be opened') 1826 1810 1827 1811 try: … … 1866 1850 statfile.close() 1867 1851 except: 1868 print 'Warning: Writing of stat file failed'1852 log.critical('Warning: Writing of stat file failed') 1869 1853 1870 1854 # ----------------------------------------------------------------------------- … … 1933 1917 for FN in SF: 1934 1918 input = open(SD+FN,'r') 1935 print 'Reading file ', SD+FN1919 log.critical('Reading file %s' % SD+FN) 1936 1920 1937 1921 while True: … … 1941 1925 for record in A: 1942 1926 record = tuple(split(rstrip(record),',')) 1943 #print record, len(record)1944 1927 1945 1928 if len(record) == 9: … … 1973 1956 compression = 0 1974 1957 else: 1975 print 'Unknown value of compression', record[4]1976 print record1958 log.critical('Unknown value of compression %s' % str(record[4])) 1959 log.critical(str(record)) 1977 1960 total_discarded = total_discarded + 1 1978 1961 continue … … 2002 1985 2003 1986 else: 2004 #print 'Record discarded'2005 #print record2006 1987 total_discarded = total_discarded + 1 2007 1988 … … 2013 1994 if total_read == 0: 2014 1995 printline(Widths,'=') 2015 print 'CACHING STATISTICS: No valid records read'1996 log.critical('CACHING STATISTICS: No valid records read') 2016 1997 printline(Widths,'=') 2017 1998 return 2018 1999 2019 print2000 log.critical() 2020 2001 printline(Widths,'=') 2021 print 'CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)2002 log.critical('CACHING STATISTICS: '+ctime(firstday)+' to '+ctime(lastday)) 2022 2003 printline(Widths,'=') 2023 #print ' Period:', ctime(firstday), 'to', ctime(lastday) 2024 print ' Total number of valid records', total_read 2025 print ' Total number of discarded records', total_discarded 2026 print ' Total number of hits', total_hits 2027 print 2028 2029 print ' Fields', Fields[2:], 'are averaged over number of hits' 2030 print ' Time is measured in seconds and size in bytes' 2031 print ' Tables are sorted by', Fields[1:][sortidx] 2032 2033 # printline(Widths,'-') 2004 log.critical(' Total number of valid records %d' % total_read) 2005 log.critical(' Total number of discarded records %d' % total_discarded) 2006 log.critical(' Total number of hits %d' % total_hits) 2007 log.critical() 2008 2009 log.critical(' Fields %s are averaged over number of hits' % Fields[2:]) 2010 log.critical(' Time is measured in seconds and size in bytes') 2011 log.critical(' Tables are sorted by %s' % Fields[1:][sortidx]) 2034 2012 2035 2013 if showuser: … … 2053 2031 # Write Header 2054 2032 # 2055 print 2056 #print Dictnames[i], 'statistics:'; i=i+1 2033 log.critical() 2057 2034 printline(Widths,'-') 2058 2035 n = 0 … … 2060 2037 if s == Fields[0]: # Left justify 2061 2038 s = Dictnames[i] + ' ' + s; i=i+1 2062 exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1 2039 #exec "print '%-" + str(Widths[n]) + "s'%s,"; n=n+1 2040 log.critical('%-*s' % (Widths[n], s)) 2041 n += 1 2063 2042 else: 2064 exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1 2065 print 2043 #exec "print '%" + str(Widths[n]) + "s'%s,"; n=n+1 2044 log.critical('%*s' % (Widths[n], s)) 2045 n += 1 2046 log.critical() 2066 2047 printline(Widths,'-') 2067 2048 … … 2072 2053 n = 0 2073 2054 if len(key) > Widths[n]: key = key[:Widths[n]-3] + '...' 2074 exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1 2055 #exec "print '%-" + str(Widths[n]) + Types[n]+"'%key,";n=n+1 2056 log.critical('%-*s' % (Widths[n], str(key))) 2057 n += 1 2075 2058 for val in rec: 2076 exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1 2077 print 2078 print 2059 #exec "print '%" + str(Widths[n]) + Types[n]+"'%val,"; n=n+1 2060 log.critical('%*s' % (Widths[n], str(key))) 2061 n += 1 2062 log.critical() 2063 log.critical() 2079 2064 2080 2065 #============================================================================== … … 2121 2106 2122 2107 if sortidx > len(rec)-1: 2123 if options['verbose']: 2124 print 'ERROR: Sorting index to large, sortidx = ', sortidx 2125 raise IndexError 2108 msg = 'ERROR: Sorting index too large, sortidx = %s' % str(sortidx) 2109 raise IndexError, msg 2126 2110 2127 2111 val = rec[sortidx] … … 2149 2133 s = s+char 2150 2134 2151 print s2135 log.critical(s) 2152 2136 2153 2137 #============================================================================== … … 2163 2147 2164 2148 import string 2165 #print 'MESSAGE (caching.py): Evaluating function', funcname,2166 2149 2167 2150 print_header_box('Evaluating function %s' %funcname) … … 2172 2155 print_footer() 2173 2156 2174 #2175 # Old message2176 #2177 #args_present = 02178 #if args:2179 # if len(args) == 1:2180 # print 'with argument', mkargstr(args[0], textwidth2),2181 # else:2182 # print 'with arguments', mkargstr(args, textwidth2),2183 # args_present = 12184 #2185 #if kwargs:2186 # if args_present:2187 # word = 'and'2188 # else:2189 # word = 'with'2190 #2191 # if len(kwargs) == 1:2192 # print word + ' keyword argument', mkargstr(kwargs, textwidth2)2193 # else:2194 # print word + ' keyword arguments', mkargstr(kwargs, textwidth2)2195 # args_present = 12196 #else:2197 # print # Newline when no keyword args present2198 #2199 #if not args_present:2200 # print '', # Default if no args or kwargs present2201 2202 2203 2204 2157 # ----------------------------------------------------------------------------- 2205 2158 … … 2224 2177 msg8(reason) 2225 2178 2226 print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds' 2179 log.critical(string.ljust('| CPU time:', textwidth1) + 2180 str(round(comptime,2)) + ' seconds') 2227 2181 2228 2182 # ----------------------------------------------------------------------------- … … 2236 2190 2237 2191 import string 2238 print string.ljust('| Loading time:', textwidth1) + str(round(savetime,2)) + \2239 ' seconds (estimated)'2192 log.critical(string.ljust('| Loading time:', textwidth1) + 2193 str(round(savetime,2)) + ' seconds (estimated)') 2240 2194 msg5(CD,FN,deps,compression) 2241 2195 … … 2254 2208 2255 2209 msg6(funcname,args,kwargs) 2256 print string.ljust('| CPU time:', textwidth1) + str(round(comptime,2)) + ' seconds' 2257 print string.ljust('| Loading time:', textwidth1) + str(round(loadtime,2)) + ' seconds' 2258 print string.ljust('| Time saved:', textwidth1) + str(round(comptime-loadtime,2)) + \ 2259 ' seconds' 2210 log.critical(string.ljust('| CPU time:', textwidth1) + 2211 str(round(comptime,2)) + ' seconds') 2212 log.critical(string.ljust('| Loading time:', textwidth1) + 2213 str(round(loadtime,2)) + ' seconds') 2214 log.critical(string.ljust('| Time saved:', textwidth1) + 2215 str(round(comptime-loadtime,2)) + ' seconds') 2260 2216 msg5(CD,FN,deps,compression) 2261 2217 … … 2274 2230 import os, time, string 2275 2231 2276 print '|'2277 print string.ljust('| Caching dir: ', textwidth1) + CD2232 log.critical('|') 2233 log.critical(string.ljust('| Caching dir: ', textwidth1) + CD) 2278 2234 2279 2235 if compression: … … 2286 2242 for file_type in file_types: 2287 2243 file_name = FN + '_' + file_type + suffix 2288 print string.ljust('| ' + file_type + ' file: ', textwidth1) + file_name,2289 2244 stats = os.stat(CD+file_name) 2290 print '('+ str(stats[6]) + ' ' + bytetext + ')' 2291 2292 print '|' 2245 log.critical(string.ljust('| ' + file_type + ' file: ', textwidth1) + 2246 file_name + '('+ str(stats[6]) + ' ' + bytetext + ')') 2247 2248 log.critical('|') 2293 2249 if len(deps) > 0: 2294 print '| Dependencies: '2250 log.critical('| Dependencies: ') 2295 2251 dependencies = deps.keys() 2296 2252 dlist = []; maxd = 0 … … 2317 2273 s = string.rjust(slist[n], maxs) 2318 2274 2319 print '| ', d, t, ' ', s, 'bytes'2275 log.critical('| %s %s %s bytes' % (d, t, s)) 2320 2276 else: 2321 print '| No dependencies'2277 log.critical('| No dependencies') 2322 2278 print_footer() 2323 2279 … … 2332 2288 2333 2289 import string 2334 print string.ljust('| Function:', textwidth1) + funcname2290 log.critical(string.ljust('| Function:', textwidth1) + funcname) 2335 2291 2336 2292 msg7(args, kwargs) … … 2350 2306 if args: 2351 2307 if len(args) == 1: 2352 print string.ljust('| Argument:', textwidth1) + mkargstr(args[0], \2353 textwidth2)2308 log.critical(string.ljust('| Argument:', textwidth1) + 2309 mkargstr(args[0], textwidth2)) 2354 2310 else: 2355 print string.ljust('| Arguments:', textwidth1) + \2356 mkargstr(args, textwidth2)2311 log.critical(string.ljust('| Arguments:', textwidth1) + 2312 mkargstr(args, textwidth2)) 2357 2313 args_present = 1 2358 2314 2359 2315 if kwargs: 2360 2316 if len(kwargs) == 1: 2361 print string.ljust('| Keyword Arg:', textwidth1) + mkargstr(kwargs, \2362 textwidth2)2317 log.critical(string.ljust('| Keyword Arg:', textwidth1) + 2318 mkargstr(kwargs, textwidth2)) 2363 2319 else: 2364 print string.ljust('| Keyword Args:', textwidth1) + \2365 mkargstr(kwargs, textwidth2)2320 log.critical(string.ljust('| Keyword Args:', textwidth1) + 2321 mkargstr(kwargs, textwidth2)) 2366 2322 args_present = 1 2367 2323 2368 2324 if not args_present: 2369 print '| No arguments'# Default if no args or kwargs present2325 log.critical('| No arguments') # Default if no args or kwargs present 2370 2326 2371 2327 # ----------------------------------------------------------------------------- … … 2385 2341 R = 'Unknown' 2386 2342 2387 print string.ljust('| Reason:', textwidth1) + R2343 log.critical(string.ljust('| Reason:', textwidth1) + R) 2388 2344 2389 2345 # ----------------------------------------------------------------------------- … … 2406 2362 s = '+' + '-'*N + CR 2407 2363 2408 print s + '| ' + line + CR + s,2364 log.critical(s + '| ' + line + CR + s) 2409 2365 2410 2366 textwidth3 = N … … 2419 2375 s = '+' + '-'*N + CR 2420 2376 2421 print s2377 log.critical(s) 2422 2378 2423 2379 # ----------------------------------------------------------------------------- … … 2508 2464 import string 2509 2465 2510 print string.ljust(msg, textwidth4) + ' - OK'2466 log.critical(string.ljust(msg, textwidth4) + ' - OK' ) 2511 2467 2512 2468 #raise StandardError … … 2521 2477 """ 2522 2478 2523 print 'ERROR (caching.test): %s' %msg2524 print 'Please send this code example and output to '2525 print 'Ole.Nielsen@anu.edu.au'2526 print2527 print2479 log.critical('ERROR (caching.test): %s' % msg) 2480 log.critical('Please send this code example and output to ') 2481 log.critical('Ole.Nielsen@anu.edu.au') 2482 log.critical() 2483 log.critical() 2528 2484 2529 2485 raise StandardError
Note: See TracChangeset
for help on using the changeset viewer.