Changeset 8331
- Timestamp:
- Feb 10, 2012, 10:34:08 AM (13 years ago)
- Location:
- trunk/anuga_work/development/mem_time_tests
- Files:
-
- 22 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/anuga_work/development/mem_time_tests/hardware/cairns/main.py
r8328 r8331 19 19 home = os.getenv('INUNDATIONHOME') 20 20 scenariodir = add_directories(home, ["data","mem_time_test", "parallel","cairns"]) 21 file1 = 'ex1.csv' 22 file_path = os.path.join(scenariodir, file1) 23 24 firstex1 = open(file_path, 'wb') 25 spamWriter = csv.writer(firstex1) 26 spamWriter.writerow(['Number Of Processors' , 'Space Used MB' , 'Time Taken (s)']) 27 21 meta_path = os.path.join(scenariodir, 'metalog.csv') 22 final_path = os.path.join(scenariodir, 'final.csv') 28 23 29 24 # main loop that runs macpus - 1 times … … 31 26 32 27 for n in range(1,maxcpus,1): 33 z = time.time() # time it34 35 28 #the necessary ways to run this script in parallel on the different hosts we use 36 29 if (host == 'cyclone.agso.gov.au'): 37 30 subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_cyclone', '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py']) 38 if (host == 'tornado.agso.gov.au'):31 elif (host == 'tornado.agso.gov.au'): 39 32 subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_tornado', '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py']) 40 if (host == 'vayu1'): 41 subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py']) 42 if (host == 'rhe-compute1.ga.gov.au'): 33 elif (host == 'rhe-compute1.ga.gov.au'): 43 34 subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python2.6', 'runcairns.py']) 44 if (host == 'xe'):35 else: 45 36 subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py']) 46 47 y = time.time()# time it 48 spamWriter.writerow([n,'x' ,(y-z)]) # write recorded results 37 49 38 print 'DONE' 50 39 51 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #get the memory usage from the log files 40 #get the important data for the experiments from the anuga experiments 41 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 52 42 53 #close ex1.csv so we can read from it for a different csv reader object 54 firstex1.close() 55 56 #merge the metalog useful memory info and the ex1 recorded info into one csv file named final 57 ex1 = csv.reader(open(file_path,'rb')) 43 #open files to read from and write to 58 44 metalog = csv.reader(open(meta_path,'rb')) 59 45 final = csv.writer(open(final_path,'wb')) 60 46 47 #list stores the index of the values requird 48 indexlist = [] 61 49 62 for row in ex1: 63 d = metalog.next() 64 final.writerow([row[0],row[1],d[0],d[3],d[6],d[9],d[12]])#,d[15],d[18]]) 50 #read in the first row 51 firstrow = metalog.next() 52 53 #get the indices of the values we want, so that the data can be condensed 54 indexlist.append(firstrow.index("beforetime")) 55 indexlist.append(firstrow.index("aftertime")) 56 indexlist.append(firstrow.index("beforesimulationmemory")) 57 indexlist.append(firstrow.index("aftermeshmemory")) 58 indexlist.append(firstrow.index("afterinitialconditionsmemory")) 59 indexlist.append(firstrow.index("afterboundarymemory")) 60 indexlist.append(firstrow.index("aftersimulationmemory")) 61 indexlist.append(firstrow.index("variable1")) 62 63 #not all experiments have 3 variables, but some do 64 try: 65 indexlist.append(firstrow.index("variable2")) 66 except ValueError: 67 indexlist.append(firstrow.index("beforetime")) 68 try: 69 indexlist.append(firstrow.index("variable3")) 70 except ValueError: 71 indexlist.append(firstrow.index("aftertime")) 72 73 #write the header for the final csv 74 final.writerow(["TimeTaken(s)",firstrow[(indexlist[2])], 75 firstrow[(indexlist[3])],firstrow[(indexlist[4])], 76 firstrow[(indexlist[5])],firstrow[(indexlist[6])], 77 firstrow[(indexlist[7])],firstrow[(indexlist[8])], 78 firstrow[(indexlist[9])]]) 79 80 #write the data for each column in the final csv 81 for row in metalog: 82 83 #manipulate the beginning and end time to get the time taken 84 begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') 85 end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') 86 taken = time.mktime(end) - time.mktime(begin) 87 88 #write to file 89 final.writerow([str(taken),row[(indexlist[2])], 90 row[(indexlist[3])],row[(indexlist[4])], 91 row[(indexlist[5])],row[(indexlist[6])], 92 row[(indexlist[7])],row[(indexlist[8])], 93 row[(indexlist[9])]]) -
trunk/anuga_work/development/mem_time_tests/hardware/cairns/runcairns.py
r8326 r8331 34 34 # set up variables for the correct output directories 35 35 home = os.getenv('INUNDATIONHOME') 36 scenariodir = add_directories(home, ["data", "mem_time_test", "parallel", "cairns"]) 36 scenariodirV = add_directories(home, ["data","mem_time_test", "parallel", 37 "cairns", "parrallel-" + str(numprocs) +"-"+str(myid)]) 37 38 h = 'CAIRNS.msh' 38 39 file_pathh = os.path.join(scenariodir, h) 39 store ='store.txt'40 file_path_store = os.path.join(scenariodir, store)41 scenariodirV = add_directories(home, ["data","mem_time_test", "parallel",42 "cairns", "parrallel-" + str(numprocs) +"-"+str(myid)])43 40 log.log_filename = os.path.join(scenariodirV, "anuga.log") 44 41 log._setup = False 45 42 43 log.timingInfo(msg=('variable1,'+str(numprocs))) #write the variable to be measured to file 44 log.timingInfo(msg=('variable2,'+str(myid))) #write the variable to be measured to file 46 45 47 log.resource_usage_timing(prefix = 'BeforeSimulation')#get memory statistics at this point 46 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp()))) #get the time at the beginning of the simulation 47 48 log.resource_usage_timing(prefix = 'beforesimulation')#get memory statistics at this point 48 49 #------------------------------------------------------------------------------ 49 50 # Preparation of topographic data … … 81 82 domain = None 82 83 83 log.resource_usage_timing(prefix = ' AfterMesh')#get memory statistics at this point84 log.resource_usage_timing(prefix = 'aftermesh')#get memory statistics at this point 84 85 85 86 #parallel … … 186 187 187 188 log.resource_usage_timing(prefix='aftersimulation') #get memory statistics at this point 189 log.timingInfo(msg=('aftertime,'+str(log.TimeStamp()))) #get the time at the end of the simulation -
trunk/anuga_work/development/mem_time_tests/hardware/template/main.py
r8328 r8331 6 6 #------------------------------------------------------------------------------ 7 7 import anuga 8 import random9 8 import subprocess 10 9 import csv … … 20 19 home = os.getenv('INUNDATIONHOME') 21 20 scenariodir = add_directories(home, ["data","mem_time_test", "parallel","template"]) 22 file1 = 'ex1.csv' 23 file_path = os.path.join(scenariodir, file1) 24 25 firstex1 = open(file_path, 'wb') 26 spamWriter = csv.writer(firstex1) 27 spamWriter.writerow(['Number Of Processors' ,'Time Taken (s)']) 21 meta_path = os.path.join(scenariodir, 'metalog.csv') 22 final_path = os.path.join(scenariodir, 'final.csv') 28 23 29 24 … … 32 27 33 28 for n in range(1,maxcpus,1): 34 z = time.time()# time it 35 29 36 30 #the different ways each HOST requires the parallel script to be run 37 31 if (host == 'cyclone.agso.gov.au'): 38 32 subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_cyclone', '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py']) 39 if (host == 'tornado.agso.gov.au'):33 elif (host == 'tornado.agso.gov.au'): 40 34 subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_tornado', '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py']) 41 if (host == 'vayu1'): 42 subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py']) 43 if (host == 'rhe-compute1.ga.gov.au'): 35 elif (host == 'rhe-compute1.ga.gov.au'): 44 36 subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python2.6', 'runcairns.py']) 45 if (host == 'xe'):37 else: 46 38 subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py']) 47 39 48 40 49 y = time.time() #time it50 spamWriter.writerow([n,(y-z)]) #record it51 52 41 print 'Done' 53 42 54 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #get the memory usage from the log files 43 #get the important data for the experiments from the anuga experiments 44 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 55 45 56 #close ex1.csv so we can read from it for a different csv reader object 57 firstex1.close() 58 59 #merge the metalog useful memory info and the ex1 recorded info into one csv file named final 60 ex1 = csv.reader(open(file_path,'rb')) 46 #open files to read from and write to 61 47 metalog = csv.reader(open(meta_path,'rb')) 62 48 final = csv.writer(open(final_path,'wb')) 63 49 50 #list stores the index of the values requird 51 indexlist = [] 64 52 65 for row in ex1: 66 d = metalog.next() 67 final.writerow([row[0],row[1],d[0],d[3],d[6],d[9],d[12]])#,d[15],d[18]]) 53 #read in the first row 54 firstrow = metalog.next() 55 56 #get the indices of the values we want, so that the data can be condensed 57 indexlist.append(firstrow.index("beforetime")) 58 indexlist.append(firstrow.index("aftertime")) 59 indexlist.append(firstrow.index("beforesimulationmemory")) 60 indexlist.append(firstrow.index("aftermeshmemory")) 61 indexlist.append(firstrow.index("afterinitialconditionsmemory")) 62 indexlist.append(firstrow.index("afterboundarymemory")) 63 indexlist.append(firstrow.index("aftersimulationmemory")) 64 indexlist.append(firstrow.index("variable1")) 65 66 #not all experiments have 3 variables, but some do 67 try: 68 indexlist.append(firstrow.index("variable2")) 69 except ValueError: 70 indexlist.append(firstrow.index("beforetime")) 71 try: 72 indexlist.append(firstrow.index("variable3")) 73 except ValueError: 74 indexlist.append(firstrow.index("aftertime")) 75 76 #write the header for the final csv 77 final.writerow(["TimeTaken(s)",firstrow[(indexlist[2])], 78 firstrow[(indexlist[3])],firstrow[(indexlist[4])], 79 firstrow[(indexlist[5])],firstrow[(indexlist[6])], 80 firstrow[(indexlist[7])],firstrow[(indexlist[8])], 81 firstrow[(indexlist[9])]]) 82 83 #write the data for each column in the final csv 84 for row in metalog: 85 86 #manipulate the beginning and end time to get the time taken 87 begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') 88 end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') 89 taken = time.mktime(end) - time.mktime(begin) 90 91 #write to file 92 final.writerow([str(taken),row[(indexlist[2])], 93 row[(indexlist[3])],row[(indexlist[4])], 94 row[(indexlist[5])],row[(indexlist[6])], 95 row[(indexlist[7])],row[(indexlist[8])], 96 row[(indexlist[9])]]) -
trunk/anuga_work/development/mem_time_tests/hardware/template/runcairns.py
r8326 r8331 19 19 #set up variables for the correct I/O directories for data storage 20 20 home = os.getenv('INUNDATIONHOME') 21 scenariodir = add_directories(home, ["data", "mem_time_test", "parallel", "template"])22 21 scenariodirV = add_directories(home, ["data","mem_time_test", "parallel", 23 22 "template", "template-" + str(numprocs) +"-"+ str(myid)]) … … 27 26 log._setup = False 28 27 29 log.resource_usage_timing(prefix = 'BeforeSimulation')#get memory statistics here 28 log.timingInfo(msg=('variable1,'+str(numprocs))) #write the variable to be measured to file 29 log.timingInfo(msg=('variable2,'+str(myid))) #write the variable to be measured to file 30 31 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp()))) #get the time at the beginning of the simulation 32 33 log.resource_usage_timing(prefix = 'beforesimulation')#get memory statistics here 30 34 #------------------------------------------------------------------------------ 31 35 #Create the domain and mesh for the resource experiment on only one processor … … 47 51 domain.set_name('CAIRNS') # Name of sww file 48 52 domain.set_datadir(scenariodirV)# Store sww output here 49 log.resource_usage_timing(prefix = ' AfterMesh')#get memory statistics here53 log.resource_usage_timing(prefix = 'aftermesh')#get memory statistics here 50 54 51 55 #------------------------------------------------------------------------------ … … 80 84 print domain.timestepping_statistics() 81 85 82 log.resource_usage_timing(prefix='aftersimulation')#get memory statistics here 86 log.resource_usage_timing(prefix='aftersimulation')#get memory statistics here 87 log.timingInfo(msg=('aftertime,'+str(log.TimeStamp()))) #get the time at the end of the simulation 88 -
trunk/anuga_work/development/mem_time_tests/parameters/nothing/ex1.py
r8328 r8331 18 18 log._setup = False 19 19 20 log.resource_usage_timing(prefix = 'BeforeSimulation') #get memory usage here 20 log.timingInfo(msg=('variable1,'+str(n))) #write the variable to be measured to file 21 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp()))) #get the time at the beginning of the simulation 22 log.resource_usage_timing(prefix = 'beforesimulation') #get memory usage here 21 23 #------------------------------------------------------------------------------ 22 24 # Setup computational domain … … 28 30 domain.set_datadir(scenariodirV) 29 31 30 log.resource_usage_timing(prefix = ' AfterMesh') #get memory usage here32 log.resource_usage_timing(prefix = 'aftermesh') #get memory usage here 31 33 32 34 #------------------------------------------------------------------------------ … … 55 57 56 58 log.resource_usage_timing(prefix='aftersimulation') #get memory usage here 59 log.timingInfo(msg=('aftertime,'+str(log.TimeStamp()))) #get the time at the end of the simulation -
trunk/anuga_work/development/mem_time_tests/parameters/nothing/main.py
r8328 r8331 4 4 import anuga 5 5 import time 6 import random7 6 import subprocess 8 7 import csv … … 17 16 home = os.getenv('INUNDATIONHOME') 18 17 scenariodir = add_directories(home, ["data","mem_time_test", "parameters","nothing"]) 19 file = 'ex1.csv'20 file_path = os.path.join(scenariodir, file)21 18 meta = 'metalog.csv' 22 19 meta_path = os.path.join(scenariodir, meta) … … 24 21 final_path = os.path.join(scenariodir, final) 25 22 26 #set up needed files27 firstex1 = open(file_path, 'wb')28 spamWriter = csv.writer(firstex1)29 spamWriter.writerow(['Run Number' , 'Time Taken(s)','Space Used'])30 31 23 #main loop, that stores the current run number so that unique folders exist for the information 32 for n in range(0,300,1): 33 34 x = time.time() #time it 24 for n in range(0,100,1): 35 25 subprocess.call(['python2.5', 'ex1.py' ,str(n)]) #run script 36 y = time.time() #time it37 38 spamWriter.writerow([n ,(y-x)])39 40 26 print "DONE" 41 27 42 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #get the memory usage from the log files 28 #get the important data for the experiments from the anuga experiments 29 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 43 30 44 #close ex1.csv so we can read from it for a different csv reader object 45 firstex1.close() 46 47 #merge the metalog useful memory info and the ex1 recorded info into one csv file named final 48 ex1 = csv.reader(open(file_path,'rb')) 31 #open files to read from and write to 49 32 metalog = csv.reader(open(meta_path,'rb')) 50 33 final = csv.writer(open(final_path,'wb')) 51 34 35 #list stores the index of the values requird 36 indexlist = [] 52 37 53 for row in ex1: 54 d = metalog.next() 55 final.writerow([row[0],row[1],d[0],d[3],d[6],d[9],d[12]])#,d[15],d[18]]) 38 #read in the first row 39 firstrow = metalog.next() 40 41 #get the indices of the values we want, so that the data can be condensed 42 indexlist.append(firstrow.index("beforetime")) 43 indexlist.append(firstrow.index("aftertime")) 44 indexlist.append(firstrow.index("beforesimulationmemory")) 45 indexlist.append(firstrow.index("aftermeshmemory")) 46 indexlist.append(firstrow.index("afterinitialconditionsmemory")) 47 indexlist.append(firstrow.index("afterboundarymemory")) 48 indexlist.append(firstrow.index("aftersimulationmemory")) 49 indexlist.append(firstrow.index("variable1")) 50 51 #not all experiments have 3 variables, but some do 52 try: 53 indexlist.append(firstrow.index("variable2")) 54 except ValueError: 55 indexlist.append(firstrow.index("beforetime")) 56 try: 57 indexlist.append(firstrow.index("variable3")) 58 except ValueError: 59 indexlist.append(firstrow.index("aftertime")) 60 61 #write the header for the final csv 62 final.writerow(["TimeTaken(s)",firstrow[(indexlist[2])], 63 firstrow[(indexlist[3])],firstrow[(indexlist[4])], 64 firstrow[(indexlist[5])],firstrow[(indexlist[6])], 65 firstrow[(indexlist[7])],firstrow[(indexlist[8])], 66 firstrow[(indexlist[9])]]) 67 68 #write the data for each column in the final csv 69 for row in metalog: 70 71 #manipulate the beginning and end time to get the time taken 72 begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') 73 end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') 74 taken = time.mktime(end) - time.mktime(begin) 75 76 #write to file 77 final.writerow([str(taken),row[(indexlist[2])], 78 row[(indexlist[3])],row[(indexlist[4])], 79 row[(indexlist[5])],row[(indexlist[6])], 80 row[(indexlist[7])],row[(indexlist[8])], 81 row[(indexlist[9])]]) 82 -
trunk/anuga_work/development/mem_time_tests/parameters/timelen-over-timestep/ex1.py
r8328 r8331 18 18 log._setup = False 19 19 20 log.resource_usage_timing(prefix = 'BeforeSimulation') #get memory usage here 20 21 log.timingInfo(msg=('variable1,'+str(g))) #write the variable to be measured to file 22 log.timingInfo(msg=('variable2,'+str(h))) #write the variable to be measured to file 23 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp()))) #get the time at the beginning of the simulation 24 log.resource_usage_timing(prefix = 'beforesimulation') #get memory usage here 21 25 22 26 #------------------------------------------------------------------------------ … … 28 32 domain.set_datadir(scenariodirV) 29 33 30 log.resource_usage_timing(prefix = ' AfterMesh') #get memory usage here34 log.resource_usage_timing(prefix = 'aftermesh') #get memory usage here 31 35 32 36 #------------------------------------------------------------------------------ … … 34 38 #------------------------------------------------------------------------------ 35 39 def topography(x, y): 36 return -x/10 # linear bed slope40 return 0 # linear bed slope 37 41 38 42 domain.set_quantity('elevation', topography) # Use function for elevation 39 43 domain.set_quantity('friction', 0.01) # Constant friction 40 domain.set_quantity('stage', expression='elevation')44 domain.set_quantity('stage', 10) #constant depth 41 45 log.resource_usage_timing(prefix='afterinitialconditions') #get memory usage here 42 46 … … 44 48 # Setup boundary conditions 45 49 #------------------------------------------------------------------------------ 46 Bi = anuga.Dirichlet_boundary([0.4, 0, 0]) # Inflow47 50 Br = anuga.Reflective_boundary(domain) # Solid reflective wall 48 domain.set_boundary({'left': B i, 'right': Br, 'top': Br, 'bottom': Br})51 domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br}) 49 52 log.resource_usage_timing(prefix='afterboundary') #get memory usage here 50 53 #------------------------------------------------------------------------------ … … 55 58 56 59 log.resource_usage_timing(prefix='aftersimulation') #get memory usage here 60 log.timingInfo(msg=('aftertime,'+str(log.TimeStamp()))) #get the time at the end of the simulation -
trunk/anuga_work/development/mem_time_tests/parameters/timelen-over-timestep/main.py
r8328 r8331 5 5 import anuga 6 6 import time 7 import random8 7 import subprocess 9 8 import csv … … 19 18 scenariodir = add_directories(home, ["data","mem_time_test", "parameters", 20 19 "timelen-over-timestep"]) 21 file = 'ex1.csv'22 file_path = os.path.join(scenariodir, file)23 20 meta = 'metalog.csv' 24 21 meta_path = os.path.join(scenariodir, meta) … … 26 23 final_path = os.path.join(scenariodir, final) 27 24 28 #create each file 29 firstex1 = open(file_path, 'wb') 30 spamWriter = csv.writer(firstex1) 31 spamWriter.writerow(['Time Length(s)','Time Step (s)' , 'Time Taken','Space Used']) 25 # main loops that give the time length (m) and the time step (n) 26 for m in range(10,1000,200): 32 27 33 # main loops that give the time length (m) and the time step (n) 34 for m in range(1,1000,50): 35 36 for n in range(1,1000,5): 28 for n in range(10,1000,100): 37 29 n = n/100.0 #adjust it so its small enough to see the nature of the relationship 38 39 x = time.time()#time it40 30 subprocess.call(['python2.5', 'ex1.py',str(m),str(n)])#run it 41 y = time.time()#time it42 spamWriter.writerow([m,n ,(y-x),'b'])#write the results43 44 31 print 'Done' 45 32 46 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #get the memory usage from the log files 33 #get the important data for the experiments from the anuga experiments 34 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 47 35 48 #close ex1.csv so we can read from it for a different csv reader object 49 firstex1.close() 50 51 #merge the metalog useful memory info and the ex1 recorded info into one csv file named final 52 ex1 = csv.reader(open(file_path,'rb')) 36 #open files to read from and write to 53 37 metalog = csv.reader(open(meta_path,'rb')) 54 38 final = csv.writer(open(final_path,'wb')) 55 39 40 #list stores the index of the values requird 41 indexlist = [] 56 42 57 for row in ex1: 58 d = metalog.next() 59 final.writerow([row[0],row[1],d[0],d[3],d[6],d[9],d[12]])#,d[15],d[18]]) 43 #read in the first row 44 firstrow = metalog.next() 45 46 #get the indices of the values we want, so that the data can be condensed 47 indexlist.append(firstrow.index("beforetime")) 48 indexlist.append(firstrow.index("aftertime")) 49 indexlist.append(firstrow.index("beforesimulationmemory")) 50 indexlist.append(firstrow.index("aftermeshmemory")) 51 indexlist.append(firstrow.index("afterinitialconditionsmemory")) 52 indexlist.append(firstrow.index("afterboundarymemory")) 53 indexlist.append(firstrow.index("aftersimulationmemory")) 54 indexlist.append(firstrow.index("variable1")) 55 56 #not all experiments have 3 variables, but some do 57 try: 58 indexlist.append(firstrow.index("variable2")) 59 except ValueError: 60 indexlist.append(firstrow.index("beforetime")) 61 try: 62 indexlist.append(firstrow.index("variable3")) 63 except ValueError: 64 indexlist.append(firstrow.index("aftertime")) 65 66 #write the header for the final csv 67 final.writerow(["TimeTaken(s)",firstrow[(indexlist[2])], 68 firstrow[(indexlist[3])],firstrow[(indexlist[4])], 69 firstrow[(indexlist[5])],firstrow[(indexlist[6])], 70 firstrow[(indexlist[7])],firstrow[(indexlist[8])], 71 firstrow[(indexlist[9])]]) 72 73 #write the data for each column in the final csv 74 for row in metalog: 75 76 #manipulate the beginning and end time to get the time taken 77 begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') 78 end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') 79 taken = time.mktime(end) - time.mktime(begin) 80 81 #write to file 82 final.writerow([str(taken),row[(indexlist[2])], 83 row[(indexlist[3])],row[(indexlist[4])], 84 row[(indexlist[5])],row[(indexlist[6])], 85 row[(indexlist[7])],row[(indexlist[8])], 86 row[(indexlist[9])]]) 87 88 -
trunk/anuga_work/development/mem_time_tests/parameters/timelen/ex1.py
r8328 r8331 11 11 #set up variables to store the results and the log files of each experiment 12 12 f = sys.argv[1] 13 g = float(f)/100.0 13 14 home = os.getenv('INUNDATIONHOME') 14 15 scenariodirV = add_directories(home, ["data","mem_time_test", "parameters", … … 17 18 log._setup = False 18 19 19 log.resource_usage_timing(prefix = 'BeforeSimulation') #get memory usage here 20 log.timingInfo(msg=('variable1,'+str(f))) #write the variable to be measured to file 21 22 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp()))) #get the time at the beginning of the simulation 23 24 log.resource_usage_timing(prefix = 'beforesimulation') #get memory usage here 20 25 21 26 #------------------------------------------------------------------------------ … … 27 32 domain.set_datadir(scenariodirV) 28 33 29 log.resource_usage_timing(prefix = ' AfterMesh') #get memory usage here34 log.resource_usage_timing(prefix = 'aftermesh') #get memory usage here 30 35 #------------------------------------------------------------------------------ 31 36 # Setup initial conditions 32 37 #------------------------------------------------------------------------------ 33 38 def topography(x, y): 34 return -x/10 # linear bed slope39 return 0 # linear bed slope 35 40 domain.set_quantity('elevation', topography) # Use function for elevation 36 41 domain.set_quantity('friction', 0.01) # Constant friction 37 domain.set_quantity('stage', expression='elevation')42 domain.set_quantity('stage', 10) 38 43 39 44 log.resource_usage_timing(prefix='afterinitialconditions')#get memory usage here … … 42 47 # Setup boundary conditions 43 48 #------------------------------------------------------------------------------ 44 Bi = anuga.Dirichlet_boundary([0.4, 0, 0]) # Inflow45 49 Br = anuga.Reflective_boundary(domain) # Solid reflective wall 46 domain.set_boundary({'left': B i, 'right': Br, 'top': Br, 'bottom': Br})50 domain.set_boundary({'left': Br, 'right': Br, 'top': Br, 'bottom': Br}) 47 51 48 52 log.resource_usage_timing(prefix='afterboundary')#get memory usage here … … 51 55 # Evolve system through time 52 56 #------------------------------------------------------------------------------ 53 for t in domain.evolve(yieldstep= 0.2, finaltime=f):57 for t in domain.evolve(yieldstep=g, finaltime=f): 54 58 print domain.timestepping_statistics() 55 59 56 60 57 61 log.resource_usage_timing(prefix='aftersimulation') #get memory usage here 58 62 log.timingInfo(msg=('aftertime,'+str(log.TimeStamp()))) #get the time at the end of the simulation -
trunk/anuga_work/development/mem_time_tests/parameters/timelen/main.py
r8328 r8331 18 18 scenariodir = add_directories(home, ["data","mem_time_test", "parameters", 19 19 "timelength"]) 20 file = 'ex1.csv'21 file_path = os.path.join(scenariodir, file)22 20 meta = 'metalog.csv' 23 21 meta_path = os.path.join(scenariodir, meta) … … 25 23 final_path = os.path.join(scenariodir, final) 26 24 27 #set up needed files28 firstex1 = open(file_path, 'wb')29 spamWriter = csv.writer(firstex1)30 spamWriter.writerow(['Time Length (s)' , 'Time Taken (s)','Space Used'])31 32 25 #main loop that does the same experiment with different time lengths 33 for n in range(1,1000,100): 34 35 x = time.time() #time it 26 for n in range(1,1000,20): 36 27 subprocess.call(['python2.5', 'ex1.py', str(n)]) #run script 37 y = time.time() #time it38 spamWriter.writerow([n ,(y-x)]) #write results39 40 28 print 'Done' 41 29 42 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #get the memory usage from the log files 30 #get the important data for the experiments from the anuga experiments 31 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 43 32 44 #close ex1.csv so we can read from it for a different csv reader object 45 firstex1.close() 46 47 #merge the metalog useful memory info and the ex1 recorded info into one csv file named final 48 ex1 = csv.reader(open(file_path,'rb')) 33 #open files to read from and write to 49 34 metalog = csv.reader(open(meta_path,'rb')) 50 35 final = csv.writer(open(final_path,'wb')) 51 36 37 #list stores the index of the values requird 38 indexlist = [] 52 39 53 for row in ex1: 54 d = metalog.next() 55 final.writerow([row[0],row[1],d[0],d[3],d[6],d[9],d[12]])#,d[15],d[18]]) 40 #read in the first row 41 firstrow = metalog.next() 42 43 #get the indices of the values we want, so that the data can be condensed 44 indexlist.append(firstrow.index("beforetime")) 45 indexlist.append(firstrow.index("aftertime")) 46 indexlist.append(firstrow.index("beforesimulationmemory")) 47 indexlist.append(firstrow.index("aftermeshmemory")) 48 indexlist.append(firstrow.index("afterinitialconditionsmemory")) 49 indexlist.append(firstrow.index("afterboundarymemory")) 50 indexlist.append(firstrow.index("aftersimulationmemory")) 51 indexlist.append(firstrow.index("variable1")) 52 53 #not all experiments have 3 variables, but some do 54 try: 55 indexlist.append(firstrow.index("variable2")) 56 except ValueError: 57 indexlist.append(firstrow.index("beforetime")) 58 try: 59 indexlist.append(firstrow.index("variable3")) 60 except ValueError: 61 indexlist.append(firstrow.index("aftertime")) 62 63 #write the header for the final csv 64 final.writerow(["TimeTaken(s)",firstrow[(indexlist[2])], 65 firstrow[(indexlist[3])],firstrow[(indexlist[4])], 66 firstrow[(indexlist[5])],firstrow[(indexlist[6])], 67 firstrow[(indexlist[7])],firstrow[(indexlist[8])], 68 firstrow[(indexlist[9])]]) 69 70 #write the data for each column in the final csv 71 for row in metalog: 72 73 #manipulate the beginning and end time to get the time taken 74 begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') 75 end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') 76 taken = time.mktime(end) - time.mktime(begin) 77 78 #write to file 79 final.writerow([str(taken),row[(indexlist[2])], 80 row[(indexlist[3])],row[(indexlist[4])], 81 row[(indexlist[5])],row[(indexlist[6])], 82 row[(indexlist[7])],row[(indexlist[8])], 83 row[(indexlist[9])]]) 84 -
trunk/anuga_work/development/mem_time_tests/scenarios/channelflow/ex1.py
r8328 r8331 19 19 log._setup = False 20 20 21 log.resource_usage_timing(prefix = 'BeforeSimulation')#get memory usage 21 log.timingInfo(msg=('variable1,'+str(length))) #write the variable to be measured to file 22 log.timingInfo(msg=('variable2,'+str(lower))) #write the variable to be measured to file 23 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp()))) #get the time at the beginning of the simulation 24 25 log.resource_usage_timing(prefix = 'beforesimulation')#get memory usage 22 26 #------------------------------------------------------------------------------ 23 27 # Setup computational domain 24 28 #------------------------------------------------------------------------------ 25 points, vertices, boundary = anuga.rectangular_cross( 50,150,len1=length, len2=length) # Mesh29 points, vertices, boundary = anuga.rectangular_cross(100,300,len1=length, len2=length) # Mesh 26 30 domain = anuga.Domain(points, vertices, boundary) # Create domain 27 31 domain.set_datadir(scenariodirV) 28 32 domain.set_name('channel1.sww') # Output name 29 33 30 log.resource_usage_timing(prefix = ' AfterMesh') #get memory usage34 log.resource_usage_timing(prefix = 'aftermesh') #get memory usage 31 35 #-------------------------------------------------------------------------- 32 36 # Setup Initial Conditions 33 37 #-------------------------------------------------------------------------- 34 38 def topography(x, y): 35 z = -x/1039 z = 0 36 40 N = len(x) 37 41 … … 52 56 # Setup boundary conditions 53 57 #------------------------------------------------------------------------------ 54 Bi = anuga.Dirichlet_boundary([0.5, 1 , 0]) # Inflow55 Bo = anuga.Dirichlet_boundary([- 100,0,0])58 Bi = anuga.Dirichlet_boundary([0.5, 10, 0]) # Inflow 59 Bo = anuga.Dirichlet_boundary([-0.5,0,0]) #outflow 56 60 Br = anuga.Reflective_boundary(domain) # Solid reflective wall 57 61 domain.set_boundary({'left': Bi, 'right': Bo, 'top': Br, 'bottom': Br}) … … 61 65 # Evolve system through time 62 66 #------------------------------------------------------------------------------ 63 for t in domain.evolve(yieldstep= 0.2, finaltime=40.0):67 for t in domain.evolve(yieldstep=120, finaltime=3600): 64 68 domain.write_time() 65 69 66 70 log.resource_usage_timing(prefix='aftersimulation') #get memory usage 71 log.timingInfo(msg=('aftertime,'+str(log.TimeStamp()))) #get the time at the end of the simulation -
trunk/anuga_work/development/mem_time_tests/scenarios/channelflow/main.py
r8328 r8331 3 3 #------------------------------------------------------------------------------ 4 4 import anuga 5 import random6 5 import subprocess 7 6 import csv … … 17 16 scenariodir = add_directories(home, ["data","mem_time_test", "scenarios", 18 17 "channelflow"]) 19 file = 'ex1.csv'20 file_path = os.path.join(scenariodir, file)21 18 meta = 'metalog.csv' 22 19 meta_path = os.path.join(scenariodir, meta) … … 24 21 final_path = os.path.join(scenariodir, final) 25 22 26 #set up needed files27 firstex1 = open(file_path, 'wb')28 spamWriter = csv.writer(firstex1)29 spamWriter.writerow(['Side Lenght(m)','Percentage Of Water Coverage' , 'Time Taken(s)','Space Used'])30 31 23 #the main loops that give the length of the inflow boundary (i) and the place to begin the step (n) 32 for i in range(1,1000,100): 33 for n in range(0,110,10): 34 24 for i in range(900,1000,100): 25 for n in range(0,110,5): 35 26 #adjust 36 27 n = n*(i/100.0) 37 38 z = time.time() #time it39 28 subprocess.call(['python2.5', 'ex1.py',str(i),str(n)])#run the script 40 y = time.time() #time it41 42 spamWriter.writerow([i,n/(i/100.0),(y-z), 'b'])#record it43 29 print 'DONE' 44 30 45 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #get the memory usage from the log files 31 #get the important data for the experiments from the anuga experiments 32 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 46 33 47 #close ex1.csv so we can read from it for a different csv reader object 48 firstex1.close() 49 50 #merge the metalog useful memory info and the ex1 recorded info into one csv file named final 51 ex1 = csv.reader(open(file_path,'rb')) 34 #open files to read from and write to 52 35 metalog = csv.reader(open(meta_path,'rb')) 53 36 final = csv.writer(open(final_path,'wb')) 54 37 38 #list stores the index of the values requird 39 indexlist = [] 55 40 56 for row in ex1: 57 d = metalog.next() 58 final.writerow([row[0],row[1],d[0],d[3],d[6],d[9],d[12]])#,d[15],d[18]]) 41 #read in the first row 42 firstrow = metalog.next() 43 44 #get the indices of the values we want, so that the data can be condensed 45 indexlist.append(firstrow.index("beforetime")) 46 indexlist.append(firstrow.index("aftertime")) 47 indexlist.append(firstrow.index("beforesimulationmemory")) 48 indexlist.append(firstrow.index("aftermeshmemory")) 49 indexlist.append(firstrow.index("afterinitialconditionsmemory")) 50 indexlist.append(firstrow.index("afterboundarymemory")) 51 indexlist.append(firstrow.index("aftersimulationmemory")) 52 indexlist.append(firstrow.index("variable1")) 53 54 #not all experiments have 3 variables, but some do 55 try: 56 indexlist.append(firstrow.index("variable2")) 57 except ValueError: 58 indexlist.append(firstrow.index("beforetime")) 59 try: 60 indexlist.append(firstrow.index("variable3")) 61 except ValueError: 62 indexlist.append(firstrow.index("aftertime")) 63 64 #write the header for the final csv 65 final.writerow(["TimeTaken(s)",firstrow[(indexlist[2])], 66 firstrow[(indexlist[3])],firstrow[(indexlist[4])], 67 firstrow[(indexlist[5])],firstrow[(indexlist[6])], 68 firstrow[(indexlist[7])],firstrow[(indexlist[8])], 69 firstrow[(indexlist[9])]]) 70 71 #write the data for each column in the final csv 72 for row in metalog: 73 74 #manipulate the beginning and end time to get the time taken 75 begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') 76 end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') 77 taken = time.mktime(end) - time.mktime(begin) 78 79 #write to file 80 final.writerow([str(taken),row[(indexlist[2])], 81 row[(indexlist[3])],row[(indexlist[4])], 82 row[(indexlist[5])],row[(indexlist[6])], 83 row[(indexlist[7])],row[(indexlist[8])], 84 row[(indexlist[9])]]) 85 -
trunk/anuga_work/development/mem_time_tests/scenarios/stage/ex1.py
r8328 r8331 19 19 log._setup = False 20 20 21 log.resource_usage_timing(prefix = 'BeforeSimulation')# get memory usage 21 log.timingInfo(msg=('variable1,'+str(l))) #get the variable to be measured and write it to the log file 22 log.timingInfo(msg=('variable2,'+str(tide))) #get the variable to be measured and write it to the log file 23 24 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp())))#write the begin time to the log file 25 26 log.resource_usage_timing(prefix = 'beforesimulation')# get memory usage 22 27 #------------------------------------------------------------------------------ 23 28 # Setup computational domain … … 29 34 domain.set_name('channel1') # Output name 30 35 31 log.resource_usage_timing(prefix = ' AfterMesh')# get memory usage36 log.resource_usage_timing(prefix = 'aftermesh')# get memory usage 32 37 33 38 #-------------------------------------------------------------------------- … … 57 62 58 63 log.resource_usage_timing(prefix='aftersimulation')# get memory usage 64 log.timingInfo(msg=('aftertime,'+str(log.TimeStamp()))) #write the end time to log file -
trunk/anuga_work/development/mem_time_tests/scenarios/stage/main.py
r8328 r8331 3 3 #------------------------------------------------------------------------------ 4 4 import anuga 5 import random6 5 import subprocess 7 6 import csv … … 17 16 scenariodir = add_directories(home, ["data","mem_time_test", "scenarios", 18 17 "stage"]) 19 file = 'ex1.csv'20 file_path = os.path.join(scenariodir, file)21 18 meta = 'metalog.csv' 22 19 meta_path = os.path.join(scenariodir, meta) … … 24 21 final_path = os.path.join(scenariodir, final) 25 22 26 #create files27 firstex1 = open(file_path, 'wb')28 spamWriter = csv.writer(firstex1)29 spamWriter.writerow(['Depth(m)','Extent(m^2)' , 'Time Taken(s)','Space Used'])30 31 23 #these are the main loops that determine the side length of the bounding square(m) 32 24 #and the depth of the initial water (n) 33 for m in range(100,1000,100): 34 for n in range(1,100,10): 35 36 z = time.time() #time it 25 for m in range(900,1000,100): 26 for n in range(90,100,10): 37 27 subprocess.call(['python2.5', 'ex1.py',str(m),str(n)]) # run the simulation script 38 y = time.time() #time it39 spamWriter.writerow([n ,((m**2)/2),(y-z),'a']) #record it40 28 print 'DONE' 41 29 42 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #get the memory usage from the log files 30 #get the important data for the experiments from the anuga experiments 31 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 43 32 44 #close ex1.csv so we can read from it for a different csv reader object 45 firstex1.close() 46 47 #merge the metalog useful memory info and the ex1 recorded info into one csv file named final 48 ex1 = csv.reader(open(file_path,'rb')) 33 #open files to read from and write to 49 34 metalog = csv.reader(open(meta_path,'rb')) 50 35 final = csv.writer(open(final_path,'wb')) 51 36 37 #list stores the index of the values required 38 indexlist = [] 52 39 53 for row in ex1: 54 d = metalog.next() 55 final.writerow([row[0],row[1],d[0],d[3],d[6],d[9],d[12]])#,d[15],d[18]]) 40 #read in the first row 41 firstrow = metalog.next() 42 43 #get the indices of the values we want, so that the data can be condensed 44 indexlist.append(firstrow.index("beforetime")) 45 indexlist.append(firstrow.index("aftertime")) 46 indexlist.append(firstrow.index("beforesimulationmemory")) 47 indexlist.append(firstrow.index("aftermeshmemory")) 48 indexlist.append(firstrow.index("afterinitialconditionsmemory")) 49 indexlist.append(firstrow.index("afterboundarymemory")) 50 indexlist.append(firstrow.index("aftersimulationmemory")) 51 indexlist.append(firstrow.index("variable1")) 52 53 #not all experiments have 3 variables, but some do 54 try: 55 indexlist.append(firstrow.index("variable2")) 56 except ValueError: 57 indexlist.append(firstrow.index("beforetime")) 58 try: 59 indexlist.append(firstrow.index("variable3")) 60 except ValueError: 61 indexlist.append(firstrow.index("aftertime")) 62 63 #write the header for the final csv 64 final.writerow(["TimeTaken(s)",firstrow[(indexlist[2])], 65 firstrow[(indexlist[3])],firstrow[(indexlist[4])], 66 firstrow[(indexlist[5])],firstrow[(indexlist[6])], 67 firstrow[(indexlist[7])],firstrow[(indexlist[8])], 68 firstrow[(indexlist[9])]]) 69 70 #write the data for each column in the final csv 71 for row in metalog: 72 73 #manipulate the beginning and end time to get the time taken 74 begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') 75 end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') 76 taken = time.mktime(end) - time.mktime(begin) 77 78 #write to file 79 final.writerow([str(taken),row[(indexlist[2])], 80 row[(indexlist[3])],row[(indexlist[4])], 81 row[(indexlist[5])],row[(indexlist[6])], 82 row[(indexlist[7])],row[(indexlist[8])], 83 row[(indexlist[9])]]) -
trunk/anuga_work/development/mem_time_tests/scenarios/vel2/ex1.py
r8328 r8331 14 14 15 15 home = os.getenv('INUNDATIONHOME') 16 scenariodir = add_directories(home, ["data","mem_time_test", "scenarios",17 "velocity"])18 16 scenariodirV = add_directories(home, ["data","mem_time_test", "scenarios", 19 17 "velocity", "velocity-" + str(k) +"-"+ str(l)]) 20 18 21 store = 'store.txt' 22 store_path = os.path.join(scenariodir, store) 19 log.timingInfo(msg=('variable1,'+str(k))) #write the variable to be measured to file 20 log.timingInfo(msg=('variable2,'+str(l))) #write the variable to be measured to file 21 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp()))) #get the time at the beginning of the simulation 23 22 24 23 log.log_filename = os.path.join(scenariodirV, "anuga.log") 25 24 log._setup = False 26 25 27 log.resource_usage_timing(prefix = ' BeforeSimulation') #get memory usage26 log.resource_usage_timing(prefix = 'beforesimulation') #get memory usage 28 27 #------------------------------------------------------------------------------ 29 28 # Setup computational domain … … 34 33 domain.set_datadir(scenariodirV) 35 34 36 log.resource_usage_timing(prefix = ' AfterMesh') #get memory usage35 log.resource_usage_timing(prefix = 'aftermesh') #get memory usage 37 36 38 37 #get the number of triangles 39 38 number = len(domain) 39 log.timingInfo(msg=('variable3,'+str(number))) #write the variable to be measured to file 40 40 41 41 #-------------------------------------------------------------------------- … … 67 67 68 68 log.resource_usage_timing(prefix='aftersimulation') #get memory usage 69 70 #write the number of triangles to the file 71 c = open(store_path,'r+') 72 c.write(str(number)) 69 log.timingInfo(msg=('aftertime,'+str(log.TimeStamp()))) #get the time at the end of the simulation -
trunk/anuga_work/development/mem_time_tests/scenarios/vel2/main.py
r8328 r8331 16 16 scenariodir = add_directories(home, ["data","mem_time_test", "scenarios", 17 17 "velocity"]) 18 file = 'ex1.csv'19 file_path = os.path.join(scenariodir, file)20 18 meta = 'metalog.csv' 21 19 meta_path = os.path.join(scenariodir, meta) 22 20 final = 'final.csv' 23 21 final_path = os.path.join(scenariodir, final) 24 store = 'store.txt'25 store_path = os.path.join(scenariodir, store)26 27 #set up needed files28 firstex1 = open(file_path, 'wb')29 spamWriter = csv.writer(firstex1)30 spamWriter.writerow(['Velocity(m/s)','Number Of Triangles','Extent(m^2)' , 'Time Taken(s)','Space Used'])31 storeopen = open(store_path,'a')32 storeopen.close33 22 34 23 # these are the main loops that give the velocity of inflow (m) and the map side length(n) 35 24 for m in range(0,250,10): 36 25 for n in range(1,2000,300): 37 38 z = time.time() #time it 39 subprocess.call(['python2.5', 'ex1.py', str(m), str(n)])#run simulation script 40 y = time.time() #time it 41 42 # read the number of triangles from this text file 43 f = open(store_path,'r+') 44 h = float(f.readline()) 45 f.close() 46 47 spamWriter.writerow([m,h,(n*n),(y-z), 'b'])# record it 26 subprocess.call(['python2.5', 'ex1.py', str(m), str(n)])#run simulation script 48 27 print 'DONE' 49 28 50 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #get the memory usage from the log files 29 #get the important data for the experiments from the anuga experiments 30 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 51 31 52 #close ex1.csv so we can read from it for a different csv reader object 53 firstex1.close() 54 55 #merge the metalog useful memory info and the ex1 recorded info into one csv file named final 56 ex1 = csv.reader(open(file_path,'rb')) 32 #open files to read from and write to 57 33 metalog = csv.reader(open(meta_path,'rb')) 58 34 final = csv.writer(open(final_path,'wb')) 59 35 36 #list stores the index of the values requird 37 indexlist = [] 60 38 61 for row in ex1: 62 d = metalog.next() 63 final.writerow([row[0],row[1],d[0],d[3],d[6],d[9],d[12]])#,d[15],d[18]]) 39 #read in the first row 40 firstrow = metalog.next() 41 42 #get the indices of the values we want, so that the data can be condensed 43 indexlist.append(firstrow.index("beforetime")) 44 indexlist.append(firstrow.index("aftertime")) 45 indexlist.append(firstrow.index("beforesimulationmemory")) 46 indexlist.append(firstrow.index("aftermeshmemory")) 47 indexlist.append(firstrow.index("afterinitialconditionsmemory")) 48 indexlist.append(firstrow.index("afterboundarymemory")) 49 indexlist.append(firstrow.index("aftersimulationmemory")) 50 indexlist.append(firstrow.index("variable1")) 51 52 #not all experiments have 3 variables, but some do 53 try: 54 indexlist.append(firstrow.index("variable2")) 55 except ValueError: 56 indexlist.append(firstrow.index("beforetime")) 57 try: 58 indexlist.append(firstrow.index("variable3")) 59 except ValueError: 60 indexlist.append(firstrow.index("aftertime")) 61 62 #write the header for the final csv 63 final.writerow(["TimeTaken(s)",firstrow[(indexlist[2])], 64 firstrow[(indexlist[3])],firstrow[(indexlist[4])], 65 firstrow[(indexlist[5])],firstrow[(indexlist[6])], 66 firstrow[(indexlist[7])],firstrow[(indexlist[8])], 67 firstrow[(indexlist[9])]]) 68 69 #write the data for each column in the final csv 70 for row in metalog: 71 72 #manipulate the beginning and end time to get the time taken 73 begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') 74 end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') 75 taken = time.mktime(end) - time.mktime(begin) 76 77 #write to file 78 final.writerow([str(taken),row[(indexlist[2])], 79 row[(indexlist[3])],row[(indexlist[4])], 80 row[(indexlist[5])],row[(indexlist[6])], 81 row[(indexlist[7])],row[(indexlist[8])], 82 row[(indexlist[9])]]) 83 84 -
trunk/anuga_work/development/mem_time_tests/triangles/area/main.py
r8328 r8331 17 17 host = os.getenv('HOST') 18 18 scenariodir = add_directories(home, ["data","mem_time_test", "triangles","area"]) 19 file1 = 'ex1.csv'20 19 meta = 'metalog.csv' 21 20 meta_path = os.path.join(scenariodir, meta) 22 21 final = 'final.csv' 23 22 final_path = os.path.join(scenariodir, final) 24 storen ='storen.txt'25 file_path_storen = os.path.join(scenariodir, storen)26 27 #set up needed files28 firstex1 = open(file_path, 'wb')29 spamWriter = csv.writer(firstex1)30 spamWriter.writerow(['Number of Triangles','Max Area(m^2)','Extent(m^2)' , 'Time Taken(s)','Space Used'])31 e = open(file_path_storen,'a')32 e.close()33 23 34 24 # this is the main loops that assigns the maximum triangle area (m) and the map side length(l) … … 37 27 for m in range(90,100,10): 38 28 for l in range(100,2000,100): 39 40 z = time.time()# time it41 42 29 #the different ways each host calls MPI properly 43 30 if (host == 'cyclone.agso.gov.au'): 44 31 subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_cyclone', '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py',str(m),str(l)]) 45 if (host == 'tornado.agso.gov.au'):32 elif (host == 'tornado.agso.gov.au'): 46 33 subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_tornado', '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py',str(m),str(l)]) 47 if (host == 'vayu1'): 48 subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py',str(m),str(l)]) 49 if (host == 'rhe-compute1.ga.gov.au'): 34 elif (host == 'rhe-compute1.ga.gov.au'): 50 35 subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python2.6', 'runcairns.py',str(m),str(l)]) 51 if (host == 'xe'):36 else: 52 37 subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py',str(m),str(l)]) 53 38 54 y = time.time()# time it55 56 # read the number of triangles from this file57 f = open(file_path_storen,'r+')58 i = float(f.readline())59 f.close()60 61 spamWriter.writerow([i,m,(l*l),'x' ,(y-z)])# record it62 63 39 print 'Done' 64 40 65 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #get the memory usage from the log files 41 #get the important data for the experiments from the anuga experiments 42 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 66 43 67 #close ex1.csv so we can read from it for a different csv reader object 68 firstex1.close() 69 70 #merge the metalog useful memory info and the ex1 recorded info into one csv file named final 71 ex1 = csv.reader(open(file_path,'rb')) 44 #open files to read from and write to 72 45 metalog = csv.reader(open(meta_path,'rb')) 73 46 final = csv.writer(open(final_path,'wb')) 74 47 48 #list stores the index of the values requird 49 indexlist = [] 75 50 76 for row in ex1: 77 d = metalog.next() 78 final.writerow([row[0],row[1],d[0],d[3],d[6],d[9],d[12]])#,d[15],d[18]]) 51 #read in the first row 52 firstrow = metalog.next() 79 53 54 #get the indices of the values we want, so that the data can be condensed 55 indexlist.append(firstrow.index("beforetime")) 56 indexlist.append(firstrow.index("aftertime")) 57 indexlist.append(firstrow.index("beforesimulationmemory")) 58 indexlist.append(firstrow.index("aftermeshmemory")) 59 indexlist.append(firstrow.index("afterinitialconditionsmemory")) 60 indexlist.append(firstrow.index("afterboundarymemory")) 61 indexlist.append(firstrow.index("aftersimulationmemory")) 62 indexlist.append(firstrow.index("variable1")) 63 64 #not all experiments have 3 variables, but some do 65 try: 66 indexlist.append(firstrow.index("variable2")) 67 except ValueError: 68 indexlist.append(firstrow.index("beforetime")) 69 try: 70 indexlist.append(firstrow.index("variable3")) 71 except ValueError: 72 indexlist.append(firstrow.index("aftertime")) 73 74 #write the header for the final csv 75 final.writerow(["TimeTaken(s)",firstrow[(indexlist[2])], 76 firstrow[(indexlist[3])],firstrow[(indexlist[4])], 77 firstrow[(indexlist[5])],firstrow[(indexlist[6])], 78 firstrow[(indexlist[7])],firstrow[(indexlist[8])], 79 firstrow[(indexlist[9])]]) 80 81 #write the data for each column in the final csv 82 for row in metalog: 83 84 #manipulate the beginning and end time to get the time taken 85 begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') 86 end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') 87 taken = time.mktime(end) - time.mktime(begin) 88 89 #write to file 90 final.writerow([str(taken),row[(indexlist[2])], 91 row[(indexlist[3])],row[(indexlist[4])], 92 row[(indexlist[5])],row[(indexlist[6])], 93 row[(indexlist[7])],row[(indexlist[8])], 94 row[(indexlist[9])]]) -
trunk/anuga_work/development/mem_time_tests/triangles/area/runcairns.py
r8328 r8331 11 11 from anuga.utilities import log 12 12 13 #set up the variables for the temporary data files14 home = os.getenv('INUNDATIONHOME')15 scenariodir = add_directories(home, ["data", "mem_time_test", "triangles", "area"])16 storen ='storen.txt'17 file_path_storen = os.path.join(scenariodir, storen)18 19 13 #set up the variables for the output data and the log files 20 14 length = sys.argv[2] … … 28 22 log._setup = False 29 23 30 log.resource_usage_timing(prefix = 'BeforeSimulation') #get memory usage 24 log.timingInfo(msg=('variable1,'+str(length))) #write the variable to be measured to file 25 log.timingInfo(msg=('variable2,'+str(area))) #write the variable to be measured to file 26 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp()))) #get the time at the beginning of the simulation 27 28 log.resource_usage_timing(prefix = 'beforesimulation') #get memory usage 31 29 #------------------------------------------------------------------------------ 32 30 # Create the triangular mesh and domain on one processor … … 46 44 else: 47 45 domain = None 48 46 log.timingInfo(msg=('variable3,'+str(n))) #write the variable to be measured to file 47 49 48 #parallel 50 49 domain = distribute(domain) … … 53 52 domain.set_datadir(scenariodirV)# Store sww output here 54 53 55 log.resource_usage_timing(prefix = ' AfterMesh') #get memory usage54 log.resource_usage_timing(prefix = 'aftermesh') #get memory usage 56 55 #------------------------------------------------------------------------------ 57 56 # Setup initial conditions … … 86 85 87 86 log.resource_usage_timing(prefix='aftersimulation') #get memory usage 87 log.timingInfo(msg=('aftertime,'+str(log.TimeStamp()))) #get the time at the end of the simulation 88 88 89 #write the number of triangles to file90 i = open(file_path_storen, 'r+')91 i.write(str(n))92 93 -
trunk/anuga_work/development/mem_time_tests/triangles/fromregions/main.py
r8328 r8331 16 16 scenariodir = add_directories(home, ["data","mem_time_test", "triangles", 17 17 "fromregions"]) 18 storea ='storea.txt'19 file_path_storea = os.path.join(scenariodir, storea)20 18 meta = 'metalog.csv' 21 19 meta_path = os.path.join(scenariodir, meta) 22 20 final = 'final.csv' 23 21 final_path = os.path.join(scenariodir, final) 24 file = 'ex1.csv'25 file_path = os.path.join(scenariodir, file)26 27 #create and set up the files28 firstex1 = open(file_path, 'wb')29 spamWriter = csv.writer(firstex1)30 spamWriter.writerow(['Number Of Triangles','Maximum Area(m^2)','Extent(m^2)' , 'Time Taken(s)','Space Used'])31 e = open(file_path_storea,'a')32 e.close()33 22 34 23 #these are the main loops that determine the maximum triangle area (m) and the map side length(n) 35 24 for m in range(20,1000,50): 36 25 for n in range(1,100000,10000): 37 38 z = time.time() #time it39 26 subprocess.call(['python2.5', 'runcairns.py',str(m),str(n)])#run simulation 40 y = time.time() #time it41 42 #read the number of triangles from this text file43 f = open(file_path_storea,'r+')44 h = float(f.readline())45 f.close()46 47 spamWriter.writerow([h,m,(n*n),'x' ,(y-z)]) #record it48 27 print 'DONE' 49 28 50 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #get the memory usage from the log files 29 #get the important data for the experiments from the anuga experiments 30 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 51 31 52 #close ex1.csv so we can read from it for a different csv reader object 53 firstex1.close() 54 55 #merge the metalog useful memory info and the ex1 recorded info into one csv file named final 56 ex1 = csv.reader(open(file_path,'rb')) 32 #open files to read from and write to 57 33 metalog = csv.reader(open(meta_path,'rb')) 58 34 final = csv.writer(open(final_path,'wb')) 59 35 36 #list stores the index of the values requird 37 indexlist = [] 60 38 61 for row in ex1: 62 d = metalog.next() 63 final.writerow([row[0],row[1],d[0],d[3],d[6],d[9],d[12]])#,d[15],d[18]]) 39 #read in the first row 40 firstrow = metalog.next() 41 42 #get the indices of the values we want, so that the data can be condensed 43 indexlist.append(firstrow.index("beforetime")) 44 indexlist.append(firstrow.index("aftertime")) 45 indexlist.append(firstrow.index("beforesimulationmemory")) 46 indexlist.append(firstrow.index("aftermeshmemory")) 47 indexlist.append(firstrow.index("afterinitialconditionsmemory")) 48 indexlist.append(firstrow.index("afterboundarymemory")) 49 indexlist.append(firstrow.index("aftersimulationmemory")) 50 indexlist.append(firstrow.index("variable1")) 51 52 #not all experiments have 3 variables, but some do 53 try: 54 indexlist.append(firstrow.index("variable2")) 55 except ValueError: 56 indexlist.append(firstrow.index("beforetime")) 57 try: 58 indexlist.append(firstrow.index("variable3")) 59 except ValueError: 60 indexlist.append(firstrow.index("aftertime")) 61 62 #write the header for the final csv 63 final.writerow(["TimeTaken(s)",firstrow[(indexlist[2])], 64 firstrow[(indexlist[3])],firstrow[(indexlist[4])], 65 firstrow[(indexlist[5])],firstrow[(indexlist[6])], 66 firstrow[(indexlist[7])],firstrow[(indexlist[8])], 67 firstrow[(indexlist[9])]]) 68 69 #write the data for each column in the final csv 70 for row in metalog: 71 72 #manipulate the beginning and end time to get the time taken 73 begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') 74 end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') 75 taken = time.mktime(end) - time.mktime(begin) 76 77 #write to file 78 final.writerow([str(taken),row[(indexlist[2])], 79 row[(indexlist[3])],row[(indexlist[4])], 80 row[(indexlist[5])],row[(indexlist[6])], 81 row[(indexlist[7])],row[(indexlist[8])], 82 row[(indexlist[9])]]) 83 -
trunk/anuga_work/development/mem_time_tests/triangles/fromregions/runcairns.py
r8328 r8331 9 9 from anuga.utilities import log 10 10 11 # set up the variables for the temporary files12 home = os.getenv('INUNDATIONHOME')13 scenariodir = add_directories(home, ["data","mem_time_test", "triangles",14 "fromregions"])15 storea = 'storea.txt'16 file_path_storea = os.path.join(scenariodir, storea)17 18 19 11 #set up the variables for the simulation output and the log files 20 12 a = sys.argv[1] 21 13 l = sys.argv[2] 22 14 15 home = os.getenv('INUNDATIONHOME') 23 16 scenariodirV = add_directories(home, ["data","mem_time_test", "triangles", 24 17 "fromregions", "triangles-" + str(a) +"-"+ str(l)]) … … 28 21 log.log_filename = os.path.join(scenariodirV, "anuga.log") 29 22 30 log.resource_usage_timing(prefix = 'BeforeSimulation') #get memory usage 23 log.timingInfo(msg=('variable1,'+str(a))) #write the variable to be measured to file 24 log.timingInfo(msg=('variable2,'+str(l))) #write the variable to be measured to file 25 26 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp()))) #get the time at the beginning of the simulation 27 28 log.resource_usage_timing(prefix = 'beforesimulation') #get memory usage 31 29 #------------------------------------------------------------------------------ 32 30 # Create the triangular mesh and domain … … 42 40 domain.set_name('CAIRNS.sww') # Name of sww file 43 41 domain.set_datadir(scenariodirV)# Store sww output here 44 log.resource_usage_timing(prefix = ' AfterMesh') #get memory usage42 log.resource_usage_timing(prefix = 'aftermesh') #get memory usage 45 43 #------------------------------------------------------------------------------ 46 44 # Setup initial conditions … … 49 47 #get the number of triangles 50 48 number=len(domain) 49 log.timingInfo(msg=('variable3,'+str(number))) #write the variable to be measured to file 51 50 52 51 def topography(x,y): … … 78 77 79 78 log.resource_usage_timing(prefix='aftersimulation')#get memory usage 80 81 #write the number of triangles to file 82 i = open(file_path_storea, 'r+') 83 i.write(str(number)) 79 log.timingInfo(msg=('aftertime,'+str(log.TimeStamp()))) #get the time at the end of the simulation -
trunk/anuga_work/development/mem_time_tests/triangles/rectanglecross/main.py
r8328 r8331 16 16 scenariodir = add_directories(home, ["data","mem_time_test", "triangles", 17 17 "rectanglecross"]) 18 storea ='storea.txt'19 file_path_storea = os.path.join(scenariodir, storea)20 18 meta = 'metalog.csv' 21 19 meta_path = os.path.join(scenariodir, meta) 22 20 final = 'final.csv' 23 21 final_path = os.path.join(scenariodir, final) 24 file = 'ex1.csv'25 file_path = os.path.join(scenariodir, file)26 27 #set up needed files28 firstex1 = open(file_path, 'wb')29 spamWriter = csv.writer(firstex1)30 spamWriter.writerow(['Number of Triangles','Matrix Size','Extent(m^2)' , 'Time Taken(s)','Space Used'])31 e = open(file_path_storea,'a')32 e.close()33 22 34 23 #these main loops assign the matrix size[number of triangles] (m) and the map side length(n) 35 24 for m in range(1,300,30): 36 25 for n in range(1,1000,100): 37 38 z = time.time() #time it39 26 subprocess.call(['python2.5', 'runcairns.py',str(m),str(n)]) #run the simulation 40 y = time.time() #time it41 42 #read the number of triangles from this file43 f = open(file_path_storea,'r+')44 h = float(f.readline())45 f.close()46 47 spamWriter.writerow([h,m,(n*n),'x' ,(y-z)]) #record it48 27 print 'DONE' 49 28 50 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #get the memory usage from the log files 29 #get the important data for the experiments from the anuga experiments 30 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 51 31 52 #close ex1.csv so we can read from it for a different csv reader object 53 firstex1.close() 54 55 #merge the metalog useful memory info and the ex1 recorded info into one csv file named final 56 ex1 = csv.reader(open(file_path,'rb')) 32 #open files to read from and write to 57 33 metalog = csv.reader(open(meta_path,'rb')) 58 34 final = csv.writer(open(final_path,'wb')) 59 35 36 #list stores the index of the values requird 37 indexlist = [] 60 38 61 for row in ex1: 62 d = metalog.next() 63 final.writerow([row[0],row[1],d[0],d[3],d[6],d[9],d[12]])#,d[15],d[18]]) 39 #read in the first row 40 firstrow = metalog.next() 64 41 42 #get the indices of the values we want, so that the data can be condensed 43 indexlist.append(firstrow.index("beforetime")) 44 indexlist.append(firstrow.index("aftertime")) 45 indexlist.append(firstrow.index("beforesimulationmemory")) 46 indexlist.append(firstrow.index("aftermeshmemory")) 47 indexlist.append(firstrow.index("afterinitialconditionsmemory")) 48 indexlist.append(firstrow.index("afterboundarymemory")) 49 indexlist.append(firstrow.index("aftersimulationmemory")) 50 indexlist.append(firstrow.index("variable1")) 51 52 #not all experiments have 3 variables, but some do 53 try: 54 indexlist.append(firstrow.index("variable2")) 55 except ValueError: 56 indexlist.append(firstrow.index("beforetime")) 57 try: 58 indexlist.append(firstrow.index("variable3")) 59 except ValueError: 60 indexlist.append(firstrow.index("aftertime")) 61 62 #write the header for the final csv 63 final.writerow(["TimeTaken(s)",firstrow[(indexlist[2])], 64 firstrow[(indexlist[3])],firstrow[(indexlist[4])], 65 firstrow[(indexlist[5])],firstrow[(indexlist[6])], 66 firstrow[(indexlist[7])],firstrow[(indexlist[8])], 67 firstrow[(indexlist[9])]]) 68 69 #write the data for each column in the final csv 70 for row in metalog: 71 72 #manipulate the beginning and end time to get the time taken 73 begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') 74 end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') 75 taken = time.mktime(end) - time.mktime(begin) 76 77 #write to file 78 final.writerow([str(taken),row[(indexlist[2])], 79 row[(indexlist[3])],row[(indexlist[4])], 80 row[(indexlist[5])],row[(indexlist[6])], 81 row[(indexlist[7])],row[(indexlist[8])], 82 row[(indexlist[9])]]) 83 84 85 -
trunk/anuga_work/development/mem_time_tests/triangles/rectanglecross/runcairns.py
r8328 r8331 9 9 from anuga.utilities import log 10 10 11 #set up the variables for the temporary data files12 home = os.getenv('INUNDATIONHOME')13 scenariodir = add_directories(home, ["data","mem_time_test", "triangles",14 "rectanglecross"])15 storea = 'storea.txt'16 file_path_storea = os.path.join(scenariodir, storea)17 18 11 #set up the variables for the simulation out put and log files 19 12 a = int(sys.argv[1]) 20 13 l = float(sys.argv[2]) 21 14 15 home = os.getenv('INUNDATIONHOME') 22 16 scenariodirV = add_directories(home, ["data","mem_time_test", "triangles", 23 17 "rectanglecross", "triangles-" + str(a) +"-"+ str(l)]) … … 25 19 log._setup = False 26 20 27 log.resource_usage_timing(prefix = 'BeforeSimulation')#get memory usage 21 log.timingInfo(msg=('variable1,'+str(a))) #write the variable to be measured to file 22 log.timingInfo(msg=('variable2,'+str(l))) #write the variable to be measured to file 23 24 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp()))) #get the time at the beginning of the simulation 25 26 log.resource_usage_timing(prefix = 'beforesimulation')#get memory usage 28 27 #------------------------------------------------------------------------------ 29 28 # Create the triangular mesh and domain … … 33 32 domain.set_name('CAIRNS.sww') # Name of sww file 34 33 domain.set_datadir(scenariodirV)# Store sww output here 35 log.resource_usage_timing(prefix = ' AfterMesh') #get memory usage34 log.resource_usage_timing(prefix = 'aftermesh') #get memory usage 36 35 #------------------------------------------------------------------------------ 37 36 # Setup initial conditions … … 40 39 #get the number of triangles 41 40 number=len(domain) 41 log.timingInfo(msg=('variable3,'+str(number))) #write the variable to be measured to file 42 42 43 43 def topography(x,y): … … 67 67 68 68 log.resource_usage_timing(prefix='aftersimulation')#get memory usage 69 70 #write the number of triangles to the text file 71 v = open(file_path_storea, 'r+') 72 v.write(str(number)) 69 log.timingInfo(msg=('aftertime,'+str(log.TimeStamp()))) #get the time at the end of the simulation
Note: See TracChangeset
for help on using the changeset viewer.