#------------------------------------------------------------------------------ # Import necessary modules #------------------------------------------------------------------------------ import anuga import subprocess import csv import os import numpy import time import liststore from anuga.abstract_2d_finite_volumes.util import add_directories from anuga.utilities.log_analyser import analyse_log #------------------------------------------------------------------------------ # Set up variables for the correct directories to store the output #------------------------------------------------------------------------------ home = os.getenv('INUNDATIONHOME') host = os.getenv('HOST') scenariodir = add_directories(home, ["data","mem_time_test", "linearregression","run"]) meta = 'metalog.csv' meta_path = os.path.join(scenariodir, meta) final = 'final.csv' final_path = os.path.join(scenariodir, final) equation = 'equation.csv' equation_path = os.path.join(scenariodir, equation) #------------------------------------------------------------------------------ # MAIN LOOP # this is the main loops that assigns the maximum triangle area (m) # and the map side length(l) #------------------------------------------------------------------------------ n = 1 #number of processors to use for seed in range(0,30): stdsidelength = 154.90 stdtimelength = 3600.0 stdtimestep = stdtimelength / 60.0 varsidelength = 5 + seed * (800.0/30.0) vartimelength = 10 + seed * (86400.0/30.0) vartimestep = 0.5 + seed *(1800 / 30) vartimestep2 =vartimelength / 60.0 # side length experiment if (host == 'cyclone.agso.gov.au'): subprocess.call(['mpirun', '-np', str(n), '-hostfile' , '~/machinefiles/lnrrun.machines_cyclone', '-x','PYTHONPATH','-x', 'INUNDATIONHOME','python', 'runcairns.py',str(stdtimestep), str(stdtimelength),str(varsidelength)]) elif (host == 'tornado.agso.gov.au'): subprocess.call(['mpirun', '-np', str(n), '-hostfile' , '~/machinefiles/lnrrun.machines_tornado', '-x','PYTHONPATH','-x', 'INUNDATIONHOME','python', 'runcairns.py',str(stdtimestep), str(stdtimelength),str(varsidelength)]) elif (host == 'rhe-compute1.ga.gov.au'): subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x', 'INUNDATIONHOME','python2.6', 'runcairns.py',str(stdtimestep), str(stdtimelength),str(varsidelength)]) else: subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x', 'INUNDATIONHOME','python', 'runcairns.py',str(stdtimestep), str(stdtimelength),str(varsidelength)]) # time length experiment if (host == 'cyclone.agso.gov.au'): subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/lnrrun.machines_cyclone', '-x', 'PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py', str(vartimestep2),str(vartimelength),str(stdsidelength)]) elif (host == 'tornado.agso.gov.au'): subprocess.call(['mpirun', '-np', str(n), '-hostfile' , '~/machinefiles/lnrrun.machines_tornado', '-x','PYTHONPATH','-x', 'INUNDATIONHOME','python', 'runcairns.py',str(vartimestep2), str(vartimelength),str(stdsidelength)]) elif (host == 'rhe-compute1.ga.gov.au'): subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x', 'INUNDATIONHOME','python2.6', 'runcairns.py',str(vartimestep2), str(vartimelength),str(stdsidelength)]) else: subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x', 'INUNDATIONHOME','python', 'runcairns.py',str(vartimestep2), str(vartimelength),str(stdsidelength)]) # time step experiment if (host == 'cyclone.agso.gov.au'): subprocess.call(['mpirun', '-np', str(n), '-hostfile' , '~/machinefiles/lnrrun.machines_cyclone', '-x','PYTHONPATH','-x', 'INUNDATIONHOME','python', 'runcairns.py',str(vartimestep), str(stdtimelength),str(stdsidelength)]) elif (host == 'tornado.agso.gov.au'): subprocess.call(['mpirun', '-np', str(n), '-hostfile' , '~/machinefiles/lnrrun.machines_tornado', '-x','PYTHONPATH','-x', 'INUNDATIONHOME','python', 'runcairns.py',str(vartimestep), str(stdtimelength),str(stdsidelength)]) elif (host == 'rhe-compute1.ga.gov.au'): subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x', 'INUNDATIONHOME','python2.6', 'runcairns.py',str(vartimestep), str(stdtimelength),str(stdsidelength)]) else: subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x', 'INUNDATIONHOME','python', 'runcairns.py',str(vartimestep), str(stdtimelength),str(stdsidelength)]) print 'Done' #------------------------------------------------------------------------------ # LOG ANALYSE # and the Linear Regression #------------------------------------------------------------------------------ #get the important data for the experiments from the anuga experiments analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) #open files to read from and write to metalog = csv.reader(open(meta_path,'rb')) final = csv.writer(open(final_path,'wb')) eq = csv.writer(open(equation_path,'wb')) #lists used to condense the metalog file and the regression equation indexlist = [] triangles = [] trianglessquared = [] timelength = [] filewrites = [] timelist = [] space = [] #read in the first row firstrow = metalog.next() #get the indices of the values we want, so that the data can be condensed indexlist.append(firstrow.index("beforetime")) indexlist.append(firstrow.index("aftertime")) indexlist.append(firstrow.index("aftermeshtime")) indexlist.append(firstrow.index("beforesimulationmemory")) indexlist.append(firstrow.index("aftermeshmemory")) indexlist.append(firstrow.index("afterinitialconditionsmemory")) indexlist.append(firstrow.index("afterboundarymemory")) indexlist.append(firstrow.index("aftersimulationmemory")) indexlist.append(firstrow.index("numberoftriangles")) indexlist.append(firstrow.index("finaltime")) indexlist.append(firstrow.index("timestep")) indexlist.append(firstrow.index("extent")) indexlist.append(firstrow.index("sidelength")) indexlist.append(firstrow.index("maxtrianglearea")) indexlist.append(firstrow.index("velocity")) indexlist.append(firstrow.index("depthofwater")) indexlist.append(firstrow.index("percentageofwatercover")) indexlist.append(firstrow.index("numberofcpus")) indexlist.append(firstrow.index("host")) #write the header for the final csv final.writerow(["TimeTaken(s)","MeshTime(s)", firstrow[(indexlist[3])],firstrow[(indexlist[4])], firstrow[(indexlist[5])],firstrow[(indexlist[6])], firstrow[(indexlist[7])],firstrow[(indexlist[8])], firstrow[(indexlist[9])],firstrow[(indexlist[10])], firstrow[(indexlist[11])],firstrow[(indexlist[12])], firstrow[(indexlist[13])],firstrow[(indexlist[14])], firstrow[(indexlist[15])],firstrow[(indexlist[16])], firstrow[(indexlist[17])],firstrow[(indexlist[18])]]) #write the data for each column in the final csv for row in metalog: #manipulate the beginning and end time to get the time taken begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S') end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S') aftermesh = time.strptime(row[(indexlist[2])],'%Y%m%d_%H%M%S') taken = time.mktime(end) - time.mktime(begin) mesh = time.mktime(aftermesh) - time.mktime(begin) #write to file final.writerow([str(taken),str(mesh), row[(indexlist[3])],row[(indexlist[4])], row[(indexlist[5])],row[(indexlist[6])], row[(indexlist[7])],row[(indexlist[8])], row[(indexlist[9])],row[(indexlist[10])], row[(indexlist[11])],row[(indexlist[12])], row[(indexlist[13])],row[(indexlist[14])], row[(indexlist[15])],row[(indexlist[16])], row[(indexlist[17])],row[(indexlist[18])]]) #make a list of all the data points, to be used in the regression timelist.append(float(taken)) space.append(float(row[(indexlist[7])])) triangles.append(float(row[(indexlist[8])])) trianglessquared.append(float(row[(indexlist[8])])**2) timelength.append(float(row[(indexlist[9])])) filewrites.append(float(row[(indexlist[9])])/float(row[(indexlist[10])])) #convert these lists to arrays spa = numpy.asarray(space) tim = numpy.asarray(timelist) tri = numpy.asarray(triangles) trisq = numpy.asarray(trianglessquared) timlen = numpy.asarray(timelength) filwrit = numpy.asarray(filewrites) # Manipulate the arrays, to stack them into one array that will work the # way we want for regression multivariatearray = numpy.column_stack([tri.flat,trisq.flat,timlen.flat, filwrit.flat, numpy.ones(90,float)]) multivariatearrayspace = numpy.column_stack([tri.flat, numpy.ones(90,float)]) #get the equations for both time and memory timequation = numpy.linalg.lstsq(multivariatearray,tim)[0] spaequation = numpy.linalg.lstsq(multivariatearrayspace,spa)[0] #calculate the R squared values for each equation resid1 = numpy.linalg.lstsq(multivariatearray,tim)[1] resid2 = numpy.linalg.lstsq(multivariatearrayspace,spa)[1] r2 = 1 - resid1 / (tim.size * tim.var()) r3 = 1 - resid2 / (spa.size * spa.var()) #write these values back to lists listtime = timequation.tolist() listspace = spaequation.tolist() #write the equation data to file eq.writerow(['Dependent Variable', 'Number of Triangles', 'Number of Triangles Squared','Time Length','File Writes', 'Constant', 'R Squared']) eq.writerow(['Time']+listtime + r2.tolist()) eq.writerow(['Dependent Variable', 'Number of Triangles','Constant', 'R Squared']) eq.writerow(['Space']+listspace + r3.tolist())