source: trunk/anuga_work/development/mem_time_tests/linearregression/area/main.py @ 8427

#------------------------------------------------------------------------------
# Import necessary modules
#------------------------------------------------------------------------------
import anuga
import subprocess
import csv
import os
import numpy
import time
import liststore
from anuga.abstract_2d_finite_volumes.util import add_directories
from anuga.utilities.log_analyser import analyse_log

#------------------------------------------------------------------------------
# Set up variables for the correct directories to store the output
#------------------------------------------------------------------------------
home = os.getenv('INUNDATIONHOME')
host = os.getenv('HOST')
scenariodir = add_directories(home, ["data","mem_time_test", 
                                     "linearregression","run"])
meta = 'metalog.csv'
meta_path = os.path.join(scenariodir, meta)
final = 'final.csv'
final_path = os.path.join(scenariodir, final)
equation = 'equation.csv'
equation_path = os.path.join(scenariodir, equation)

#------------------------------------------------------------------------------
# MAIN LOOP
# this is the main loops that assigns the maximum triangle area (m)  
# and the map side length(l)
#------------------------------------------------------------------------------
n = 1 #number of processors to use
for seed in range(0,30):
    stdsidelength = 154.90
    stdtimelength = 3600.0
    stdtimestep = stdtimelength / 60.0
 
    varsidelength = 5 + seed * (800.0/30.0)
    vartimelength = 10 + seed * (86400.0/30.0)
    vartimestep = 0.5 + seed *(1800 / 30)
    vartimestep2 =vartimelength / 60.0

    # side length experiment
    if (host == 'cyclone.agso.gov.au'):
       subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,
       '~/machinefiles/lnrrun.machines_cyclone', '-x','PYTHONPATH','-x',
       'INUNDATIONHOME','python', 'runcairns.py',str(stdtimestep),
       str(stdtimelength),str(varsidelength)]) 
    elif (host == 'tornado.agso.gov.au'):
         subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,
         '~/machinefiles/lnrrun.machines_tornado', '-x','PYTHONPATH','-x',
         'INUNDATIONHOME','python', 'runcairns.py',str(stdtimestep),
         str(stdtimelength),str(varsidelength)])
    elif (host == 'rhe-compute1.ga.gov.au'):
         subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x',
         'INUNDATIONHOME','python2.6', 'runcairns.py',str(stdtimestep),
         str(stdtimelength),str(varsidelength)])    
    else:
         subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x',
         'INUNDATIONHOME','python', 'runcairns.py',str(stdtimestep),
         str(stdtimelength),str(varsidelength)]) 

    # time length experiment
    if (host == 'cyclone.agso.gov.au'):
       subprocess.call(['mpirun', '-np', str(n), 
       '-hostfile' ,'~/machinefiles/lnrrun.machines_cyclone', '-x',
       'PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py',
       str(vartimestep2),str(vartimelength),str(stdsidelength)]) 
    elif (host == 'tornado.agso.gov.au'):
         subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,
         '~/machinefiles/lnrrun.machines_tornado', '-x','PYTHONPATH','-x',
         'INUNDATIONHOME','python', 'runcairns.py',str(vartimestep2),
         str(vartimelength),str(stdsidelength)])
    elif (host == 'rhe-compute1.ga.gov.au'):
         subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x',
         'INUNDATIONHOME','python2.6', 'runcairns.py',str(vartimestep2),
         str(vartimelength),str(stdsidelength)])    
    else:
         subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x',
         'INUNDATIONHOME','python', 'runcairns.py',str(vartimestep2),
         str(vartimelength),str(stdsidelength)]) 


    # time step  experiment
    if (host == 'cyclone.agso.gov.au'):
       subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,
       '~/machinefiles/lnrrun.machines_cyclone', '-x','PYTHONPATH','-x',
       'INUNDATIONHOME','python', 'runcairns.py',str(vartimestep),
       str(stdtimelength),str(stdsidelength)]) 
    elif (host == 'tornado.agso.gov.au'):
         subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,
         '~/machinefiles/lnrrun.machines_tornado', '-x','PYTHONPATH','-x',
         'INUNDATIONHOME','python', 'runcairns.py',str(vartimestep),
         str(stdtimelength),str(stdsidelength)])
    elif (host == 'rhe-compute1.ga.gov.au'):
         subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x',
         'INUNDATIONHOME','python2.6', 'runcairns.py',str(vartimestep),
         str(stdtimelength),str(stdsidelength)])    
    else:
         subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x',
         'INUNDATIONHOME','python', 'runcairns.py',str(vartimestep),
         str(stdtimelength),str(stdsidelength)]) 

print 'Done'

#------------------------------------------------------------------------------
# LOG ANALYSE
# and the Linear Regression
#------------------------------------------------------------------------------

#get the important data for the experiments from the anuga experiments
analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 

#open files to read from and write to
metalog = csv.reader(open(meta_path,'rb'))
final = csv.writer(open(final_path,'wb'))
eq = csv.writer(open(equation_path,'wb'))

#lists used to condense the metalog file and the regression equation
indexlist = []
triangles = []
trianglessquared = []
timelength = []
filewrites = []
timelist = []
space = []

#read in the first row
firstrow = metalog.next()

#get the indices of the values we want, so that the data can be condensed
indexlist.append(firstrow.index("beforetime"))
indexlist.append(firstrow.index("aftertime"))
indexlist.append(firstrow.index("aftermeshtime"))
indexlist.append(firstrow.index("beforesimulationmemory"))
indexlist.append(firstrow.index("aftermeshmemory"))
indexlist.append(firstrow.index("afterinitialconditionsmemory"))
indexlist.append(firstrow.index("afterboundarymemory"))
indexlist.append(firstrow.index("aftersimulationmemory"))

indexlist.append(firstrow.index("numberoftriangles"))
indexlist.append(firstrow.index("finaltime"))
indexlist.append(firstrow.index("timestep"))
indexlist.append(firstrow.index("extent"))
indexlist.append(firstrow.index("sidelength"))
indexlist.append(firstrow.index("maxtrianglearea"))
indexlist.append(firstrow.index("velocity"))
indexlist.append(firstrow.index("depthofwater"))
indexlist.append(firstrow.index("percentageofwatercover"))
indexlist.append(firstrow.index("numberofcpus"))
indexlist.append(firstrow.index("host"))

#write the header for the final csv
final.writerow(["TimeTaken(s)","MeshTime(s)",
                firstrow[(indexlist[3])],firstrow[(indexlist[4])],
                firstrow[(indexlist[5])],firstrow[(indexlist[6])],
                firstrow[(indexlist[7])],firstrow[(indexlist[8])],
                firstrow[(indexlist[9])],firstrow[(indexlist[10])],
                firstrow[(indexlist[11])],firstrow[(indexlist[12])],
                firstrow[(indexlist[13])],firstrow[(indexlist[14])],
                firstrow[(indexlist[15])],firstrow[(indexlist[16])],
                firstrow[(indexlist[17])],firstrow[(indexlist[18])]])

#write the data for each column in the final csv
for row in metalog:

    #manipulate the beginning and end time to get the time taken
    begin = time.strptime(row[(indexlist[0])],'%Y%m%d_%H%M%S')
    end = time.strptime(row[(indexlist[1])],'%Y%m%d_%H%M%S')
    aftermesh = time.strptime(row[(indexlist[2])],'%Y%m%d_%H%M%S')
    taken = time.mktime(end) - time.mktime(begin)
    mesh = time.mktime(aftermesh) - time.mktime(begin)

    #write to file
    final.writerow([str(taken),str(mesh),
                   row[(indexlist[3])],row[(indexlist[4])],
                   row[(indexlist[5])],row[(indexlist[6])],
                   row[(indexlist[7])],row[(indexlist[8])],
                   row[(indexlist[9])],row[(indexlist[10])],
                   row[(indexlist[11])],row[(indexlist[12])],
                   row[(indexlist[13])],row[(indexlist[14])],
                   row[(indexlist[15])],row[(indexlist[16])],
                   row[(indexlist[17])],row[(indexlist[18])]])
     
    #make a list of all the data points, to be used in the regression
    timelist.append(float(taken))
    space.append(float(row[(indexlist[7])]))
    triangles.append(float(row[(indexlist[8])]))
    trianglessquared.append(float(row[(indexlist[8])])**2)
    timelength.append(float(row[(indexlist[9])]))
    filewrites.append(float(row[(indexlist[9])])/float(row[(indexlist[10])]))

#convert these lists to arrays
spa = numpy.asarray(space)
tim = numpy.asarray(timelist)     
tri = numpy.asarray(triangles)
trisq = numpy.asarray(trianglessquared)
timlen = numpy.asarray(timelength)
filwrit = numpy.asarray(filewrites)

# Manipulate the arrays, to stack them into one array that will work the 
# way we want for regression
multivariatearray = numpy.column_stack([tri.flat,trisq.flat,timlen.flat,
                                        filwrit.flat, numpy.ones(90,float)])
multivariatearrayspace = numpy.column_stack([tri.flat, numpy.ones(90,float)])

#get the equations for both time and memory
timequation = numpy.linalg.lstsq(multivariatearray,tim)[0]
spaequation = numpy.linalg.lstsq(multivariatearrayspace,spa)[0]

#calculate the R squared values for each equation
resid1 = numpy.linalg.lstsq(multivariatearray,tim)[1]
resid2 = numpy.linalg.lstsq(multivariatearrayspace,spa)[1]
r2 = 1 - resid1 / (tim.size * tim.var())
r3 = 1 - resid2 / (spa.size * spa.var())

#write these values back to lists
listtime = timequation.tolist()
listspace = spaequation.tolist()

#write the equation data to file
eq.writerow(['Dependent Variable', 'Number of Triangles',
             'Number of Triangles Squared','Time Length','File Writes',
             'Constant', 'R Squared'])
eq.writerow(['Time']+listtime + r2.tolist())
eq.writerow(['Dependent Variable', 'Number of Triangles','Constant', 
             'R Squared'])
eq.writerow(['Space']+listspace + r3.tolist())