Changeset 8338

Timestamp:

Feb 15, 2012, 12:18:12 PM (13 years ago)

Author:

pittj

Message:

linear regression added to get the equation

Location:

trunk/anuga_work/development/mem_time_tests/linearregression/area

Files:

• liststore.py (deleted)
• main.py (modified) (7 diffs)
• pmain.py (modified) (7 diffs)
• pruncairns.py (modified) (2 diffs)

trunk/anuga_work/development/mem_time_tests/linearregression/area/main.py

@@ -6 +6 @@
 import csv
 import os
+import numpy
 import time
 import liststore
@@ -21 +22 @@
 final = 'final.csv'
 final_path = os.path.join(scenariodir, final)
+equation = 'equation.csv'
+equation_path = os.path.join(scenariodir, equation)
 
+#------------------------------------------------------------------------------
+# MAIN LOOP
 # this is the main loops that assigns the maximum triangle area (m) and the map side length(l)
-
+#------------------------------------------------------------------------------
 n = 1 #number of processors to use
 for seed in range(0,30):
@@ -57 +62 @@
 
     # time step  experiment
-
     if (host == 'cyclone.agso.gov.au'):
        subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/lnrrun.machines_cyclone', '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py',str(vartimestep),str(stdtimelength),str(stdsidelength)]) 
@@ -69 +73 @@
 print 'Done'
 
+#------------------------------------------------------------------------------
+# LOG ANALYSE
+# and the Linear Regression
+#------------------------------------------------------------------------------
+
 #get the important data for the experiments from the anuga experiments
 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 
@@ -75 +84 @@
 metalog = csv.reader(open(meta_path,'rb'))
 final = csv.writer(open(final_path,'wb'))
+eq = csv.writer(open(equation_path,'wb'))
 
-#list stores the index of the values requird
+#lists used to condense the metalog file and the regression equation
 indexlist = []
+triangles = []
+trianglessquared = []
+timelength = []
+filewrites = []
+timelist = []
+space = []
 
 #read in the first row
@@ -103 +119 @@
 indexlist.append(firstrow.index("numberofcpus"))
 indexlist.append(firstrow.index("host"))
-
-
 
 #write the header for the final csv
@@ -137 +151 @@
                    row[(indexlist[15])],row[(indexlist[16])],
                    row[(indexlist[17])],row[(indexlist[18])]])
+     
+    #make a list of all the data points, to be used in the regression
+    timelist.append(float(taken))
+    space.append(float(row[(indexlist[7])]))
+    triangles.append(float(row[(indexlist[8])]))
+    trianglessquared.append(float(row[(indexlist[8])])**2)
+    timelength.append(float(row[(indexlist[9])]))
+    filewrites.append(float(row[(indexlist[9])])/float(row[(indexlist[10])]))
 
-subprocess.call(['python','pmain.py'])
+#convert these lists to arrays
+spa = numpy.asarray(space)
+tim = numpy.asarray(timelist)     
+tri = numpy.asarray(triangles)
+trisq = numpy.asarray(trianglessquared)
+timlen = numpy.asarray(timelength)
+filwrit = numpy.asarray(filewrites)
+
+#manipulate the arrays, to stack them into one array that will work the way we want for regression
+multivariatearray = numpy.column_stack([tri.flat,trisq.flat,timlen.flat,filwrit.flat, numpy.ones(90,float)])
+multivariatearrayspace = numpy.column_stack([tri.flat, numpy.ones(90,float)])
+
+#get the equations for both time and memory
+timequation = numpy.linalg.lstsq(multivariatearray,tim)[0]
+spaequation = numpy.linalg.lstsq(multivariatearrayspace,spa)[0]
+
+#calculate the R squared values for each equation
+resid1 = numpy.linalg.lstsq(multivariatearray,tim)[1]
+resid2 = numpy.linalg.lstsq(multivariatearrayspace,spa)[1]
+r2 = 1 - resid1 / (tim.size * tim.var())
+r3 = 1 - resid2 / (spa.size * spa.var())
+
+#write these values back to lists
+listtime = timequation.tolist()
+listspace = spaequation.tolist()
+
+#write the equation data to file
+eq.writerow(['Dependent Variable', 'Number of Triangles','Number of Triangles Squared','Time Length','File Writes','Constant', 'R Squared'])
+eq.writerow(['Time']+listtime + r2.tolist())
+eq.writerow(['Dependent Variable', 'Number of Triangles','Constant', 'R Squared'])
+eq.writerow(['Space']+listspace + r3.tolist())
+
+

trunk/anuga_work/development/mem_time_tests/linearregression/area/pmain.py

@@ -7 +7 @@
 import os
 import time
-import liststore
 from anuga.abstract_2d_finite_volumes.util import add_directories
 from anuga.utilities.log_analyser import analyse_log
@@ -22 +21 @@
 final_path = os.path.join(scenariodir, final)
 
+#------------------------------------------------------------------------------
+# MAIN LOOP
 # this is the main loops that assigns the maximum triangle area (m) and the map side length(l)
-
+#------------------------------------------------------------------------------
 
 for n in range(0,24):
@@ -42 +43 @@
 print 'Done'
 
+#------------------------------------------------------------------------------
+# LOG ANALYSE
+# and the Linear Regression (This has not been implemented as the nature of the
+# relationship between the number of cpus and time as well as memory)
+#------------------------------------------------------------------------------
+
 #get the important data for the experiments from the anuga experiments
 analyse_log(scenariodir, os.path.join(scenariodir,'metalog.csv')) 
@@ -49 +56 @@
 final = csv.writer(open(final_path,'wb'))
 
-#list stores the index of the values requird
+#lists used to condense the metalog file
 indexlist = []
 
@@ -75 +82 @@
 indexlist.append(firstrow.index("percentageofwatercover"))
 indexlist.append(firstrow.index("numberofcpus"))
+indexlist.append(firstrow.index("myid"))
 indexlist.append(firstrow.index("host"))
-
-
 
 #write the header for the final csv
@@ -88 +94 @@
                 firstrow[(indexlist[13])],firstrow[(indexlist[14])],
                 firstrow[(indexlist[15])],firstrow[(indexlist[16])],
-                firstrow[(indexlist[17])],firstrow[(indexlist[18])]])
+                firstrow[(indexlist[17])],firstrow[(indexlist[18])]],
+                firstrow[(indexlist[19])]])
 
 #write the data for each column in the final csv
@@ -109 +116 @@
                    row[(indexlist[13])],row[(indexlist[14])],
                    row[(indexlist[15])],row[(indexlist[16])],
-                   row[(indexlist[17])],row[(indexlist[18])]])
-
+                   row[(indexlist[17])],row[(indexlist[18])]],
+                   row[(indexlist[19])]])
+     

trunk/anuga_work/development/mem_time_tests/linearregression/area/pruncairns.py

@@ -5 +5 @@
 import time
 import sys
-import liststore
 import anuga
 from anuga_parallel import distribute, numprocs, myid
@@ -37 +36 @@
 log.timingInfo(msg=('numberofcpus,'+str(numprocs))) #write the variable to be measured to file
 log.timingInfo(msg=('host,'+str(os.getenv('HOST')))) #write the variable to be measured to file
-
+log.timingInfo(msg=('myid,'+str(myid)) #write the variable to be measured to file
 
 log.timingInfo(msg=('beforetime,'+str(log.TimeStamp()))) #get the time at the beginning of the simulation