Ignore:
Timestamp:
Jan 31, 2012, 10:54:12 AM (13 years ago)
Author:
pittj
Message:

formatted the experiment scripts

Location:
trunk/anuga_work/development/mem_time_tests/hardware
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/anuga_work/development/mem_time_tests/hardware/cairns/main.py

    r8320 r8326  
    55# Import necessary modules
    66#------------------------------------------------------------------------------
    7 # Import standard shallow water domain and standard boundaries.
    87import anuga
    9 import random
    108import subprocess
    119import csv
    1210import os
    1311import time
    14 import liststore
    15 from anuga.utilities import system_tools, log
    1612from anuga.abstract_2d_finite_volumes.util import add_directories
     13from anuga.utilities.log_analyser import analyse_log
    1714
    18 
    19 #------------------------------------------------------------------------------
    20 # Setup computational domain
    21 #------------------------------------------------------------------------------
    22 a = []
     15#--------------------------------------------------------------------------------------
     16# Set up variables for the correct directories to store the output
     17#--------------------------------------------------------------------------------------
    2318host = os.getenv('HOST')
    24 home1 = os.getenv('INUNDATIONHOME')
    25 scenariodir = add_directories(home1, ["data","mem_time_test", "parallel","cairns"])
     19home = os.getenv('INUNDATIONHOME')
     20scenariodir = add_directories(home, ["data","mem_time_test", "parallel","cairns"])
    2621file1 = 'ex1.csv'
    2722file_path = os.path.join(scenariodir, file1)
    28 store ='store.txt'
    29 file_path_store = os.path.join(scenariodir, store)
    30 
    31 log.log_filename = os.path.join(scenariodir, 'log.txt')
    3223
    3324spamWriter = csv.writer(open(file_path, 'wb'))
    3425spamWriter.writerow(['Number Of Processors' , 'Space Used MB' , 'Time Taken s'])
    35 e = open(file_path_store,'a')
    36 e.close()
    3726
     27
     28# main loop that runs macpus - 1 times
    3829maxcpus = 18
    3930
    4031for n in range(1,maxcpus,1):
    41     z = time.time()
    42     y = time.time()
     32    z = time.time() # time it
    4333
     34    #the necessary ways to run this script in parallel on the different hosts we use
    4435    if (host == 'cyclone.agso.gov.au'):
    45         subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_cyclone', '-x','PYTHONPATH','-x','INUNDATIONHOME','python2.6', 'runcairns.py'])
     36        subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_cyclone', '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py'])
    4637    if (host == 'tornado.agso.gov.au'):
    47         subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_tornado', '-x','PYTHONPATH','-x','INUNDATIONHOME','python2.6', 'runcairns.py'])
     38        subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_tornado', '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py'])
    4839    if (host == 'vayu1'):
    4940        subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py'])
    … …  
    5344        subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py'])
    5445
    55     liststore.spacelist = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
    56     f = open(file_path_store,'r+')
    57     x = float(f.readline())
    58     f.close()
    59     spamWriter.writerow([n,x ,(y-z)])
    60 print a
     46    y = time.time()# time it
     47    spamWriter.writerow([n,'x' ,(y-z)]) # write recorded results
     48print 'DONE'
    6149
     50analyse_log(scenariodir, 'metalog.csv') # interrogate log files for the memory usage information

  • trunk/anuga_work/development/mem_time_tests/hardware/cairns/project.py

    r8311 r8326  
    11""" Common filenames and locations for topographic data, meshes and outputs.
    22    This file defines the parameters of the scenario you wish to run.
     3
     4    Parameters differ from example to make the code run faster, and so the data is stored and found in the correct places
    35"""
    46

  • trunk/anuga_work/development/mem_time_tests/hardware/cairns/runcairns.py

    r8311 r8326  
    1010
    1111Geoscience Australia, 2004-present
     12
     13This has remained unchanged aside from the parallelism added, the resource statistics
     14and the output directories, its also been scaled down so it runs faster
    1215"""
    1316
    … …  
    1922import time
    2023import sys
     24import anuga
     25from anuga_parallel import distribute, myid, numprocs
     26from anuga.abstract_2d_finite_volumes.util import add_directories
     27from anuga.utilities import log
    2128
    22 # Related major packages
    23 import anuga
    24 import liststore
    2529
    2630# Application specific imports
    2731import project                 # Definition of file names and polygons
    2832
    29 from anuga_parallel import distribute, myid, numprocs, finalize, barrier
    30 from anuga.abstract_2d_finite_volumes.util import add_directories
    31 from anuga.utilities import system_tools, log
     33
     34# set up variables for the correct output directories
     35home = os.getenv('INUNDATIONHOME')
     36scenariodir = add_directories(home, ["data", "mem_time_test", "parallel", "cairns"])
     37h = 'CAIRNS.msh'
     38file_pathh = os.path.join(scenariodir, h)
     39store ='store.txt'
     40file_path_store = os.path.join(scenariodir, store)
     41scenariodirV = add_directories(home, ["data","mem_time_test", "parallel",
     42                                       "cairns", "parrallel-" + str(numprocs) +"-"+str(myid)])
     43log.log_filename = os.path.join(scenariodirV, "anuga.log")
     44log._setup = False
    3245
    3346
    34 home2 = os.getenv('INUNDATIONHOME')
    35 
    36 scenariodir2 = add_directories(home2, ["data", "mem_time_test", "parallel", "cairns"])
    37 
    38 h = 'CAIRNS.msh'
    39 file_pathh = os.path.join(scenariodir2, h)
    40 store ='store.txt'
    41 file_path_store = os.path.join(scenariodir2, store)
    42 
    43 
    44 system_tools.MemoryUpdate()
    45 
     47log.resource_usage_timing(prefix = 'BeforeSimulation')#get memory statistics at this point
    4648#------------------------------------------------------------------------------
    4749# Preparation of topographic data
    … …  
    4951#------------------------------------------------------------------------------
    5052# Create DEM from asc data
    51 anuga.asc2dem(os.path.join(scenariodir2, 'cairns.asc'), use_cache=True, verbose=True)
     53anuga.asc2dem(os.path.join(scenariodir, 'cairns.asc'), use_cache=True, verbose=True)
    5254
    5355# Create pts file for onshore DEM
    54 anuga.dem2pts(os.path.join(scenariodir2,'cairns.dem'), use_cache=True, verbose=True)
     56anuga.dem2pts(os.path.join(scenariodir,'cairns.dem'), use_cache=True, verbose=True)
    5557
    5658#------------------------------------------------------------------------------
    … …  
    5860# overall clipping polygon with a tagged
    5961# boundary and interior regions as defined in project.py
     62# (in serial, so the set up only runs once)
    6063#------------------------------------------------------------------------------
    6164if myid == 0:
    … …  
    7780else:
    7881    domain = None
     82
     83log.resource_usage_timing(prefix = 'AfterMesh')#get memory statistics at this point
    7984     
     85#parallel
    8086domain = distribute(domain)
    8187                               
    … …  
    8490#------------------------------------------------------------------------------
    8591domain.set_name('cairns_' + project.scenario) # Name of sww file
    86 domain.set_datadir(scenariodir2)                       # Store sww output here
     92domain.set_datadir(scenariodirV)                       # Store sww output here
    8793domain.set_minimum_storable_height(0.01)      # Store only depth > 1cm
    8894
    … …  
    95101domain.set_quantity('friction', 0.0)
    96102domain.set_quantity('elevation',
    97                     filename=os.path.join(scenariodir2, 'cairns.pts'),
     103                    filename=os.path.join(scenariodir, 'cairns.pts'),
    98104                    use_cache=True,
    99105                    verbose=True,
    100106                    alpha=0.1)
     107log.resource_usage_timing(prefix='afterinitialconditions') #get memory statistics at this point
    101108
    102109#------------------------------------------------------------------------------
    … …  
    138145                         'onshore': Bd,
    139146                         'top': Bd})
    140 
     147log.resource_usage_timing(prefix='afterboundary') #get memory statistics at this point
    141148#------------------------------------------------------------------------------
    142149# Evolve system through time
    … …  
    175182        print domain.boundary_statistics(tags='ocean_east')
    176183           
     184
    177185print 'That took %.2f seconds' %(time.time()-t0)
    178186
    179 liststore.spacelist[myid] = system_tools.MemoryUpdate()[0]
    180 a = sum(liststore.spacelist)
    181 
    182 f = open(file_path_store, 'r+')
    183 f.write(str(a))
    184 
     187log.resource_usage_timing(prefix='aftersimulation') #get memory statistics at this point

  • trunk/anuga_work/development/mem_time_tests/hardware/template/main.py

    r8320 r8326  
    55# Import necessary modules
    66#------------------------------------------------------------------------------
    7 # Import standard shallow water domain and standard boundaries.
    87import anuga
    98import random
    109import subprocess
    11 import run_parallel
    1210import csv
    1311import os
    1412import time
    15 import liststore
    16 from anuga.utilities import system_tools, log
    1713from anuga.abstract_2d_finite_volumes.util import add_directories
     14from anuga.utilities.log_analyser import analyse_log
    1815
    19 
    20 #------------------------------------------------------------------------------
    21 # Setup computational domain
    22 #------------------------------------------------------------------------------
    23 a = []
     16#-------------------------------------------------------------------------------
     17# Set up variables for the correct directories to store the output
     18#-------------------------------------------------------------------------------
    2419host = os.getenv('HOST')
    2520home = os.getenv('INUNDATIONHOME')
    … …  
    2722file1 = 'ex1.csv'
    2823file_path = os.path.join(scenariodir, file1)
    29 storel ='storel.txt'
    30 file_path_storel = os.path.join(scenariodir, storel)
    31 storea ='storea.txt'
    32 file_path_storea = os.path.join(scenariodir, storea)
    33 store ='store.txt'
    34 file_path_store = os.path.join(scenariodir, store)
    35 storen ='storen.txt'
    36 file_path_storen = os.path.join(scenariodir, storen)
    37 
    38 log.log_filename = os.path.join(scenariodir, 'log.txt')
    39 
    4024spamWriter = csv.writer(open(file_path, 'wb'))
    4125spamWriter.writerow(['Number Of Processors' , 'Space Used MB' , 'Time Taken s','Total Space'])
    42 e = open(file_path_store,'a')
    43 e.close()
    4426
    4527
     28#main loop that runs maxcpus - 1 times
     29maxcpus = 40
    4630
    47 for n in range(1,40,1):
    48     z = time.time()
     31for n in range(1,maxcpus,1):
     32    z = time.time()# time it
     33
     34    #the different ways each HOST requires the parallel script to be run
    4935    if (host == 'cyclone.agso.gov.au'):
    50         subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_cyclone', '-x','PYTHONPATH','-x','INUNDATIONHOME','python2.6', 'runcairns.py'])
     36        subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_cyclone', '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py'])
    5137    if (host == 'tornado.agso.gov.au'):
    52         subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_tornado', '-x','PYTHONPATH','-x','INUNDATIONHOME','python2.6', 'runcairns.py'])
     38        subprocess.call(['mpirun', '-np', str(n), '-hostfile' ,'~/machinefiles/test.machines_tornado', '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py'])
    5339    if (host == 'vayu1'):
    5440        subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py'])
    … …  
    5743    if (host == 'xe'):
    5844        subprocess.call(['mpirun', '-np', str(n), '-x','PYTHONPATH','-x','INUNDATIONHOME','python', 'runcairns.py'])
    59  
    60     y = time.time()
    61     f = open(file_path_store,'r+')
    62     x = float(f.readline())
    63     f.close()
    64     spamWriter.writerow([n,liststore.spacelist,(y-z),x])
    65     liststore.spacelist = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
    66 print a
     45   
    6746
     47    y = time.time() #time it
     48    spamWriter.writerow([n,(y-z)]) #record it
     49
     50print 'Done'
     51
     52analyse_log(scenariodir, 'metalog.csv') #get all the memory usage statistics

  • trunk/anuga_work/development/mem_time_tests/hardware/template/runcairns.py

    r8311 r8326  
    1 """Script for running a tsunami inundation scenario for Cairns, QLD Australia.
     1"""
     2This experiment was based heavily on the cairns demo given in the user manual and found in
     3the anuga repository.
    24
    3 Source data such as elevation and boundary data is assumed to be available in
    4 directories specified by project.py
    5 The output sww file is stored in directory named after the scenario, i.e
    6 slide or fixed_wave.
    7 
    8 The scenario is defined by a triangular mesh created from project.polygon,
    9 the elevation data and a tsunami wave generated by a submarine mass failure.
    10 
    11 Geoscience Australia, 2004-present
     5However, it has been simplified so that it doesnt require all the files the cairns demo does
    126"""
    137
    … …  
    159# Import necessary modules
    1610#------------------------------------------------------------------------------
    17 # Standard modules
    1811import os
    1912import time
    2013import sys
     14import anuga
     15from anuga_parallel import distribute, myid, numprocs
     16from anuga.abstract_2d_finite_volumes.util import add_directories
     17from anuga.utilities import log
    2118
    22 # Related major packages
    23 import anuga
     19#set up variables for the correct I/O directories for data storage
     20home = os.getenv('INUNDATIONHOME')
     21scenariodir = add_directories(home, ["data", "mem_time_test", "parallel", "template"])
     22scenariodirV = add_directories(home, ["data","mem_time_test", "parallel",
     23                                       "template", "template-" + str(numprocs) +"-"+ str(myid)])
     24h = 'CAIRNS.msh'
     25file_pathh = os.path.join(scenariodirV, h)
     26log.log_filename = os.path.join(scenariodirV, "anuga.log")
     27log._setup = False
    2428
    25 from anuga_parallel import distribute, myid
    26 import liststore
    27 from anuga.abstract_2d_finite_volumes.util import add_directories
    28 from anuga.utilities import system_tools, log
     29log.resource_usage_timing(prefix = 'BeforeSimulation')#get memory statistics here
     30#------------------------------------------------------------------------------
     31#Create the domain and mesh for the resource experiment on only one processor
     32#------------------------------------------------------------------------------
    2933
    30 
    31 home2 = os.getenv('INUNDATIONHOME')
    32 
    33 scenariodir2 = add_directories(home2, ["data", "mem_time_test", "triangles", "area"])
    34 
    35 store ='store.txt'
    36 file_path_store = os.path.join(scenariodir2, store)
    37 h = 'cairnsmesh.msh'
    38 file_pathh = os.path.join(scenariodir2, h)
    39 
    40 
    41 system_tools.MemoryUpdate()
    42 #------------------------------------------------------------------------------
    43 # Create the triangular mesh and domain based on
    44 # overall clipping polygon with a tagged
    45 # boundary and interior regions as defined in project.py
    46 #------------------------------------------------------------------------------
    4734if myid == 0:
    4835    domain = anuga.create_domain_from_regions([(0.0,0.0),(10000.0,10000.0),(0.0,10000.0),(10000.0,0.0)],
    … …  
    5239                                                   'left': [3]},
    5340                                    maximum_triangle_area=100.0,
    54                                     mesh_filename=file_pathh
    55                                     #,interior_regions=INTERIORREGIONS#,
    56                                     #use_cache=True,
    57                                     #verbose=True)
    58                                     )
    59 
    60     # Print some stats about mesh and domain
    61     #print 'Number of triangles = ', len(domain)
    62     #print 'The extent is ', domain.get_extent()
    63     #print domain.statistics()
     41                                    mesh_filename=file_pathh)
    6442else:
    6543    domain = None
    66      
     44
     45#parallel   
    6746domain = distribute(domain)
    68                                
    69 #------------------------------------------------------------------------------
    70 # Setup parameters of computational domain
    71 #------------------------------------------------------------------------------
    7247domain.set_name('CAIRNS') # Name of sww file
    73 domain.set_datadir(scenariodir2)                       # Store sww output here
     48domain.set_datadir(scenariodirV)# Store sww output here
     49log.resource_usage_timing(prefix = 'AfterMesh')#get memory statistics here
    7450
    7551#------------------------------------------------------------------------------
    … …  
    8662domain.set_quantity('elevation',topography,alpha=0.1)
    8763
    88 
     64log.resource_usage_timing(prefix='afterinitialconditions') #get memory statistics here
    8965
    9066#------------------------------------------------------------------------------
    … …  
    9470Bi = anuga.Dirichlet_boundary([tide, 223.52, 0]) # inflow
    9571Bo = anuga.Dirichlet_boundary([-tide, 223.52, 0]) # inflow
    96 Bs = anuga.Transmissive_stage_zero_momentum_boundary(domain) # Neutral boundary
    9772Br = anuga.Reflective_boundary(domain)
    98 #Bw = anuga.Time_boundary(domain=domain,function=lambda t: [(60<t<3660)*50, 0, 0])
    99 domain.set_boundary({'right': Bo,
    100                      'bottom': Br,
    101                      'left': Bi,
    102                      'top': Br})
     73domain.set_boundary({'right': Bo,'bottom': Br,'left': Bi,'top': Br})
    10374
     75log.resource_usage_timing(prefix='afterboundary') #get memory statistics here
    10476#------------------------------------------------------------------------------
    10577# Evolve system through time
    10678#------------------------------------------------------------------------------
    107 
    108 # Save every two mins leading up to wave approaching land
    10979for t in domain.evolve(yieldstep=120, finaltime=2000):
    11080    print domain.timestepping_statistics()
    111    
    11281
    113 liststore.spacelist[myid] = system_tools.MemoryUpdate()[0]
    114 a = sum(liststore.spacelist)
    115 print liststore.spacelist
    116 
    117 f = open(file_path_store, 'r+')
    118 f.write(str(a))
    119 
     82log.resource_usage_timing(prefix='aftersimulation')#get memory statistics here 
Note: See TracChangeset for help on using the changeset viewer.