#!/bin/bash
#PBS -P j23
#PBS -q normal
#PBS -l walltime=120,ncpus=4,vmem=400mb,other=mpi
#PBS -wd

#########################
# The above #PBS directives do the following:
#########################
# #PBS -P j23
# Set the project to be charged to be j23
#
# #PBS -q normal
# Run this job in the normal queue
#
# #PBS -l walltime=120,ncpus=4,vmem=400mb,other=mpi 
# This job will not take longer than 120 seconds.
# This job requires 4 processors.
# This job will not consume more than 400mb.
# This job requires MPI.
#
# #PBS -wd
# Start the job in the directory it was submitted from.
# (This script must be submitted from the ga/inundation/parallel directory)
#
#########################

#########################
# The APAC cluster uses modules to customise the user's environment.
# Python and LAM-MPI are required for this job. MPICH will not work, because it
# cannot pass the PYTHONPATH environment variable into the mpi job.
#########################
module load python
module load lam

#########################
# As with a standard ANUGA installation, PYTHONPATH must be correctly set.
# The job will fail strangely on the APAC Linux Cluster if LD_ASSUME_KERNEL
# is not set. This is not required on other machines.
#########################
export PYTHONPATH=/home/661/jdk661/inundation
export LD_ASSUME_KERNEL=2.4.1

#########################
# Start the actual MPI job on 4 nodes.
# Ensure that the MPI environment has the LD_LIBRARY_PATH (used because the
# modular environment places some libraries in nonstandard locations),
# PYTHONPATH and LD_ASSUME_KERNEL environment variables set correctly.
#########################
lamboot
mpirun -np 4 -x LD_LIBRARY_PATH,PYTHONPATH,LD_ASSUME_KERNEL run_parallel_sw_merimbula_metis.py
lamhalt