#!/usr/bin/env python ''' Program to update a DVD jurisdiction 'anuga' directory from the original source directory. usage: update.py where is one of: Hobart BatemansBay Gosford GoldCoast ie, the name of one of the DVD jurisdiction staging directories (case-insensitive). ''' import os import sys import shutil import time # common base path to all data and project files main_path = '/nas/gemd/georisk_models/inundation' # structures holding data for each jurisdiction # data_src_path : sub-path to data directory # proj_src_path : sub-path to project files # data_dst_path : where to copy data files to # proj_dst_path : where to copy project files to # copy_data_dirs : data directories to completely copy # make_dst_dirs : make these data directories (may be left empty) # copy_data_files: individual data files to copy to # copy_proj_files: individual project files to copy to hobart_data = \ {'jurisdiction': 'Hobart', # jurisdiction name # paths to various source directories 'data_src_path': 'data/tasmania/hobart_tsunami_scenario_2009/anuga', 'arcgis_src_path': 'data/tasmania/hobart_tsunami_scenario_2009/ArcGIS', 'proj_src_path': 'sandpits/kvanputten/ANUGA/anuga_work/production/hobart_2009', # paths to destination directories (under 'jurisdiction' root) 'data_dst_path': 'data/tasmania/hobart_tsunami_scenario_2009/anuga', 'proj_dst_path': 'project', 'arcgis_dst_path': 'ArcGIS', # copy or create whole directories 'make_dst_dirs': ['gauges', 'outputs', 'topographies/elevation_fix', 'meshes'], 'copy_data_dirs': ['boundaries', 'polygons'], # copy 'data' files or directories 'copy_data_files': ['gauges/schema.ini', 'gauges/tsunamipointsMGA.csv', 'topographies/hobart_combined_elevation.pts', 'topographies/hobart_combined_elevation.txt', 'topographies/elevation_fix/topo_grid_notsw.pts', 'topographies/elevation_fix/topo_grid_notsw.dem', 'topographies/elevation_fix/topo_grid_notsw.asc', 'topographies/elevation_fix/topo_grid_notsw.prj'], # copy 'project' files or directories 'copy_proj_files': ['build_elevation.py', 'build_urs_boundary.py', 'combine_gauges.py', 'export_results_max.py', 'file_length.py', 'get_runup_v2.py', 'get_timeseries.py', 'project.py', 'run_model.py', 'setup_model.py'], # copy 'arcgis' files or directories 'copy_arc_files': [] } batemans_bay_data = \ {'jurisdiction': 'BatemansBay', # jurisdiction name # paths to various source directories 'data_src_path': 'data/new_south_wales/batemans_bay_tsunami_scenario_2009/anuga', 'arcgis_src_path': 'data/new_south_wales/batemans_bay_tsunami_scenario_2009/ArcGIS', 'proj_src_path': 'sandpits/jgriffin/ANUGA/anuga_work/production/new_south_wales/batemans_bay', # paths to destination directories (under 'jurisdiction' root) 'data_dst_path': 'data/new_south_wales/batemans_bay_tsunami_scenario_2009/anuga', 'proj_dst_path': 'project', 'arcgis_dst_path': 'ArcGIS', # copy or create whole directories 'make_dst_dirs': ['outputs', 'topographies', 'meshes'], 'copy_data_dirs': ['boundaries', 'polygons', 'gauges'], # copy 'data' files or directories 'copy_data_files': ['topographies/1a.asc', 'topographies/1b.asc', 'topographies/2b.asc', 'topographies/2a_3.asc', 'topographies/3b.asc', 'topographies/3a.asc', 'topographies/4a_2.asc', 'topographies/4b.asc', 'topographies/off1.asc', 'topographies/off2.asc', 'topographies/off3.asc', 'topographies/bbhd.asc', 'topographies/sd100031996_p.asc', 'topographies/sd100031996_p2.asc', 'topographies/sd100031996_p3.asc', 'topographies/sd100031996_p4.asc', 'topographies/1a.prj', 'topographies/1b.prj', 'topographies/2b.prj', 'topographies/2a_3.prj', 'topographies/3b.prj', 'topographies/3a.prj', 'topographies/4a_2.prj', 'topographies/4b.prj', 'topographies/off1.prj', 'topographies/off2.prj', 'topographies/off3.prj', 'topographies/bbhd.prj', 'topographies/sd100031996_p.prj', 'topographies/sd100031996_p2.prj', 'topographies/sd100031996_p3.prj', 'topographies/sd100031996_p4.prj', 'topographies/SD100031996_jgriffin_clip.csv', 'topographies/tomaga_offshore_AHD_MGA_1997.csv', 'topographies/Batemans_BBHD_MGA_1995.csv', 'topographies/moruya_AHD_MGA_2000.csv', # these files are generated by build_elevation.py, but included as a courtesy (if space permits) 'topographies/batemans_bay_combined_elevation.pts', 'topographies/batemans_bay_combined_elevation.txt'], # copy 'project' files or directories #'copy_proj_files': ['project.py', 'run_model_250m.py', 'run_model.py', 'setup_model_250m.py', # 'setup_model.py', 'build_elevation_250m.py', 'build_elevation.py', # 'build_urs_boundary.py', 'export_results_max.py', 'extract_from_csv.py', # 'file_length.py', 'get_runup.py', ], 'copy_proj_files': ['project.py', 'run_model_250m.py', 'setup_model_250m.py', 'setup_model.py', 'build_elevation_250m.py', 'build_elevation.py', 'build_urs_boundary.py', 'export_results_max.py', 'extract_from_csv.py', 'file_length.py', 'get_runup.py', ], # copy 'arcgis' files or directories 'copy_arc_files': [] } gosford_data = \ {'jurisdiction': 'Gosford', # jurisdiction name # paths to various source directories 'data_src_path': 'data/new_south_wales/gosford_tsunami_scenario_2009/anuga', 'arcgis_src_path': 'data/new_south_wales/gosford_tsunami_scenario_2009/ArcGIS', 'proj_src_path': 'sandpits/jgriffin/ANUGA/anuga_work/production/new_south_wales/gosford', # paths to destination directories (under 'jurisdiction' root) 'data_dst_path': 'data/new_south_wales/gosford_tsunami_scenario_2009/anuga', 'proj_dst_path': 'project', 'arcgis_dst_path': 'ArcGIS', # copy or create whole directories 'make_dst_dirs': ['outputs', 'topographies/original', 'polygons', 'gauges', 'meshes'], 'copy_data_dirs': ['boundaries'], # copy 'data' files or directories 'copy_data_files': ['topographies/original/e151_s34_clip2.asc', 'topographies/original/e151_s34_clip2.prj', 'topographies/original/aus197_topo2raster_clip2.asc', 'topographies/original/aus197_topo2raster_clip2.prj', 'topographies/original/hydro_topo2raster_clip2.asc', 'topographies/original/hydro_topo2raster_clip2.prj', 'topographies/original/file_list.csv', 'topographies/original/G3346294.txt', 'topographies/original/G3346296.txt', 'topographies/original/G3366284.txt', 'topographies/original/G3366286.txt', 'topographies/original/G3366288.txt', 'topographies/original/G3366290.txt', 'topographies/original/G3366294.txt', 'topographies/original/G3366296.txt', 'topographies/original/G3366298.txt', 'topographies/original/G3386284.txt', 'topographies/original/G3386286.txt', 'topographies/original/G3386288.txt', 'topographies/original/G3386290.txt', 'topographies/original/G3386292.txt', 'topographies/original/G3386294.txt', 'topographies/original/G3386296.txt', 'topographies/original/G3386298.txt', 'topographies/original/G3386300.txt', 'topographies/original/G3406284.txt', 'topographies/original/G3406286.txt', 'topographies/original/G3406288.txt', 'topographies/original/G3406290.txt', 'topographies/original/G3406292.txt', 'topographies/original/G3406294.txt', 'topographies/original/G3406296.txt', 'topographies/original/G3406298.txt', 'topographies/original/G3406300.txt', 'topographies/original/G3426284.txt', 'topographies/original/G3426286.txt', 'topographies/original/G3426288.txt', 'topographies/original/G3426290.txt', 'topographies/original/G3426292.txt', 'topographies/original/G3426294.txt', 'topographies/original/G3426296.txt', 'topographies/original/G3426298.txt', 'topographies/original/G3426300.txt', 'topographies/original/G3426302.txt', 'topographies/original/G3426304.txt', 'topographies/original/G3446284.txt', 'topographies/original/G3446286.txt', 'topographies/original/G3446288.txt', 'topographies/original/G3446290.txt', 'topographies/original/G3446292.txt', 'topographies/original/G3446294.txt', 'topographies/original/G3446296.txt', 'topographies/original/G3446298.txt', 'topographies/original/G3446300.txt', 'topographies/original/G3446302.txt', 'topographies/original/G3446304.txt', 'topographies/original/G3446306.txt', 'topographies/original/G3466286.txt', 'topographies/original/G3466288.txt', 'topographies/original/G3466290.txt', 'topographies/original/G3466292.txt', 'topographies/original/G3466294.txt', 'topographies/original/G3466296.txt', 'topographies/original/G3466298.txt', 'topographies/original/G3466300.txt', 'topographies/original/G3466302.txt', 'topographies/original/G3466304.txt', 'topographies/original/G3466306.txt', 'topographies/original/G3486288.txt', 'topographies/original/G3486290.txt', 'topographies/original/G3486292.txt', 'topographies/original/G3486294.txt', 'topographies/original/G3486296.txt', 'topographies/original/G3486298.txt', 'topographies/original/G3486300.txt', 'topographies/original/G3486302.txt', 'topographies/original/G3486304.txt', 'topographies/original/G3486306.txt', 'topographies/original/G3506288.txt', 'topographies/original/G3506290.txt', 'topographies/original/G3506292.txt', 'topographies/original/G3506294.txt', 'topographies/original/G3506296.txt', 'topographies/original/G3506298.txt', 'topographies/original/G3506300.txt', 'topographies/original/G3506302.txt', 'topographies/original/G3506304.txt', 'topographies/original/G3506306.txt', 'topographies/original/G3526288.txt', 'topographies/original/G3526290.txt', 'topographies/original/G3526292.txt', 'topographies/original/G3526294.txt', 'topographies/original/G3526296.txt', 'topographies/original/G3526298.txt', 'topographies/original/G3526300.txt', 'topographies/original/G3526302.txt', 'topographies/original/G3526304.txt', 'topographies/original/G3546290.txt', 'topographies/original/G3546292.txt', 'topographies/original/G3546294.txt', 'topographies/original/G3546296.txt', 'topographies/original/G3546298.txt', 'topographies/original/G3546300.txt', 'topographies/original/G3546302.txt', 'topographies/original/G3546304.txt', 'topographies/original/G3566296.txt', 'topographies/original/G3566298.txt', 'topographies/original/G3566300.txt', 'topographies/original/G3566302.txt', 'topographies/original/G3566304.txt', 'topographies/original/G3586302.txt', 'topographies/original/estuaries.txt', 'topographies/original/ENT5C1S08_03_AHD_prepared.txt', 'topographies/original/ENT5C1S09_03_AHD_prepared.txt', 'topographies/original/ENT5C1S10_03_AHD_prepared.txt', 'topographies/original/hydro_clip_neg.txt', 'polygons/bounding_polygon.csv', 'polygons/aoi_umina_large.csv', 'polygons/aoi_terrigal.csv', 'polygons/aos_umina_large.csv', 'polygons/aos_terrigal.csv', 'polygons/initial_conditions.csv', 'polygons/images.csv', 'gauges/gauges.csv'], # copy 'project' files or directories 'copy_proj_files': ['build_urs_boundary.py', 'export_results_max.py', 'file_length.py', 'get_runup.py', 'prepare_data_AUS197.py', 'prepare_data_hydro.py', 'project.py', 'run_model.py', 'build_elevation.py', 'extract_from_csv.py', 'get_timeseries.py', 'prepare_data_ENT5CIS.py', 'prepare_data.py', 'setup_model.py'], # copy 'arcgis' files or directories 'copy_arc_files': [] } gold_coast_data = \ {'jurisdiction': 'GoldCoast', # jurisdiction name # paths to various source directories 'data_src_path': 'data/queensland/gold_coast_tsunami_scenario_2009/anuga', 'arcgis_src_path': 'data/queensland/gold_coast_tsunami_scenario_2009/ArcGIS', 'proj_src_path': 'sandpits/lfountain/anuga_work/production/gold_coast_2009', # paths to destination directories (under 'jurisdiction' root) 'data_dst_path': 'data/queensland/gold_coast_tsunami_scenario_2009/anuga', 'proj_dst_path': 'project', 'arcgis_dst_path': 'ArcGIS', # copy or create whole directories 'make_dst_dirs': ['outputs', 'meshes', 'polygons', 'topographies', 'gauges'], 'copy_data_dirs': ['boundaries'], # copy 'data' files or directories 'copy_data_files': ['outputs/Event1_HAT', 'outputs/Event1_MSL', 'outputs/Event2_HAT', 'outputs/Event2_MSL', 'outputs/Event3_HAT', 'outputs/Event3_MSL', 'polygons/bounding_polygon_sml.csv', 'polygons/initial_conditions_sml.csv', 'polygons/images.csv', 'topographies/gold_coast_combined_elevation_250m.pts', 'topographies/grid250m_all_pro.asc', 'topographies/grid250m_all_pro.prj'], # copy 'project' files or directories 'copy_proj_files': ['build_elevation.py', 'export_results_max.py', 'file_length.py', 'get_runup.py', 'project.py', 'run_model.py', 'setup_model.py', 'build_urs_boundary.py', 'combine_gauges.py', 'get_timeseries.py', 'run_multiple_events.py' ], # copy 'arcgis' files or directories 'copy_arc_files': ['gold_coast.mxd'] } # dictionary mapping lower-case jurisdiction name to jurisdiction data dictionary source_jurisdiction_path = {'hobart': hobart_data, 'batemansbay': batemans_bay_data, 'gosford': gosford_data, 'goldcoast': gold_coast_data } ###### # Routines to automate the script data above. ###### def log(msg=''): print(msg) def dir_copy(src, dst): cmd = 'cp -R %s %s' % (src, dst) log('Doing: %s' % cmd) fd = os.popen(cmd) fd.close() def copy_file_or_dir(src, dst): '''Copy a file or complete directory.''' # could be a file or directory being copied try: shutil.copyfile(src, dst) except IOError, e: if 'Is a directory' in str(e): shutil.copytree(src, dst) else: log('*' *72) log('* %s' % str(e)) log('*' *72) def update_staging(jurisdiction): # create a list of jurisdiction names jurisdiction_names = [] for k in source_jurisdiction_path: jurisdiction_names.append(source_jurisdiction_path[k]['jurisdiction']) # get ready j_dict = source_jurisdiction_path[jurisdiction] j_name = j_dict['jurisdiction'] data_src_path = os.path.join(main_path, j_dict['data_src_path']) data_dst_path = os.path.join(os.getcwd(), j_name, j_dict['data_dst_path']) proj_src_path = os.path.join(main_path, j_dict['proj_src_path']) proj_dst_path = os.path.join(os.getcwd(), j_name, j_dict['proj_dst_path']) arcgis_src_path = os.path.join(main_path, j_dict['arcgis_src_path']) arcgis_dst_path = os.path.join(os.getcwd(), j_name, j_dict['arcgis_dst_path']) # create new output directory, delete old if there if os.path.exists(j_name): log('Deleting existing staging directory: %s' % j_name) shutil.rmtree(j_name) log('Creating staging directory: %s' % j_name) os.makedirs(j_name) # create required directories for dir in j_dict['make_dst_dirs']: new_dir = os.path.join(data_dst_path, dir) log('Creating directory: %s' % dir) os.makedirs(new_dir) # copy required full directories for copy_dir in j_dict['copy_data_dirs']: src_dir = os.path.join(data_src_path, copy_dir) dst_dir = os.path.join(data_dst_path, copy_dir) log('Copying directory: %s' % copy_dir) copy_file_or_dir(src_dir, dst_dir) # copy required data files for copy_file in j_dict['copy_data_files']: src_file = os.path.join(data_src_path, copy_file) new_file = os.path.join(data_dst_path, copy_file) log('Copying: %s' % copy_file) copy_file_or_dir(src_file, new_file) # copy required project files log('Creating directory: %s' % proj_dst_path) os.makedirs(proj_dst_path) for copy_file in j_dict['copy_proj_files']: src_file = os.path.join(proj_src_path, copy_file) new_file = os.path.join(proj_dst_path, copy_file) log('Copying: %s' % copy_file) copy_file_or_dir(src_file, new_file) # copy required ArcGIS files log('Creating directory: %s' % arcgis_dst_path) os.makedirs(arcgis_dst_path) for copy_file in j_dict['copy_arc_files']: src_file = os.path.join(arcgis_src_path, copy_file) new_file = os.path.join(arcgis_dst_path, copy_file) log('Copying: %s' % copy_file) copy_file_or_dir(src_file, new_file) # now copy jurisdiction-specific DVD files src_dir = 'extra_files/%s/*' % j_name dst_dir = j_name dir_copy(src_dir, dst_dir) # copy the extra_files and special directories src_dir = 'extra_files/browser_files' dst_dir = j_name dir_copy(src_dir, dst_dir) src_dir = 'extra_files/documents' dir_copy(src_dir, dst_dir) src_file = 'extra_files/autorun.inf' dir_copy(src_file, dst_dir) src_dir = 'extra_files/.cache' dir_copy(src_dir, dst_dir) # get size of the staging directory cmd = 'du -sh %s' % j_name fd = os.popen(cmd, 'r') res = fd.read() fd.close() (res, _) = res.split('\t', 1) log('Staging directory %s has size %s' % (j_name, res)) def usage(): print('usage: update.py ') print('where is one of the jurisdiction staging directories.') if len(sys.argv) != 2: usage() sys.exit(10) jurisdiction = sys.argv[1].lower() # remove any trailing '/' - from TAB completion if jurisdiction.endswith('/'): jurisdiction = jurisdiction[:-1] start_time = time.time() update_staging(jurisdiction) stop_time = time.time() elapsed_time = stop_time - start_time log('Elapsed time is %.1fs' % elapsed_time)