source: trunk/DVD_images/update_DVD_images.py @ 7852

Last change on this file since 7852 was 7639, checked in by griffin, 15 years ago

removed error checking print statements
  • Property svn:executable set to *
File size: 21.0 KB
Line 
1#!/usr/bin/env python
2
3'''
4Program to update a DVD jurisdiction 'anuga' directory from the
5original source directory.
6
7usage: update.py <source>
8
9where <source> is one of:
10                  Hobart
11                  BatemansBay
12                  Gosford
13                  GoldCoast
14ie, the name of one of the DVD jurisdiction staging directories
15(case-insensitive).
16'''
17
18import os
19import sys
20import shutil
21import time
22
23
24# common base path to all data and project files
25main_path = '/nas/gemd/georisk_models/inundation'
26
27# structures holding data for each jurisdiction
28#   data_src_path  : sub-path to data directory
29#   proj_src_path  : sub-path to project files
30#   data_dst_path  : where to copy data files to
31#   proj_dst_path  : where to copy project files to
32#   copy_data_dirs : data directories to completely copy
33#   make_dst_dirs  : make these data directories (may be left empty)
34#   copy_data_files: individual data files to copy to <data_dst_path>
35#   copy_proj_files: individual project files to copy to <proj_dst_path>
36hobart_data = \
37{'jurisdiction':    'Hobart',           # jurisdiction name
38
39 # paths to various source directories
40 'data_src_path':   'data/tasmania/hobart_tsunami_scenario_2009/anuga',
41 'arcgis_src_path': 'data/tasmania/hobart_tsunami_scenario_2009/ArcGIS',
42 'viz_src_path':    'data/tasmania/hobart_tsunami_scenario_2009/visualisations',
43 'proj_src_path':   'sandpits/kvanputten/ANUGA/anuga_work/production/hobart_2009/For_DVD',
44 'report_src_path': 'data/four_east_coast_communities_reporting/Professional_opinions_final',
45
46 # paths to destination directories (under 'jurisdiction' root)
47 'data_dst_path':   'data/tasmania/hobart_tsunami_scenario_2009/anuga',
48 'proj_dst_path':   'project',
49 'viz_dst_path':    'documents',
50 'arcgis_dst_path': 'data/tasmania/hobart_tsunami_scenario_2009/ArcGIS',
51 'report_dst_path': 'documents',
52 
53 # copy or create whole directories
54 'make_dst_dirs':   ['outputs', 'topographies'],
55 'copy_data_dirs':  ['gauges','boundaries', 'polygons'],
56
57 # copy 'data' files or directories
58 'copy_data_files': ['outputs/Event1_HAT',
59                     'outputs/Event1_MSL',
60                     'outputs/Event2_HAT',
61                     'outputs/Event2_MSL',
62                     'outputs/Event3_MSL',
63                     'outputs/Event3_HAT',
64                     'outputs/elevation.gdb',
65                     'topographies/hobart_combined_elevation.pts',
66                     'topographies/hobart_combined_elevation.txt',
67                     'topographies/Metadata.pdf'],
68
69 # copy 'project' files or directories
70 'copy_proj_files': ['build_elevation.py', 'project.py', 'run_model.py', 
71                     'export_results_max.py',  'get_timeseries.py',
72                     'setup_model.py', 'build_urs_boundary.py',
73                     'file_length.py'],
74
75 # copy 'visualisations' files or directories
76 'copy_viz_files': ['Figures'], 
77
78 # copy 'arcgis' files or directories
79 'copy_arc_files':  ['Hobart.gdb','Hobart.mxd','layer files'],
80
81 # copy 'report' file
82 'copy_report_files':  ['professionalopinion_2009_07_TAS_inundation_modelling.pdf']
83}
84
85batemans_bay_data = \
86{'jurisdiction':    'BatemansBay',      # jurisdiction name
87
88 # paths to various source directories
89 'data_src_path':   'data/new_south_wales/batemans_bay_tsunami_scenario_2009/anuga',
90 'arcgis_src_path': 'data/new_south_wales/batemans_bay_tsunami_scenario_2009/ArcGIS',
91 'viz_src_path':    'data/new_south_wales/batemans_bay_tsunami_scenario_2009/visualisations',
92 'proj_src_path':   'sandpits/jgriffin/ANUGA/anuga_work/production/new_south_wales/batemans_bay',
93 'report_src_path': 'data/four_east_coast_communities_reporting/Professional_opinions_final',
94
95 # paths to destination directories (under 'jurisdiction' root)
96 'data_dst_path':   'data/new_south_wales/batemans_bay_tsunami_scenario_2009/anuga',
97 'proj_dst_path':   'project',
98 'viz_dst_path':    'documents',
99 'arcgis_dst_path': 'data/new_south_wales/batemans_bay_tsunami_scenario_2009/ArcGIS',
100 'report_dst_path': 'documents',
101
102 # copy or create whole directories
103 'make_dst_dirs':   ['boundaries', 'meshes', 'outputs', 'topographies'],
104 'copy_data_dirs':  ['polygons', 'gauges'],
105
106 # copy 'data' files or directories
107 'copy_data_files': ['topographies/batemans_bay_combined_elevation.pts',
108                     'topographies/Data_lineage.doc',
109                     'outputs/Event1_HAT',
110                     'outputs/Event1_MSL',
111                     'outputs/Event2_HAT',
112                     'outputs/Event2_MSL',
113                     'outputs/Event3_MSL',
114                     'outputs/Puysegur_200yr',
115                     'outputs/Puysegur_500yr',
116                     'outputs/Puysegur_1000yr',
117                     'outputs/Puysegur_5000yr',
118                     'outputs/New_Hebrides_200yr',
119                     'outputs/New_Hebrides_500yr',
120                     'outputs/New_Hebrides_1000yr',
121                     'outputs/New_Hebrides_2000yr',
122                     'outputs/New_Hebrides_5000yr',
123                     'outputs/elevation',
124                     'boundaries/51077',
125                     'boundaries/51204',
126                     'boundaries/51292',
127                     'boundaries/51347',
128                     'boundaries/51378',
129                     'boundaries/51424',
130                     'boundaries/51436',
131                     'boundaries/51445',
132                     'boundaries/58115',
133                     'boundaries/58129',
134                     'boundaries/58226',
135                     'boundaries/58272',
136                     'boundaries/58284',
137                     'boundaries/58286',
138                     'boundaries/58346',
139                     'boundaries/58368',
140                     'boundaries/boundaries.gdb',
141                     'boundaries/landward_boundary.csv',
142                     'boundaries/urs_order.csv'
143                     ],
144
145 # copy 'visualisations' files or directories
146 'copy_viz_files': ['Figures'], 
147
148 # copy 'project' files or directories
149 'copy_proj_files': ['project.py', 'run_model.py',
150                     'setup_model.py', 'build_elevation.py',
151                     'export_results_max.py',
152                     'file_length.py', 'build_urs_boundary.py'],
153
154 # copy 'arcgis' files or directories
155 'copy_arc_files':  ['BB_cbd_figure_template.mxd'],
156
157 # copy 'report' file
158 'copy_report_files':  ['professionalopinion_2009_09_NSW_inundation_modelling.pdf']
159
160}
161
162gosford_data = \
163{'jurisdiction':    'Gosford',          # jurisdiction name
164
165 # paths to various source directories
166 'data_src_path':   'data/new_south_wales/gosford_tsunami_scenario_2009/anuga',
167 'arcgis_src_path': 'data/new_south_wales/gosford_tsunami_scenario_2009/ArcGIS',
168 'viz_src_path':    'data/new_south_wales/gosford_tsunami_scenario_2009/visualisations',
169 'proj_src_path':   'sandpits/jgriffin/ANUGA/anuga_work/production/new_south_wales/gosford',
170 'report_src_path': 'data/four_east_coast_communities_reporting/Professional_opinions_final',
171 
172 # paths to destination directories (under 'jurisdiction' root)
173 'data_dst_path':   'data/new_south_wales/gosford_tsunami_scenario_2009/anuga',
174 'proj_dst_path':   'project',
175 'viz_dst_path':    'documents',
176 'arcgis_dst_path': 'data/new_south_wales/gosford_tsunami_scenario_2009/ArcGIS',
177 'report_dst_path': 'documents',
178 
179 # copy or create whole directories
180 'make_dst_dirs':   ['boundaries', 'meshes', 'outputs', 'topographies'],
181 'copy_data_dirs':  ['polygons',  'gauges'],
182
183 # copy 'data' files or directories
184 'copy_data_files': ['topographies/gosford_combined_elevation.pts',
185                     'topographies/Data_lineage.doc',
186                     'outputs/Event1_HAT',
187                     'outputs/Event1_MSL',
188                     'outputs/Event2_HAT',
189                     'outputs/Event2_MSL',
190                     'outputs/Event3_MSL',
191                     'outputs/Puysegur_200yr',
192                     'outputs/Puysegur_500yr',
193                     'outputs/Puysegur_1000yr',
194                     'outputs/Puysegur_5000yr',
195                     'outputs/elevation',
196                     'boundaries/51436',
197                     'boundaries/58025',
198                     'boundaries/58113',
199                     'boundaries/58187',
200                     'boundaries/58242',
201                     'boundaries/58284',
202                     'boundaries/58349',
203                     'boundaries/boundaries.gdb',
204                     'boundaries/landward_boundary.csv',
205                     'boundaries/urs_order.csv'
206                    ],
207
208 # copy 'visualisations' files or directories
209 'copy_viz_files': ['Figures'], 
210
211 # copy 'project' files or directories
212 'copy_proj_files': ['export_results_max.py', 'file_length.py',
213                     'project.py', 'run_model.py', 'build_elevation.py',
214                     'get_timeseries.py', 'setup_model.py',
215                     'build_urs_boundary.py'
216                    ],
217
218 # copy 'arcgis' files or directories
219 'copy_arc_files':  ['G_umina_figure_template.mxd',
220                     'avoca_bookmark.dat',
221                     'copacabana_bookmark.dat',
222                     'terrigal_bookmark.dat',
223                     'umina_bookmark.dat'],
224
225 # copy 'report' file
226 'copy_report_files':  ['professionalopinion_2009_09_NSW_inundation_modelling.pdf']
227}
228
229gold_coast_data = \
230{'jurisdiction':    'GoldCoast',        # jurisdiction name
231
232 # paths to various source directories
233 'data_src_path':   'data/queensland/gold_coast_tsunami_scenario_2009/anuga',
234 'arcgis_src_path': 'data/queensland/gold_coast_tsunami_scenario_2009/ArcGIS',
235 'viz_src_path':    'data/queensland/gold_coast_tsunami_scenario_2009/visualisations',
236 'proj_src_path':   'sandpits/lfountain/anuga_work/production/gold_coast_2009/For_DVD',
237 'report_src_path': 'data/four_east_coast_communities_reporting/Professional_opinions_final',
238 
239 # paths to destination directories (under 'jurisdiction' root)
240 'data_dst_path':   'data/queensland/gold_coast_tsunami_scenario_2009/anuga',
241 'proj_dst_path':   'project',
242 'viz_dst_path':    'documents',
243 'arcgis_dst_path': 'data/queensland/gold_coast_tsunami_scenario_2009/ArcGIS',
244 'report_dst_path': 'documents',
245
246 # copy or create whole directories
247 'make_dst_dirs':   ['outputs', 'topographies', 'boundaries'],
248 'copy_data_dirs':  ['polygons', 'gauges'],
249 
250 # copy 'data' files or directories
251 'copy_data_files': ['outputs/Event1_HAT', 'outputs/Event1_MSL',
252                     'outputs/Event2_HAT', 'outputs/Event2_MSL',
253                     'outputs/Event3_HAT', 'outputs/Event3_MSL',
254                     'boundaries/50863', 'boundaries/50994',
255                     'boundaries/51253', 'boundaries/51392',
256                     'boundaries/51423', 'boundaries/51469',
257                     'boundaries/62833','boundaries/events_chosen.xls',
258                     'boundaries/landward_boundary.csv',
259                     'boundaries/urs_order.csv',
260                     'topographies/gold_coast_combined_elevation.pts'
261                    ],
262
263 # copy 'visualisations' files or directories
264 'copy_viz_files': ['Figures'], 
265
266 # copy 'project' files or directories
267 'copy_proj_files': ['build_elevation.py', 'export_results_max.py', 'file_length.py',
268                     'project.py', 'run_model.py', 'setup_model.py',
269                     'get_timeseries.py', 'build_urs_boundary.py'
270                    ],
271
272 # copy 'arcgis' files or directories
273 'copy_arc_files':  ['gold_coast.mxd', 'gold_coast.gdb'],
274
275 # copy 'report' file
276 'copy_report_files':  ['professionalopinion_2009_08_QLD_inundation_modelling.pdf']
277}
278
279geraldton_data = \
280{'jurisdiction':    'geraldton',                # jurisdiction name
281
282 # paths to various source directories
283 'data_src_path':   'data/western_australia/geraldton_tsunami_scenario/anuga',
284 'proj_src_path':   'sandpits/jgriffin/ANUGA/anuga_work/production/wa/geraldton_2009',
285 'report_src_path': 'data/western_australia/documents', #need to update this
286 'arcgis_src_path': 'data/western_australia/geraldton_tsunami_scenario/anuga',
287 'observation_src_path': '/nas/gemd/georisk/tsunami/WA_tsunami_validation/observations',
288 'viz_src_path':    'data/western_australia/geraldton_tsunami_scenario/anuga',
289 
290 # paths to destination directories (under 'jurisdiction' root)
291 'data_dst_path':   'data/western_australia/geraldton_tsunami_scenario/anuga',
292 'proj_dst_path':   'project',
293 'report_dst_path': 'documents',
294 'viz_dst_path':    'figures',
295 'arcgis_dst_path': 'gis',
296 'observation_dst_path': 'field_observations',
297 # copy or create whole directories
298 'make_dst_dirs':   [ 'boundaries','meshes', 'outputs', 'topographies','polygons',  'gauges'],
299 'copy_data_dirs':  [],
300
301 # copy 'data' files or directories
302 'copy_data_files': ['topographies/geraldton_combined_elevation.pts',
303                     #'topographies/Data_lineage.doc',
304                     'outputs/2004_tsunami_validation',
305                     'gauges/geraldton_validation_gauges.csv',
306                     'boundaries/wa04_val',
307                     'boundaries/landward_boundary.csv',
308                     'boundaries/urs_order.csv',
309                     'boundaries/URSGA_gauges_All_WA.csv',
310                     #'boundaries/URSGA_gauges_WA',
311                     'polygons/buffer_20m.csv',
312                     'polygons/ocean_initial_condition.csv',
313                     'polygons/land_initial_condition.csv',
314                     'polygons/poly_all.csv',
315                     'polygons/harbour.csv',
316                     'polygons/CBD_500m.csv',
317                     'polygons/CBD_1km.csv', 
318                     'polygons/island_wallabi_poly2.csv',
319                     'polygons/island_dingiville_poly.csv', 
320                     'polygons/island_pelsaert_poly.csv',
321                     'polygons/land_initial_condition.csv',
322                     'polygons/ocean_initial_condition.csv',
323                     'meshes/CBD.csv',
324                     'meshes/wall.csv'
325                    ],
326
327 # copy 'visualisations' files or directories
328 'copy_viz_files': [], 
329
330 # copy 'project' files or directories
331 'copy_proj_files': ['export_results_max.py', 'file_length.py',
332                     'project.py', 'run_model.py', 'build_elevation.py',
333                     'get_timeseries.py', 'setup_model.py',
334                     'build_urs_boundary.py'
335                    ],
336
337 # copy 'arcgis' files or directories
338 'copy_arc_files':  [],
339
340 # copy 'observation' files or directories
341 'copy_observation_files':  ['PostTsunamiSurveyData_RunUp.xls'],
342
343 # copy 'report' file
344 'copy_report_files': ['Horspool_et_al_GA_record_2009_XX.pdf']
345}
346
347
348# dictionary mapping lower-case jurisdiction name to jurisdiction data dictionary
349source_jurisdiction_path = {'hobart': hobart_data,
350                            'batemansbay': batemans_bay_data,
351                            'gosford': gosford_data,
352                            'goldcoast': gold_coast_data ,
353                            'geraldton': geraldton_data
354                           }
355
356######
357# Routines to automate the script data above.
358######
359
360def log(msg=''):
361    print(msg)
362
363
364def dir_copy(src, dst):
365    cmd = 'cp -R %s %s' % (src, dst)
366    log('Doing: %s' % cmd)
367    fd = os.popen(cmd)
368    fd.close()
369
370def copy_file_or_dir(src, dst):
371    '''Copy a file or complete directory.'''
372
373##    os.system('cp -R %s %s' % (src, dst))
374
375    #print('copy_file_or_dir: %s -> %s' % (src, dst))
376   
377    # could be a file or directory being copied
378    try:
379        shutil.copyfile(src, dst)
380    except IOError, e:
381        #print 'str(e)=%s' % str(e)
382        if 'Is a directory' in str(e):
383            shutil.copytree(src, dst)
384        else:
385            log('*' *72)
386            log('* %s' % str(e))
387            log('*' *72)
388
389def update_staging(jurisdiction):
390    # create a list of jurisdiction names
391    jurisdiction_names = []
392    for k in source_jurisdiction_path:
393        jurisdiction_names.append(source_jurisdiction_path[k]['jurisdiction'])
394
395    # get ready
396    j_dict = source_jurisdiction_path[jurisdiction]
397    j_name = j_dict['jurisdiction']
398    data_src_path = os.path.join(main_path, j_dict['data_src_path'])
399    data_dst_path = os.path.join(os.getcwd(), j_name, j_dict['data_dst_path'])
400    proj_src_path = os.path.join(main_path, j_dict['proj_src_path'])
401    proj_dst_path = os.path.join(os.getcwd(), j_name, j_dict['proj_dst_path'])
402    viz_src_path = os.path.join(main_path, j_dict['viz_src_path'])
403    viz_dst_path = os.path.join(os.getcwd(), j_name, j_dict['viz_dst_path'])
404    arcgis_src_path = os.path.join(main_path, j_dict['arcgis_src_path'])
405    arcgis_dst_path = os.path.join(os.getcwd(), j_name, j_dict['arcgis_dst_path'])
406    observation_src_path = j_dict['observation_src_path']
407    observation_dst_path = os.path.join(os.getcwd(), j_name, j_dict['observation_dst_path'])
408    report_src_path = os.path.join(main_path, j_dict['report_src_path'])
409    report_dst_path = os.path.join(os.getcwd(), j_name, j_dict['report_dst_path'])
410   
411    # tell where all stuff is coming from
412    log('Getting data from: %s' % data_src_path)
413    log('Getting project from: %s' % proj_src_path)
414    log('Getting ArcGIS from: %s' % arcgis_src_path)
415    log('Getting Observations from: %s' % observation_src_path)
416    log('Getting report from: %s' % report_src_path)
417    log('')
418
419    # create new output directory, delete old if there
420    if os.path.exists(j_name):
421        log('Deleting existing staging directory: %s' % j_name)
422        shutil.rmtree(j_name)
423    log('Creating staging directory: %s' % j_name)
424    os.makedirs(j_name)
425
426    # create required directories
427    for dir in j_dict['make_dst_dirs']:
428        new_dir = os.path.join(data_dst_path, dir)
429        log('Creating directory: %s' % dir)
430        os.makedirs(new_dir)
431
432    # copy required full directories
433    for copy_dir in j_dict['copy_data_dirs']:
434        src_dir = os.path.join(data_src_path, copy_dir)
435        dst_dir = os.path.join(data_dst_path, copy_dir)
436        log('Copying directory: %s' % copy_dir)
437        copy_file_or_dir(src_dir, dst_dir)
438
439    # copy required data files
440    for copy_file in j_dict['copy_data_files']:
441        src_file = os.path.join(data_src_path, copy_file)
442        new_file = os.path.join(data_dst_path, copy_file)
443        log('Copying: %s' % copy_file)
444        copy_file_or_dir(src_file, new_file)
445
446    # copy required visualisation files
447    log('Creating directory: %s' % viz_dst_path)
448    os.makedirs(viz_dst_path)
449    for copy_file in j_dict['copy_viz_files']:
450        src_file = os.path.join(viz_src_path, copy_file)
451        new_file = os.path.join(viz_dst_path, copy_file)
452        log('Copying: %s' % copy_file)
453        copy_file_or_dir(src_file, new_file)
454
455    # copy required project files
456    log('Creating directory: %s' % proj_dst_path)
457    os.makedirs(proj_dst_path)
458    for copy_file in j_dict['copy_proj_files']:
459        src_file = os.path.join(proj_src_path, copy_file)
460        new_file = os.path.join(proj_dst_path, copy_file)
461        log('Copying: %s' % copy_file)
462        copy_file_or_dir(src_file, new_file)
463
464    # copy required ArcGIS files
465    log('Creating directory: %s' % arcgis_dst_path)
466    os.makedirs(arcgis_dst_path)
467    for copy_file in j_dict['copy_arc_files']:
468        src_file = os.path.join(arcgis_src_path, copy_file)
469        new_file = os.path.join(arcgis_dst_path, copy_file)
470        log('Copying: %s' % copy_file)
471        copy_file_or_dir(src_file, new_file)
472
473     # copy required observational files
474    log('Creating directory: %s' % observation_dst_path)
475    os.makedirs(observation_dst_path)
476    for copy_file in j_dict['copy_observation_files']:
477        src_file = os.path.join(observation_src_path, copy_file)
478        new_file = os.path.join(observation_dst_path, copy_file)
479        log('Copying: %s' % copy_file)
480        copy_file_or_dir(src_file, new_file)
481
482    # copy report
483    log('Creating directory: %s' % report_dst_path)
484    for copy_file in j_dict['copy_report_files']:
485        src_file = os.path.join(report_src_path, copy_file)
486        new_file = os.path.join(report_dst_path, copy_file)
487        print 'src', src_file
488        print 'new', new_file
489        log('Copying: %s' % copy_file)
490        copy_file_or_dir(src_file, new_file)
491
492    # now copy jurisdiction-specific DVD files
493    src_dir = 'extra_files/%s/*' % j_name
494    dst_dir = j_name
495    dir_copy(src_dir, dst_dir)
496
497    # copy the extra_files and special directories
498    src_dir = 'extra_files/browser_files'
499    dst_dir = j_name
500    dir_copy(src_dir, dst_dir)
501    src_dir = 'extra_files/documents'
502    dir_copy(src_dir, dst_dir)
503    src_file = 'extra_files/autorun.inf'
504    dir_copy(src_file, dst_dir)
505    src_dir = 'extra_files/.cache'
506    dir_copy(src_dir, dst_dir)
507
508    # get size of the staging directory
509    cmd = 'du -sh %s' % j_name
510    fd = os.popen(cmd, 'r')
511    res = fd.read()
512    fd.close()
513    (res, _) = res.split('\t', 1)
514    log('Staging directory %s has size %s' % (j_name, res))
515
516def usage():
517    print('usage: update.py <source>')
518    print('where <source> is one of the jurisdiction staging directories.')
519
520if sys.platform == 'win32':
521    print('Sorry, you can only run this script under Unix')
522    sys.exit(10)
523   
524if len(sys.argv) != 2:
525    usage()
526    sys.exit(10)
527
528jurisdiction = sys.argv[1].lower()
529# remove any trailing '/' - from TAB completion
530if jurisdiction.endswith('/'):
531    jurisdiction = jurisdiction[:-1]
532
533start_time = time.time()
534update_staging(jurisdiction)
535stop_time = time.time()
536elapsed_time = stop_time - start_time
537log('Elapsed time is %.1fs' % elapsed_time)
538print 'completed'
Note: See TracBrowser for help on using the repository browser.