source: anuga_core/source/anuga/utilities/system_tools.py @ 7340

Last change on this file since 7340 was 7317, checked in by rwilson, 16 years ago

Replaced 'print' statements with log.critical() calls.

File size: 19.1 KB
Line 
1"""Implementation of tools to do with system administration made as platform independent as possible.
2
3
4"""
5
6import sys
7import os
8import string
9import urllib
10import urllib2
11import getpass
12import tarfile
13try:
14    import hashlib
15except ImportError:
16    import md5 as hashlib
17
18import anuga.utilities.log as log
19
20
21def log_to_file(filename, s, verbose=False):
22    """Log string to file name
23    """
24
25    fid = open(filename, 'a')
26    if verbose: print s
27    fid.write(s + '\n')
28    fid.close()
29
30
31def get_user_name():
32    """Get user name provide by operating system
33    """
34
35    if sys.platform == 'win32':
36        #user = os.getenv('USERPROFILE')
37        user = os.getenv('USERNAME')
38    else:
39        user = os.getenv('LOGNAME')
40
41
42    return user   
43
44def get_host_name():
45    """Get host name provide by operating system
46    """
47
48    if sys.platform == 'win32':
49        host = os.getenv('COMPUTERNAME')
50    else:
51        host = os.uname()[1]
52
53
54    return host   
55
56def get_revision_number():
57    """Get the version number of this repository copy.
58
59    Try getting data from stored_version_info.py first, otherwise
60    try using SubWCRev.exe (Windows) or svnversion (linux), otherwise
61    try reading file .svn/entries for version information, otherwise
62    throw an exception.
63
64    NOTE: This requires that the command svn is on the system PATH
65    (simply aliasing svn to the binary will not work)
66    """
67
68    def get_revision_from_svn_entries():
69        '''Get a subversion revision number from the .svn/entires file.'''
70
71        msg = '''
72No version info stored and command 'svn' is not recognised on the system PATH.
73
74If ANUGA has been installed from a distribution e.g. as obtained from SourceForge,
75the version info should be available in the automatically generated file
76'stored_version_info.py' in the anuga root directory.
77
78If run from a Subversion sandpit, ANUGA will try to obtain the version info by
79using the command 'svn info'.  In this case, make sure the command line client
80'svn' is accessible on the system path.  Simply aliasing 'svn' to the binary will
81not work.
82
83If you are using Windows, you have to install the file svn.exe which can be
84obtained from http://www.collab.net/downloads/subversion.
85
86Good luck!
87'''
88
89        try:
90            fd = open(os.path.join('.svn', 'entries'))
91        except:
92            raise Exception, msg
93
94        line = fd.readlines()[3]
95        fd.close()
96        try:
97            revision_number = int(line)
98        except:
99            msg = ".svn/entries, line 4 was '%s'?" % line.strip()
100            raise Exception, msg
101
102        return revision_number
103
104    def get_revision_from_svn_client():
105        '''Get a subversion revision number from an svn client.'''
106
107        if sys.platform[0:3] == 'win':
108            try:
109                fid = os.popen(r'C:\Program Files\TortoiseSVN\bin\SubWCRev.exe')
110            except:
111                return get_revision_from_svn_entries()
112            else:
113                version_info = fid.read()
114                if version_info == '':
115                    return get_revision_from_svn_entries()
116
117            # split revision number from data
118            for line in version_info.split('\n'):
119                if line.startswith('Updated to revision '):
120                    break
121
122            fields = line.split(' ')
123            msg = 'Keyword "Revision" was not found anywhere in text: %s' % version_info
124            assert fields[0].startswith('Updated'), msg
125
126            try:
127                revision_number = int(fields[3])
128            except:
129                msg = ("Revision number must be an integer. I got '%s' from "
130                       "'SubWCRev.exe'." % fields[3])
131                raise Exception, msg
132        else:                   # assume Linux
133            try:
134                fid = os.popen('svnversion -n . 2>/dev/null')
135            except:
136                return get_revision_from_svn_entries()
137            else:
138                version_info = fid.read()
139                if version_info == '':
140                    return get_revision_from_svn_entries()
141
142            # split revision number from data
143            if ':' in version_info:
144                (_, revision_number) =  version_info.split(':')
145            elif version_info.endswith('M'):
146                revision_number = version_info[:-1]
147            else:
148                revision_number = version_info
149
150            try:
151                revision_number = int(revision_number)
152            except:
153                msg = ("Revision number must be an integer. I got '%s' from "
154                       "'svn'." % version_info)
155                raise Exception, msg
156
157        return revision_number
158
159    # try to get revision information from stored_version_info.py
160    try:
161        from anuga.stored_version_info import version_info
162    except:
163        return get_revision_from_svn_client()
164
165    # split revision number from data
166    for line in version_info.split('\n'):
167        if line.startswith('Revision:'):
168            break
169
170    fields = line.split(':')
171    msg = 'Keyword "Revision" was not found anywhere in text: %s' % version_info
172    assert fields[0].startswith('Revision'), msg
173
174    try:
175        revision_number = int(fields[1])
176    except:
177        msg = ("Revision number must be an integer. I got '%s'.\n"
178               'Check that the command svn is on the system path.'
179               % fields[1])
180        raise Exception, msg
181
182    return revision_number
183
184
185def store_version_info(destination_path='.', verbose=False):
186    """Obtain current version from Subversion and store it.
187   
188    Title: store_version_info()
189
190    Author: Ole Nielsen (Ole.Nielsen@ga.gov.au)
191
192    CreationDate: January 2006
193
194    Description:
195        This function obtains current version from Subversion and stores it
196        is a Python file named 'stored_version_info.py' for use with
197        get_version_info()
198
199        If svn is not available on the system PATH, an Exception is thrown
200    """
201
202    # Note (Ole): This function should not be unit tested as it will only
203    # work when running out of the sandpit. End users downloading the
204    # ANUGA distribution would see a failure.
205    #
206    # FIXME: This function should really only be used by developers (
207    # (e.g. for creating new ANUGA releases), so maybe it should move
208    # to somewhere else.
209   
210    import config
211
212    try:
213        fid = os.popen('svn info')
214    except:
215        msg = 'Command "svn" is not recognised on the system PATH'
216        raise Exception(msg)
217    else:   
218        txt = fid.read()
219        fid.close()
220
221
222        # Determine absolute filename
223        if destination_path[-1] != os.sep:
224            destination_path += os.sep
225           
226        filename = destination_path + config.version_filename
227
228        fid = open(filename, 'w')
229
230        docstring = 'Stored version info.\n\n'
231        docstring += 'This file provides the version for distributions '
232        docstring += 'that are not accessing Subversion directly.\n'
233        docstring += 'The file is automatically generated and should not '
234        docstring += 'be modified manually.\n'
235        fid.write('"""%s"""\n\n' %docstring)
236       
237        fid.write('version_info = """\n%s"""' %txt)
238        fid.close()
239
240
241        if verbose is True:
242            log.critical('Version info stored to %s' % filename)
243
244
245def safe_crc(string):
246    """64 bit safe crc computation.
247
248    See http://docs.python.org/library/zlib.html#zlib.crc32:
249
250        To generate the same numeric value across all Python versions
251        and platforms use crc32(data) & 0xffffffff.
252    """
253
254    from zlib import crc32
255
256    return crc32(string) & 0xffffffff
257
258
259def compute_checksum(filename, max_length=2**20):
260    """Compute the CRC32 checksum for specified file
261
262    Optional parameter max_length sets the maximum number
263    of bytes used to limit time used with large files.
264    Default = 2**20 (1MB)
265    """
266
267    fid = open(filename, 'rb') # Use binary for portability
268    crcval = safe_crc(fid.read(max_length))
269    fid.close()
270
271    return crcval
272
273def get_pathname_from_package(package):
274    """Get pathname of given package (provided as string)
275
276    This is useful for reading files residing in the same directory as
277    a particular module. Typically, this is required in unit tests depending
278    on external files.
279
280    The given module must start from a directory on the pythonpath
281    and be importable using the import statement.
282
283    Example
284    path = get_pathname_from_package('anuga.utilities')
285
286    """
287
288    exec('import %s as x' %package)
289
290    path = x.__path__[0]
291   
292    return path
293
294    # Alternative approach that has been used at times
295    #try:
296    #    # When unit test is run from current dir
297    #    p1 = read_polygon('mainland_only.csv')
298    #except:
299    #    # When unit test is run from ANUGA root dir
300    #    from os.path import join, split
301    #    dir, tail = split(__file__)
302    #    path = join(dir, 'mainland_only.csv')
303    #    p1 = read_polygon(path)
304       
305   
306##
307# @brief Split a string into 'clean' fields.
308# @param str The string to process.
309# @param delimiter The delimiter string to split 'line' with.
310# @return A list of 'cleaned' field strings.
311# @note Any fields that were initially zero length will be removed.
312# @note If a field contains '\n' it isn't zero length.
313def clean_line(str, delimiter):
314    """Split string on given delimiter, remove whitespace from each field."""
315
316    return [x.strip() for x in str.strip().split(delimiter) if x != '']
317
318
319################################################################################
320# The following two functions are used to get around a problem with numpy and
321# NetCDF files.  Previously, using Numeric, we could take a list of strings and
322# convert to a Numeric array resulting in this:
323#     Numeric.array(['abc', 'xy']) -> [['a', 'b', 'c'],
324#                                      ['x', 'y', ' ']]
325#
326# However, under numpy we get:
327#     numpy.array(['abc', 'xy']) -> ['abc',
328#                                    'xy']
329#
330# And writing *strings* to a NetCDF file is problematic.
331#
332# The solution is to use these two routines to convert a 1-D list of strings
333# to the 2-D list of chars form and back.  The 2-D form can be written to a
334# NetCDF file as before.
335#
336# The other option, of inverting a list of tag strings into a dictionary with
337# keys being the unique tag strings and the key value a list of indices of where
338# the tag string was in the original list was rejected because:
339#    1. It's a lot of work
340#    2. We'd have to rewite the I/O code a bit (extra variables instead of one)
341#    3. The code below is fast enough in an I/O scenario
342################################################################################
343
344##
345# @brief Convert 1-D list of strings to 2-D list of chars.
346# @param l 1-dimensional list of strings.
347# @return A 2-D list of 'characters' (1 char strings).
348# @note No checking that we supply a 1-D list.
349def string_to_char(l):
350    '''Convert 1-D list of strings to 2-D list of chars.'''
351
352    if not l:
353        return []
354
355    if l == ['']:
356        l = [' ']
357
358    maxlen = reduce(max, map(len, l))
359    ll = [x.ljust(maxlen) for x in l]
360    result = []
361    for s in ll:
362        result.append([x for x in s])
363    return result
364
365
366##
367# @brief Convert 2-D list of chars to 1-D list of strings.
368# @param ll 2-dimensional list of 'characters' (1 char strings).
369# @return A 1-dimensional list of strings.
370# @note Each string has had right-end spaces removed.
371def char_to_string(ll):
372    '''Convert 2-D list of chars to 1-D list of strings.'''
373
374    return map(string.rstrip, [''.join(x) for x in ll])
375
376################################################################################
377
378##
379# @brief Get list of variable names in a python expression string.
380# @param source A string containing a python expression.
381# @return A list of variable name strings.
382# @note Throws SyntaxError exception if not a valid expression.
383def get_vars_in_expression(source):
384    '''Get list of variable names in a python expression.'''
385
386    import compiler
387    from compiler.ast import Node
388
389    ##
390    # @brief Internal recursive function.
391    # @param node An AST parse Node.
392    # @param var_list Input list of variables.
393    # @return An updated list of variables.
394    def get_vars_body(node, var_list=[]):
395        if isinstance(node, Node):
396            if node.__class__.__name__ == 'Name':
397                for child in node.getChildren():
398                    if child not in var_list:
399                        var_list.append(child)
400            for child in node.getChildren():
401                if isinstance(child, Node):
402                    for child in node.getChildren():
403                        var_list = get_vars_body(child, var_list)
404                    break
405
406        return var_list
407
408    return get_vars_body(compiler.parse(source))
409
410
411##
412# @brief Get a file from the web.
413# @param file_url URL of the file to fetch.
414# @param file_name Path to file to create in the filesystem.
415# @param auth Auth tuple (httpproxy, proxyuser, proxypass).
416# @param blocksize Read file in this block size.
417# @return (True, auth) if successful, else (False, auth).
418# @note If 'auth' not supplied, will prompt user.
419# @note Will try using environment variable HTTP_PROXY for proxy server.
420# @note Will try using environment variable PROXY_USERNAME for proxy username.
421# @note Will try using environment variable PROXY_PASSWORD for proxy password.
422def get_web_file(file_url, file_name, auth=None, blocksize=1024*1024):
423    '''Get a file from the web (HTTP).
424
425    file_url:  The URL of the file to get
426    file_name: Local path to save loaded file in
427    auth:      A tuple (httpproxy, proxyuser, proxypass)
428    blocksize: Block size of file reads
429   
430    Will try simple load through urllib first.  Drop down to urllib2
431    if there is a proxy and it requires authentication.
432
433    Environment variable HTTP_PROXY can be used to supply proxy information.
434    PROXY_USERNAME is used to supply the authentication username.
435    PROXY_PASSWORD supplies the password, if you dare!
436    '''
437
438    # Simple fetch, if fails, check for proxy error
439    try:
440        urllib.urlretrieve(file_url, file_name)
441        return (True, auth)     # no proxy, no auth required
442    except IOError, e:
443        if e[1] == 407:     # proxy error
444            pass
445        elif e[1][0] == 113:  # no route to host
446            print 'No route to host for %s' % file_url
447            return (False, auth)    # return False
448        else:
449            print 'Unknown connection error to %s' % file_url
450            return (False, auth)
451
452    # We get here if there was a proxy error, get file through the proxy
453    # unpack auth info
454    try:
455        (httpproxy, proxyuser, proxypass) = auth
456    except:
457        (httpproxy, proxyuser, proxypass) = (None, None, None)
458
459    # fill in any gaps from the environment
460    if httpproxy is None:
461        httpproxy = os.getenv('HTTP_PROXY')
462    if proxyuser is None:
463        proxyuser = os.getenv('PROXY_USERNAME')
464    if proxypass is None:
465        proxypass = os.getenv('PROXY_PASSWORD')
466
467    # Get auth info from user if still not supplied
468    if httpproxy is None or proxyuser is None or proxypass is None:
469        print '-'*72
470        print ('You need to supply proxy authentication information.')
471        if httpproxy is None:
472            httpproxy = raw_input('                    proxy server: ')
473        else:
474            print '         HTTP proxy was supplied: %s' % httpproxy
475        if proxyuser is None:
476            proxyuser = raw_input('                  proxy username: ') 
477        else:
478            print 'HTTP proxy username was supplied: %s' % proxyuser
479        if proxypass is None:
480            proxypass = getpass.getpass('                  proxy password: ')
481        else:
482            print 'HTTP proxy password was supplied: %s' % '*'*len(proxyuser)
483        print '-'*72
484
485    # the proxy URL cannot start with 'http://', we add that later
486    httpproxy = httpproxy.lower()
487    if httpproxy.startswith('http://'):
488        httpproxy = httpproxy.replace('http://', '', 1)
489
490    # open remote file
491    proxy = urllib2.ProxyHandler({'http': 'http://' + proxyuser
492                                              + ':' + proxypass
493                                              + '@' + httpproxy})
494    authinfo = urllib2.HTTPBasicAuthHandler()
495    opener = urllib2.build_opener(proxy, authinfo, urllib2.HTTPHandler)
496    urllib2.install_opener(opener)
497    try:
498        webget = urllib2.urlopen(file_url)
499    except urllib2.HTTPError, e:
500        print 'Error received from proxy:\n%s' % str(e)
501        print 'Possibly the user/password is wrong.'
502        return (False, (httpproxy, proxyuser, proxypass))
503
504    # transfer file to local filesystem
505    fd = open(file_name, 'wb')
506    while True:
507        data = webget.read(blocksize)
508        if len(data) == 0:
509            break
510        fd.write(data)
511    fd.close
512    webget.close()
513
514    # return successful auth info
515    return (True, (httpproxy, proxyuser, proxypass))
516
517
518##
519# @brief Tar a file (or directory) into a tarfile.
520# @param files A list of files (or directories) to tar.
521# @param tarfile The created tarfile name.
522# @note 'files' may be a string (single file) or a list of strings.
523# @note We use gzip compression.
524def tar_file(files, tarname):
525    '''Compress a file or directory into a tar file.'''
526
527    if isinstance(files, basestring):
528        files = [files]
529
530    o = tarfile.open(tarname, 'w:gz')
531    for file in files:
532        o.add(file)
533    o.close()
534
535
536##
537# @brief Untar a file into an optional target directory.
538# @param tarname Name of the file to untar.
539# @param target_dir Directory to untar into.
540def untar_file(tarname, target_dir='.'):
541    '''Uncompress a tar file.'''
542
543    o = tarfile.open(tarname, 'r:gz')
544    members = o.getmembers()
545    for member in members:
546        o.extract(member, target_dir)
547    o.close()
548
549
550##
551# @brief Return a hex digest (MD5) of a given file.
552# @param filename Path to the file of interest.
553# @param blocksize Size of data blocks to read.
554# @return A hex digest string (16 bytes).
555# @note Uses MD5 digest if hashlib not available.
556def get_file_hexdigest(filename, blocksize=1024*1024*10):
557    '''Get a hex digest of a file.'''
558
559    if hashlib.__name__ == 'hashlib':
560        m = hashlib.md5()       # new - 'hashlib' module
561    else:
562        m = hashlib.new()       # old - 'md5' module - remove once py2.4 gone
563    fd = open(filename, 'r')
564           
565    while True:
566        data = fd.read(blocksize)
567        if len(data) == 0:
568            break
569        m.update(data)
570                                                               
571    fd.close()
572    return m.hexdigest()
573
574
575##
576# @brief Create a file containing a hexdigest string of a data file.
577# @param data_file Path to the file to get the hexdigest from.
578# @param digest_file Path to hexdigest file to create.
579# @note Uses MD5 digest.
580def make_digest_file(data_file, digest_file):
581    '''Create a file containing the hex digest string of a data file.'''
582   
583    hexdigest = get_file_hexdigest(data_file)
584    fd = open(digest_file, 'w')
585    fd.write(hexdigest)
586    fd.close()
587
588
589##
590# @brief Function to return the length of a file.
591# @param in_file Path to file to get length of.
592# @return Number of lines in file.
593# @note Doesn't count '\n' characters.
594# @note Zero byte file, returns 0.
595# @note No \n in file at all, but >0 chars, returns 1.
596def file_length(in_file):
597    '''Function to return the length of a file.'''
598
599    fid = open(in_file)
600    data = fid.readlines()
601    fid.close()
602    return len(data)
603
604
Note: See TracBrowser for help on using the repository browser.