source: branches/numpy/anuga/utilities/system_tools.py @ 7091

Last change on this file since 7091 was 7035, checked in by rwilson, 15 years ago

Back-merge from Numeric trunk.

File size: 19.1 KB
Line 
1"""Implementation of tools to do with system administration made as platform independent as possible.
2
3
4"""
5
6import sys
7import os
8import string
9import urllib
10import urllib2
11import getpass
12import tarfile
13try:
14    import hashlib
15except ImportError:
16    import md5 as hashlib
17
18
19def log_to_file(filename, s, verbose=False):
20    """Log string to file name
21    """
22
23    fid = open(filename, 'a')
24    if verbose: print s
25    fid.write(s + '\n')
26    fid.close()
27
28
29def get_user_name():
30    """Get user name provide by operating system
31    """
32
33    if sys.platform == 'win32':
34        #user = os.getenv('USERPROFILE')
35        user = os.getenv('USERNAME')
36    else:
37        user = os.getenv('LOGNAME')
38
39
40    return user   
41
42def get_host_name():
43    """Get host name provide by operating system
44    """
45
46    if sys.platform == 'win32':
47        host = os.getenv('COMPUTERNAME')
48    else:
49        host = os.uname()[1]
50
51
52    return host   
53
54def get_revision_number():
55    """Get the version number of this repository copy.
56
57    Try getting data from stored_version_info.py first, otherwise
58    try using SubWCRev.exe (Windows) or svnversion (linux), otherwise
59    try reading file .svn/entries for version information, otherwise
60    throw an exception.
61
62    NOTE: This requires that the command svn is on the system PATH
63    (simply aliasing svn to the binary will not work)
64    """
65
66    def get_revision_from_svn_entries():
67        '''Get a subversion revision number from the .svn/entires file.'''
68
69        msg = '''
70No version info stored and command 'svn' is not recognised on the system PATH.
71
72If ANUGA has been installed from a distribution e.g. as obtained from SourceForge,
73the version info should be available in the automatically generated file
74'stored_version_info.py' in the anuga root directory.
75
76If run from a Subversion sandpit, ANUGA will try to obtain the version info by
77using the command 'svn info'.  In this case, make sure the command line client
78'svn' is accessible on the system path.  Simply aliasing 'svn' to the binary will
79not work.
80
81If you are using Windows, you have to install the file svn.exe which can be
82obtained from http://www.collab.net/downloads/subversion.
83
84Good luck!
85'''
86
87        try:
88            fd = open(os.path.join('.svn', 'entries'))
89        except:
90            raise Exception, msg
91
92        line = fd.readlines()[3]
93        fd.close()
94        try:
95            revision_number = int(line)
96        except:
97            msg = ".svn/entries, line 4 was '%s'?" % line.strip()
98            raise Exception, msg
99
100        return revision_number
101
102    def get_revision_from_svn_client():
103        '''Get a subversion revision number from an svn client.'''
104
105        if sys.platform[0:3] == 'win':
106            try:
107                fid = os.popen(r'C:\Program Files\TortoiseSVN\bin\SubWCRev.exe')
108            except:
109                return get_revision_from_svn_entries()
110            else:
111                version_info = fid.read()
112                if version_info == '':
113                    return get_revision_from_svn_entries()
114
115            # split revision number from data
116            for line in version_info.split('\n'):
117                if line.startswith('Updated to revision '):
118                    break
119
120            fields = line.split(' ')
121            msg = 'Keyword "Revision" was not found anywhere in text: %s' % version_info
122            assert fields[0].startswith('Updated'), msg
123
124            try:
125                revision_number = int(fields[3])
126            except:
127                msg = ("Revision number must be an integer. I got '%s' from "
128                       "'SubWCRev.exe'." % fields[3])
129                raise Exception, msg
130        else:                   # assume Linux
131            try:
132                fid = os.popen('svnversion -n . 2>/dev/null')
133            except:
134                return get_revision_from_svn_entries()
135            else:
136                version_info = fid.read()
137                if version_info == '':
138                    return get_revision_from_svn_entries()
139
140            # split revision number from data
141            if ':' in version_info:
142                (_, revision_number) =  version_info.split(':')
143            elif version_info.endswith('M'):
144                revision_number = version_info[:-1]
145            else:
146                revision_number = version_info
147
148            try:
149                revision_number = int(revision_number)
150            except:
151                msg = ("Revision number must be an integer. I got '%s' from "
152                       "'svn'." % version_info)
153                raise Exception, msg
154
155        return revision_number
156
157    # try to get revision information from stored_version_info.py
158    try:
159        from anuga.stored_version_info import version_info
160    except:
161        return get_revision_from_svn_client()
162
163    # split revision number from data
164    for line in version_info.split('\n'):
165        if line.startswith('Revision:'):
166            break
167
168    fields = line.split(':')
169    msg = 'Keyword "Revision" was not found anywhere in text: %s' % version_info
170    assert fields[0].startswith('Revision'), msg
171
172    try:
173        revision_number = int(fields[1])
174    except:
175        msg = ("Revision number must be an integer. I got '%s'.\n"
176               'Check that the command svn is on the system path.'
177               % fields[1])
178        raise Exception, msg
179
180    return revision_number
181
182
183def store_version_info(destination_path='.', verbose=False):
184    """Obtain current version from Subversion and store it.
185   
186    Title: store_version_info()
187
188    Author: Ole Nielsen (Ole.Nielsen@ga.gov.au)
189
190    CreationDate: January 2006
191
192    Description:
193        This function obtains current version from Subversion and stores it
194        is a Python file named 'stored_version_info.py' for use with
195        get_version_info()
196
197        If svn is not available on the system PATH, an Exception is thrown
198    """
199
200    # Note (Ole): This function should not be unit tested as it will only
201    # work when running out of the sandpit. End users downloading the
202    # ANUGA distribution would see a failure.
203    #
204    # FIXME: This function should really only be used by developers (
205    # (e.g. for creating new ANUGA releases), so maybe it should move
206    # to somewhere else.
207   
208    import config
209
210    try:
211        fid = os.popen('svn info')
212    except:
213        msg = 'Command "svn" is not recognised on the system PATH'
214        raise Exception(msg)
215    else:   
216        txt = fid.read()
217        fid.close()
218
219
220        # Determine absolute filename
221        if destination_path[-1] != os.sep:
222            destination_path += os.sep
223           
224        filename = destination_path + config.version_filename
225
226        fid = open(filename, 'w')
227
228        docstring = 'Stored version info.\n\n'
229        docstring += 'This file provides the version for distributions '
230        docstring += 'that are not accessing Subversion directly.\n'
231        docstring += 'The file is automatically generated and should not '
232        docstring += 'be modified manually.\n'
233        fid.write('"""%s"""\n\n' %docstring)
234       
235        fid.write('version_info = """\n%s"""' %txt)
236        fid.close()
237
238
239        if verbose is True:
240            print 'Version info stored to %s' %filename
241
242
243def safe_crc(string):
244    """64 bit safe crc computation.
245
246       See Guido's 64 bit fix at http://bugs.python.org/issue1202           
247    """
248
249    from zlib import crc32
250    import os
251
252    x = crc32(string)
253       
254    if os.name == 'posix' and os.uname()[4] in ['x86_64', 'ia64']:
255        crcval = x - ((x & 0x80000000) << 1)
256    else:
257        crcval = x
258       
259    return crcval
260
261
262def compute_checksum(filename, max_length=2**20):
263    """Compute the CRC32 checksum for specified file
264
265    Optional parameter max_length sets the maximum number
266    of bytes used to limit time used with large files.
267    Default = 2**20 (1MB)
268    """
269
270    fid = open(filename, 'rb') # Use binary for portability
271    crcval = safe_crc(fid.read(max_length))
272    fid.close()
273
274    return crcval
275
276def get_pathname_from_package(package):
277    """Get pathname of given package (provided as string)
278
279    This is useful for reading files residing in the same directory as
280    a particular module. Typically, this is required in unit tests depending
281    on external files.
282
283    The given module must start from a directory on the pythonpath
284    and be importable using the import statement.
285
286    Example
287    path = get_pathname_from_package('anuga.utilities')
288
289    """
290
291    exec('import %s as x' %package)
292
293    path = x.__path__[0]
294   
295    return path
296
297    # Alternative approach that has been used at times
298    #try:
299    #    # When unit test is run from current dir
300    #    p1 = read_polygon('mainland_only.csv')
301    #except:
302    #    # When unit test is run from ANUGA root dir
303    #    from os.path import join, split
304    #    dir, tail = split(__file__)
305    #    path = join(dir, 'mainland_only.csv')
306    #    p1 = read_polygon(path)
307       
308   
309##
310# @brief Split a string into 'clean' fields.
311# @param str The string to process.
312# @param delimiter The delimiter string to split 'line' with.
313# @return A list of 'cleaned' field strings.
314# @note Any fields that were initially zero length will be removed.
315# @note If a field contains '\n' it isn't zero length.
316def clean_line(str, delimiter):
317    """Split string on given delimiter, remove whitespace from each field."""
318
319    return [x.strip() for x in str.strip().split(delimiter) if x != '']
320
321
322################################################################################
323# The following two functions are used to get around a problem with numpy and
324# NetCDF files.  Previously, using Numeric, we could take a list of strings and
325# convert to a Numeric array resulting in this:
326#     Numeric.array(['abc', 'xy']) -> [['a', 'b', 'c'],
327#                                      ['x', 'y', ' ']]
328#
329# However, under numpy we get:
330#     numpy.array(['abc', 'xy']) -> ['abc',
331#                                    'xy']
332#
333# And writing *strings* to a NetCDF file is problematic.
334#
335# The solution is to use these two routines to convert a 1-D list of strings
336# to the 2-D list of chars form and back.  The 2-D form can be written to a
337# NetCDF file as before.
338#
339# The other option, of inverting a list of tag strings into a dictionary with
340# keys being the unique tag strings and the key value a list of indices of where
341# the tag string was in the original list was rejected because:
342#    1. It's a lot of work
343#    2. We'd have to rewite the I/O code a bit (extra variables instead of one)
344#    3. The code below is fast enough in an I/O scenario
345################################################################################
346
347##
348# @brief Convert 1-D list of strings to 2-D list of chars.
349# @param l 1-dimensional list of strings.
350# @return A 2-D list of 'characters' (1 char strings).
351# @note No checking that we supply a 1-D list.
352def string_to_char(l):
353    '''Convert 1-D list of strings to 2-D list of chars.'''
354
355    if not l:
356        return []
357
358    if l == ['']:
359        l = [' ']
360
361    maxlen = reduce(max, map(len, l))
362    ll = [x.ljust(maxlen) for x in l]
363    result = []
364    for s in ll:
365        result.append([x for x in s])
366    return result
367
368
369##
370# @brief Convert 2-D list of chars to 1-D list of strings.
371# @param ll 2-dimensional list of 'characters' (1 char strings).
372# @return A 1-dimensional list of strings.
373# @note Each string has had right-end spaces removed.
374def char_to_string(ll):
375    '''Convert 2-D list of chars to 1-D list of strings.'''
376
377    return map(string.rstrip, [''.join(x) for x in ll])
378
379################################################################################
380
381##
382# @brief Get list of variable names in a python expression string.
383# @param source A string containing a python expression.
384# @return A list of variable name strings.
385# @note Throws SyntaxError exception if not a valid expression.
386def get_vars_in_expression(source):
387    '''Get list of variable names in a python expression.'''
388
389    import compiler
390    from compiler.ast import Node
391
392    ##
393    # @brief Internal recursive function.
394    # @param node An AST parse Node.
395    # @param var_list Input list of variables.
396    # @return An updated list of variables.
397    def get_vars_body(node, var_list=[]):
398        if isinstance(node, Node):
399            if node.__class__.__name__ == 'Name':
400                for child in node.getChildren():
401                    if child not in var_list:
402                        var_list.append(child)
403            for child in node.getChildren():
404                if isinstance(child, Node):
405                    for child in node.getChildren():
406                        var_list = get_vars_body(child, var_list)
407                    break
408
409        return var_list
410
411    return get_vars_body(compiler.parse(source))
412
413
414##
415# @brief Get a file from the web.
416# @param file_url URL of the file to fetch.
417# @param file_name Path to file to create in the filesystem.
418# @param auth Auth tuple (httpproxy, proxyuser, proxypass).
419# @param blocksize Read file in this block size.
420# @return (True, auth) if successful, else (False, auth).
421# @note If 'auth' not supplied, will prompt user.
422# @note Will try using environment variable HTTP_PROXY for proxy server.
423# @note Will try using environment variable PROXY_USERNAME for proxy username.
424# @note Will try using environment variable PROXY_PASSWORD for proxy password.
425def get_web_file(file_url, file_name, auth=None, blocksize=1024*1024):
426    '''Get a file from the web (HTTP).
427
428    file_url:  The URL of the file to get
429    file_name: Local path to save loaded file in
430    auth:      A tuple (httpproxy, proxyuser, proxypass)
431    blocksize: Block size of file reads
432   
433    Will try simple load through urllib first.  Drop down to urllib2
434    if there is a proxy and it requires authentication.
435
436    Environment variable HTTP_PROXY can be used to supply proxy information.
437    PROXY_USERNAME is used to supply the authentication username.
438    PROXY_PASSWORD supplies the password, if you dare!
439    '''
440
441    # Simple fetch, if fails, check for proxy error
442    try:
443        urllib.urlretrieve(file_url, file_name)
444        return (True, auth)     # no proxy, no auth required
445    except IOError, e:
446        if e[1] == 407:     # proxy error
447            pass
448        elif e[1][0] == 113:  # no route to host
449            print 'No route to host for %s' % file_url
450            return (False, auth)    # return False
451        else:
452            print 'Unknown connection error to %s' % file_url
453            return (False, auth)
454
455    # We get here if there was a proxy error, get file through the proxy
456    # unpack auth info
457    try:
458        (httpproxy, proxyuser, proxypass) = auth
459    except:
460        (httpproxy, proxyuser, proxypass) = (None, None, None)
461
462    # fill in any gaps from the environment
463    if httpproxy is None:
464        httpproxy = os.getenv('HTTP_PROXY')
465    if proxyuser is None:
466        proxyuser = os.getenv('PROXY_USERNAME')
467    if proxypass is None:
468        proxypass = os.getenv('PROXY_PASSWORD')
469
470    # Get auth info from user if still not supplied
471    if httpproxy is None or proxyuser is None or proxypass is None:
472        print '-'*72
473        print ('You need to supply proxy authentication information.')
474        if httpproxy is None:
475            httpproxy = raw_input('                    proxy server: ')
476        else:
477            print '         HTTP proxy was supplied: %s' % httpproxy
478        if proxyuser is None:
479            proxyuser = raw_input('                  proxy username: ') 
480        else:
481            print 'HTTP proxy username was supplied: %s' % proxyuser
482        if proxypass is None:
483            proxypass = getpass.getpass('                  proxy password: ')
484        else:
485            print 'HTTP proxy password was supplied: %s' % '*'*len(proxyuser)
486        print '-'*72
487
488    # the proxy URL cannot start with 'http://', we add that later
489    httpproxy = httpproxy.lower()
490    if httpproxy.startswith('http://'):
491        httpproxy = httpproxy.replace('http://', '', 1)
492
493    # open remote file
494    proxy = urllib2.ProxyHandler({'http': 'http://' + proxyuser
495                                              + ':' + proxypass
496                                              + '@' + httpproxy})
497    authinfo = urllib2.HTTPBasicAuthHandler()
498    opener = urllib2.build_opener(proxy, authinfo, urllib2.HTTPHandler)
499    urllib2.install_opener(opener)
500    try:
501        webget = urllib2.urlopen(file_url)
502    except urllib2.HTTPError, e:
503        print 'Error received from proxy:\n%s' % str(e)
504        print 'Possibly the user/password is wrong.'
505        return (False, (httpproxy, proxyuser, proxypass))
506
507    # transfer file to local filesystem
508    fd = open(file_name, 'wb')
509    while True:
510        data = webget.read(blocksize)
511        if len(data) == 0:
512            break
513        fd.write(data)
514    fd.close
515    webget.close()
516
517    # return successful auth info
518    return (True, (httpproxy, proxyuser, proxypass))
519
520
521##
522# @brief Tar a file (or directory) into a tarfile.
523# @param files A list of files (or directories) to tar.
524# @param tarfile The created tarfile name.
525# @note 'files' may be a string (single file) or a list of strings.
526# @note We use gzip compression.
527def tar_file(files, tarname):
528    '''Compress a file or directory into a tar file.'''
529
530    if isinstance(files, basestring):
531        files = [files]
532
533    o = tarfile.open(tarname, 'w:gz')
534    for file in files:
535        o.add(file)
536    o.close()
537
538
539##
540# @brief Untar a file into an optional target directory.
541# @param tarname Name of the file to untar.
542# @param target_dir Directory to untar into.
543def untar_file(tarname, target_dir='.'):
544    '''Uncompress a tar file.'''
545
546    o = tarfile.open(tarname, 'r:gz')
547    members = o.getmembers()
548    for member in members:
549        o.extract(member, target_dir)
550    o.close()
551
552
553##
554# @brief Return a hex digest (MD5) of a given file.
555# @param filename Path to the file of interest.
556# @param blocksize Size of data blocks to read.
557# @return A hex digest string (16 bytes).
558# @note Uses MD5 digest.
559def get_file_hexdigest(filename, blocksize=1024*1024*10):
560    '''Get a hex digest of a file.'''
561
562    if hashlib.__name__ == 'hashlib':
563        m = hashlib.md5()       # new - 'hashlib' module
564    else:
565        m = hashlib.new()       # old - 'md5' module - remove once py2.4 gone
566    fd = open(filename, 'r')
567           
568    while True:
569        data = fd.read(blocksize)
570        if len(data) == 0:
571            break
572        m.update(data)
573                                                               
574    fd.close()
575    return m.hexdigest()
576
577    fd = open(filename, 'r')
578
579
580##
581# @brief Create a file containing a hexdigest string of a data file.
582# @param data_file Path to the file to get the hexdigest from.
583# @param digest_file Path to hexdigest file to create.
584# @note Uses MD5 digest.
585def make_digest_file(data_file, digest_file):
586    '''Create a file containing the hex digest string of a data file.'''
587   
588    hexdigest = get_file_hexdigest(data_file)
589    fd = open(digest_file, 'w')
590    fd.write(hexdigest)
591    fd.close()
592
593
594##
595# @brief Function to return the length of a file.
596# @param in_file Path to file to get length of.
597# @return Number of lines in file.
598# @note Doesn't count '\n' characters.
599# @note Zero byte file, returns 0.
600# @note No \n in file at all, but >0 chars, returns 1.
601def file_length(in_file):
602    '''Function to return the length of a file.'''
603
604    fid = open(in_file)
605    data = fid.readlines()
606    fid.close()
607    return len(data)
608
609
Note: See TracBrowser for help on using the repository browser.