source: trunk/anuga_core/source/anuga/utilities/system_tools.py @ 7876

Last change on this file since 7876 was 7486, checked in by ole, 15 years ago

Reverted changeset:6860 as there are too many types of output from svnversion.
We are now back to svn info.
Also moved nested function declarations out and made them Python style internals using
Added simple unit test of this functionality.

File size: 19.3 KB
Line 
1"""Implementation of tools to do with system administration made as platform independent as possible.
2
3
4"""
5
6import sys
7import os
8import string
9import urllib
10import urllib2
11import getpass
12import tarfile
13import warnings
14
15try:
16    import hashlib
17except ImportError:
18    import md5 as hashlib
19
20import anuga.utilities.log as log
21
22
23def log_to_file(filename, s, verbose=False):
24    """Log string to file name
25    """
26
27    fid = open(filename, 'a')
28    if verbose: print s
29    fid.write(s + '\n')
30    fid.close()
31
32
33def get_user_name():
34    """Get user name provide by operating system
35    """
36
37    if sys.platform == 'win32':
38        #user = os.getenv('USERPROFILE')
39        user = os.getenv('USERNAME')
40    else:
41        user = os.getenv('LOGNAME')
42
43
44    return user   
45
46def get_host_name():
47    """Get host name provide by operating system
48    """
49
50    if sys.platform == 'win32':
51        host = os.getenv('COMPUTERNAME')
52    else:
53        host = os.uname()[1]
54
55
56    return host   
57
58   
59   
60   
61   
62def __get_revision_from_svn_entries__():
63    """Get a subversion revision number from the .svn/entries file."""
64
65   
66    msg = '''
67No version info stored and command 'svn' is not recognised on the system PATH.
68
69If ANUGA has been installed from a distribution e.g. as obtained from SourceForge,
70the version info should be available in the automatically generated file
71'stored_version_info.py' in the anuga root directory.
72
73If run from a Subversion sandpit, ANUGA will try to obtain the version info by
74using the command 'svn info'.  In this case, make sure the command line client
75'svn' is accessible on the system path.  Simply aliasing 'svn' to the binary will
76not work.
77
78If you are using Windows, you have to install the file svn.exe which can be
79obtained from http://www.collab.net/downloads/subversion.
80
81Good luck!
82'''
83
84    try:
85        fd = open(os.path.join('.svn', 'entries'))
86    except:
87        raise Exception, msg
88
89    line = fd.readlines()[3]
90    fd.close()
91    try:
92        revision_number = int(line)
93    except:
94        msg = ".svn/entries, line 4 was '%s'?" % line.strip()
95        raise Exception, msg
96
97    return revision_number
98
99def __get_revision_from_svn_client__():
100    """Get a subversion revision number from an svn client."""
101
102    if sys.platform[0:3] == 'win':
103        try:
104            fid = os.popen(r'C:\Program Files\TortoiseSVN\bin\SubWCRev.exe')
105        except:
106            return __get_revision_from_svn_entries__()
107        else:
108            version_info = fid.read()
109            if version_info == '':
110                return __get_revision_from_svn_entries__()
111
112        # split revision number from data
113        for line in version_info.split('\n'):
114            if line.startswith('Updated to revision '):
115                break
116
117        fields = line.split(' ')
118        msg = 'Keyword "Revision" was not found anywhere in text: %s' % version_info
119        assert fields[0].startswith('Updated'), msg
120
121        try:
122            revision_number = int(fields[3])
123        except:
124            msg = ('Revision number must be an integer. I got "%s" from '
125                   '"SubWCRev.exe".' % fields[3])
126            raise Exception, msg
127    else:                   # assume Linux
128        try:
129            fid = os.popen('svn info . 2>/dev/null')
130        except:
131            return __get_revision_from_svn_entries__()
132        else:
133            version_info = fid.read()
134            if version_info == '':
135                return __get_revision_from_svn_entries__()
136
137        # Split revision number from data
138        for line in version_info.split('\n'):
139            if line.startswith('Revision:'):
140                break
141       
142        fields = line.split(':')
143        msg = 'Keyword "Revision" was not found anywhere in text: %s' % version_info
144        assert fields[0].startswith('Revision'), msg
145       
146       
147        #if ':' in version_info:
148        #    revision_number, _ = version_info.split(':', 1)
149        #    msg = ('Some modules have not been checked in. '
150        #           'Using last version from repository: %s' % revision_number)
151        #    warnings.warn(msg)
152        #else:
153        #    revision_number = version_info
154
155        try:
156            revision_number = int(fields[1])
157        except:
158            msg = ("Revision number must be an integer. I got '%s' from "
159                   "'svn'." % fields[1])
160            raise Exception, msg
161
162    return revision_number
163
164   
165   
166   
167   
168def get_revision_number():
169    """Get the version number of this repository copy.
170
171    Try getting data from stored_version_info.py first, otherwise
172    try using SubWCRev.exe (Windows) or svnversion (linux), otherwise
173    try reading file .svn/entries for version information, otherwise
174    throw an exception.
175
176    NOTE: This requires that the command svn is on the system PATH
177    (simply aliasing svn to the binary will not work)
178    """
179
180    # try to get revision information from stored_version_info.py
181    try:
182        from anuga.stored_version_info import version_info
183    except:
184        return __get_revision_from_svn_client__()
185
186    # split revision number from data
187    for line in version_info.split('\n'):
188        if line.startswith('Revision:'):
189            break
190
191    fields = line.split(':')
192    msg = 'Keyword "Revision" was not found anywhere in text: %s' % version_info
193    assert fields[0].startswith('Revision'), msg
194
195    try:
196        revision_number = int(fields[1])
197    except:
198        msg = ("Revision number must be an integer. I got '%s'.\n"
199               'Check that the command svn is on the system path.'
200               % fields[1])
201        raise Exception, msg
202
203    return revision_number
204
205
206def store_version_info(destination_path='.', verbose=False):
207    """Obtain current version from Subversion and store it.
208   
209    Title: store_version_info()
210
211    Author: Ole Nielsen (Ole.Nielsen@ga.gov.au)
212
213    CreationDate: January 2006
214
215    Description:
216        This function obtains current version from Subversion and stores it
217        is a Python file named 'stored_version_info.py' for use with
218        get_version_info()
219
220        If svn is not available on the system PATH, an Exception is thrown
221    """
222
223    # Note (Ole): This function should not be unit tested as it will only
224    # work when running out of the sandpit. End users downloading the
225    # ANUGA distribution would see a failure.
226    #
227    # FIXME: This function should really only be used by developers (
228    # (e.g. for creating new ANUGA releases), so maybe it should move
229    # to somewhere else.
230   
231    import config
232
233    try:
234        fid = os.popen('svn info')
235    except:
236        msg = 'Command "svn" is not recognised on the system PATH'
237        raise Exception(msg)
238    else:   
239        txt = fid.read()
240        fid.close()
241
242
243        # Determine absolute filename
244        if destination_path[-1] != os.sep:
245            destination_path += os.sep
246           
247        filename = destination_path + config.version_filename
248
249        fid = open(filename, 'w')
250
251        docstring = 'Stored version info.\n\n'
252        docstring += 'This file provides the version for distributions '
253        docstring += 'that are not accessing Subversion directly.\n'
254        docstring += 'The file is automatically generated and should not '
255        docstring += 'be modified manually.\n'
256        fid.write('"""%s"""\n\n' %docstring)
257       
258        fid.write('version_info = """\n%s"""' %txt)
259        fid.close()
260
261
262        if verbose is True:
263            log.critical('Version info stored to %s' % filename)
264
265
266def safe_crc(string):
267    """64 bit safe crc computation.
268
269    See http://docs.python.org/library/zlib.html#zlib.crc32:
270
271        To generate the same numeric value across all Python versions
272        and platforms use crc32(data) & 0xffffffff.
273    """
274
275    from zlib import crc32
276
277    return crc32(string) & 0xffffffff
278
279
280def compute_checksum(filename, max_length=2**20):
281    """Compute the CRC32 checksum for specified file
282
283    Optional parameter max_length sets the maximum number
284    of bytes used to limit time used with large files.
285    Default = 2**20 (1MB)
286    """
287
288    fid = open(filename, 'rb') # Use binary for portability
289    crcval = safe_crc(fid.read(max_length))
290    fid.close()
291
292    return crcval
293
294def get_pathname_from_package(package):
295    """Get pathname of given package (provided as string)
296
297    This is useful for reading files residing in the same directory as
298    a particular module. Typically, this is required in unit tests depending
299    on external files.
300
301    The given module must start from a directory on the pythonpath
302    and be importable using the import statement.
303
304    Example
305    path = get_pathname_from_package('anuga.utilities')
306
307    """
308
309    exec('import %s as x' %package)
310
311    path = x.__path__[0]
312   
313    return path
314
315    # Alternative approach that has been used at times
316    #try:
317    #    # When unit test is run from current dir
318    #    p1 = read_polygon('mainland_only.csv')
319    #except:
320    #    # When unit test is run from ANUGA root dir
321    #    from os.path import join, split
322    #    dir, tail = split(__file__)
323    #    path = join(dir, 'mainland_only.csv')
324    #    p1 = read_polygon(path)
325       
326   
327##
328# @brief Split a string into 'clean' fields.
329# @param str The string to process.
330# @param delimiter The delimiter string to split 'line' with.
331# @return A list of 'cleaned' field strings.
332# @note Any fields that were initially zero length will be removed.
333# @note If a field contains '\n' it isn't zero length.
334def clean_line(str, delimiter):
335    """Split string on given delimiter, remove whitespace from each field."""
336
337    return [x.strip() for x in str.strip().split(delimiter) if x != '']
338
339
340################################################################################
341# The following two functions are used to get around a problem with numpy and
342# NetCDF files.  Previously, using Numeric, we could take a list of strings and
343# convert to a Numeric array resulting in this:
344#     Numeric.array(['abc', 'xy']) -> [['a', 'b', 'c'],
345#                                      ['x', 'y', ' ']]
346#
347# However, under numpy we get:
348#     numpy.array(['abc', 'xy']) -> ['abc',
349#                                    'xy']
350#
351# And writing *strings* to a NetCDF file is problematic.
352#
353# The solution is to use these two routines to convert a 1-D list of strings
354# to the 2-D list of chars form and back.  The 2-D form can be written to a
355# NetCDF file as before.
356#
357# The other option, of inverting a list of tag strings into a dictionary with
358# keys being the unique tag strings and the key value a list of indices of where
359# the tag string was in the original list was rejected because:
360#    1. It's a lot of work
361#    2. We'd have to rewite the I/O code a bit (extra variables instead of one)
362#    3. The code below is fast enough in an I/O scenario
363################################################################################
364
365##
366# @brief Convert 1-D list of strings to 2-D list of chars.
367# @param l 1-dimensional list of strings.
368# @return A 2-D list of 'characters' (1 char strings).
369# @note No checking that we supply a 1-D list.
370def string_to_char(l):
371    '''Convert 1-D list of strings to 2-D list of chars.'''
372
373    if not l:
374        return []
375
376    if l == ['']:
377        l = [' ']
378
379    maxlen = reduce(max, map(len, l))
380    ll = [x.ljust(maxlen) for x in l]
381    result = []
382    for s in ll:
383        result.append([x for x in s])
384    return result
385
386
387##
388# @brief Convert 2-D list of chars to 1-D list of strings.
389# @param ll 2-dimensional list of 'characters' (1 char strings).
390# @return A 1-dimensional list of strings.
391# @note Each string has had right-end spaces removed.
392def char_to_string(ll):
393    '''Convert 2-D list of chars to 1-D list of strings.'''
394
395    return map(string.rstrip, [''.join(x) for x in ll])
396
397################################################################################
398
399##
400# @brief Get list of variable names in a python expression string.
401# @param source A string containing a python expression.
402# @return A list of variable name strings.
403# @note Throws SyntaxError exception if not a valid expression.
404def get_vars_in_expression(source):
405    '''Get list of variable names in a python expression.'''
406
407    import compiler
408    from compiler.ast import Node
409
410    ##
411    # @brief Internal recursive function.
412    # @param node An AST parse Node.
413    # @param var_list Input list of variables.
414    # @return An updated list of variables.
415    def get_vars_body(node, var_list=[]):
416        if isinstance(node, Node):
417            if node.__class__.__name__ == 'Name':
418                for child in node.getChildren():
419                    if child not in var_list:
420                        var_list.append(child)
421            for child in node.getChildren():
422                if isinstance(child, Node):
423                    for child in node.getChildren():
424                        var_list = get_vars_body(child, var_list)
425                    break
426
427        return var_list
428
429    return get_vars_body(compiler.parse(source))
430
431
432##
433# @brief Get a file from the web.
434# @param file_url URL of the file to fetch.
435# @param file_name Path to file to create in the filesystem.
436# @param auth Auth tuple (httpproxy, proxyuser, proxypass).
437# @param blocksize Read file in this block size.
438# @return (True, auth) if successful, else (False, auth).
439# @note If 'auth' not supplied, will prompt user.
440# @note Will try using environment variable HTTP_PROXY for proxy server.
441# @note Will try using environment variable PROXY_USERNAME for proxy username.
442# @note Will try using environment variable PROXY_PASSWORD for proxy password.
443def get_web_file(file_url, file_name, auth=None, blocksize=1024*1024):
444    '''Get a file from the web (HTTP).
445
446    file_url:  The URL of the file to get
447    file_name: Local path to save loaded file in
448    auth:      A tuple (httpproxy, proxyuser, proxypass)
449    blocksize: Block size of file reads
450   
451    Will try simple load through urllib first.  Drop down to urllib2
452    if there is a proxy and it requires authentication.
453
454    Environment variable HTTP_PROXY can be used to supply proxy information.
455    PROXY_USERNAME is used to supply the authentication username.
456    PROXY_PASSWORD supplies the password, if you dare!
457    '''
458
459    # Simple fetch, if fails, check for proxy error
460    try:
461        urllib.urlretrieve(file_url, file_name)
462        return (True, auth)     # no proxy, no auth required
463    except IOError, e:
464        if e[1] == 407:     # proxy error
465            pass
466        elif e[1][0] == 113:  # no route to host
467            print 'No route to host for %s' % file_url
468            return (False, auth)    # return False
469        else:
470            print 'Unknown connection error to %s' % file_url
471            return (False, auth)
472
473    # We get here if there was a proxy error, get file through the proxy
474    # unpack auth info
475    try:
476        (httpproxy, proxyuser, proxypass) = auth
477    except:
478        (httpproxy, proxyuser, proxypass) = (None, None, None)
479
480    # fill in any gaps from the environment
481    if httpproxy is None:
482        httpproxy = os.getenv('HTTP_PROXY')
483    if proxyuser is None:
484        proxyuser = os.getenv('PROXY_USERNAME')
485    if proxypass is None:
486        proxypass = os.getenv('PROXY_PASSWORD')
487
488    # Get auth info from user if still not supplied
489    if httpproxy is None or proxyuser is None or proxypass is None:
490        print '-'*72
491        print ('You need to supply proxy authentication information.')
492        if httpproxy is None:
493            httpproxy = raw_input('                    proxy server: ')
494        else:
495            print '         HTTP proxy was supplied: %s' % httpproxy
496        if proxyuser is None:
497            proxyuser = raw_input('                  proxy username: ') 
498        else:
499            print 'HTTP proxy username was supplied: %s' % proxyuser
500        if proxypass is None:
501            proxypass = getpass.getpass('                  proxy password: ')
502        else:
503            print 'HTTP proxy password was supplied: %s' % '*'*len(proxyuser)
504        print '-'*72
505
506    # the proxy URL cannot start with 'http://', we add that later
507    httpproxy = httpproxy.lower()
508    if httpproxy.startswith('http://'):
509        httpproxy = httpproxy.replace('http://', '', 1)
510
511    # open remote file
512    proxy = urllib2.ProxyHandler({'http': 'http://' + proxyuser
513                                              + ':' + proxypass
514                                              + '@' + httpproxy})
515    authinfo = urllib2.HTTPBasicAuthHandler()
516    opener = urllib2.build_opener(proxy, authinfo, urllib2.HTTPHandler)
517    urllib2.install_opener(opener)
518    try:
519        webget = urllib2.urlopen(file_url)
520    except urllib2.HTTPError, e:
521        print 'Error received from proxy:\n%s' % str(e)
522        print 'Possibly the user/password is wrong.'
523        return (False, (httpproxy, proxyuser, proxypass))
524
525    # transfer file to local filesystem
526    fd = open(file_name, 'wb')
527    while True:
528        data = webget.read(blocksize)
529        if len(data) == 0:
530            break
531        fd.write(data)
532    fd.close
533    webget.close()
534
535    # return successful auth info
536    return (True, (httpproxy, proxyuser, proxypass))
537
538
539##
540# @brief Tar a file (or directory) into a tarfile.
541# @param files A list of files (or directories) to tar.
542# @param tarfile The created tarfile name.
543# @note 'files' may be a string (single file) or a list of strings.
544# @note We use gzip compression.
545def tar_file(files, tarname):
546    '''Compress a file or directory into a tar file.'''
547
548    if isinstance(files, basestring):
549        files = [files]
550
551    o = tarfile.open(tarname, 'w:gz')
552    for file in files:
553        o.add(file)
554    o.close()
555
556
557##
558# @brief Untar a file into an optional target directory.
559# @param tarname Name of the file to untar.
560# @param target_dir Directory to untar into.
561def untar_file(tarname, target_dir='.'):
562    '''Uncompress a tar file.'''
563
564    o = tarfile.open(tarname, 'r:gz')
565    members = o.getmembers()
566    for member in members:
567        o.extract(member, target_dir)
568    o.close()
569
570
571##
572# @brief Return a hex digest (MD5) of a given file.
573# @param filename Path to the file of interest.
574# @param blocksize Size of data blocks to read.
575# @return A hex digest string (16 bytes).
576# @note Uses MD5 digest if hashlib not available.
577def get_file_hexdigest(filename, blocksize=1024*1024*10):
578    '''Get a hex digest of a file.'''
579
580    if hashlib.__name__ == 'hashlib':
581        m = hashlib.md5()       # new - 'hashlib' module
582    else:
583        m = hashlib.new()       # old - 'md5' module - remove once py2.4 gone
584    fd = open(filename, 'r')
585           
586    while True:
587        data = fd.read(blocksize)
588        if len(data) == 0:
589            break
590        m.update(data)
591                                                               
592    fd.close()
593    return m.hexdigest()
594
595
596##
597# @brief Create a file containing a hexdigest string of a data file.
598# @param data_file Path to the file to get the hexdigest from.
599# @param digest_file Path to hexdigest file to create.
600# @note Uses MD5 digest.
601def make_digest_file(data_file, digest_file):
602    '''Create a file containing the hex digest string of a data file.'''
603   
604    hexdigest = get_file_hexdigest(data_file)
605    fd = open(digest_file, 'w')
606    fd.write(hexdigest)
607    fd.close()
608
609
610##
611# @brief Function to return the length of a file.
612# @param in_file Path to file to get length of.
613# @return Number of lines in file.
614# @note Doesn't count '\n' characters.
615# @note Zero byte file, returns 0.
616# @note No \n in file at all, but >0 chars, returns 1.
617def file_length(in_file):
618    '''Function to return the length of a file.'''
619
620    fid = open(in_file)
621    data = fid.readlines()
622    fid.close()
623    return len(data)
624
625
Note: See TracBrowser for help on using the repository browser.