source: anuga_core/source/anuga/utilities/system_tools.py @ 6666

Last change on this file since 6666 was 6666, checked in by rwilson, 15 years ago

Added comments, tweaked interaction with user.

File size: 14.2 KB
Line 
1"""Implementation of tools to do with system administration made as platform independent as possible.
2
3
4"""
5
6import sys
7import os
8import urllib
9import urllib2
10import getpass
11import tarfile
12import md5
13
14
15def log_to_file(filename, s, verbose=False):
16    """Log string to file name
17    """
18
19    fid = open(filename, 'a')
20    if verbose: print s
21    fid.write(s + '\n')
22    fid.close()
23
24
25def get_user_name():
26    """Get user name provide by operating system
27    """
28
29    if sys.platform == 'win32':
30        #user = os.getenv('USERPROFILE')
31        user = os.getenv('USERNAME')
32    else:
33        user = os.getenv('LOGNAME')
34
35
36    return user   
37
38def get_host_name():
39    """Get host name provide by operating system
40    """
41
42    if sys.platform == 'win32':
43        host = os.getenv('COMPUTERNAME')
44    else:
45        host = os.uname()[1]
46
47
48    return host   
49
50def get_revision_number():
51    """Get the version number of the SVN
52    NOTE: This requires that the command svn is on the system PATH
53    (simply aliasing svn to the binary will not work)
54    """
55
56    # Create dummy info
57    #info = 'Revision: Version info could not be obtained.'
58    #info += 'A command line version of svn must be availbable '
59    #info += 'on the system PATH, access to the subversion '
60    #info += 'repository is necessary and the output must '
61    #info += 'contain a line starting with "Revision:"'
62   
63
64    #FIXME (Ole): Change this so that svn info is attempted first.
65    # If that fails, try to read a stored file with that same info (this would be created by e.g. the release script). Failing that, throw an exception.
66
67    #FIXME (Ole): Move this and store_version_info to utilities
68
69
70    try:
71        from anuga.stored_version_info import version_info
72    except:
73        msg = 'No version info stored and command "svn" is not '
74        msg += 'recognised on the system PATH.\n\n'
75        msg += 'If ANUGA has been installed from a distribution e.g. as '
76        msg += 'obtained from SourceForge,\n'
77        msg += 'the version info should be '
78        msg += 'available in the automatically generated file '
79        msg += 'stored_version_info.py\n'
80        msg += 'in the anuga root directory.\n'
81        msg += 'If run from a Subversion sandpit, '
82        msg += 'ANUGA will try to obtain the version info '
83        msg += 'by using the command: "svn info".\n'
84        msg += 'In this case, make sure the command line client '
85        msg += 'svn is accessible on the system path. '
86        msg += 'Simply aliasing svn to the binary will not work. '
87        msg += 'If you are using Windows, you have to install the file svn.exe '
88        msg += 'which can be obtained from e.g '
89        msg += 'http://www.collab.net/downloads/subversion'
90        msg += 'Good luck!'
91
92        # No file available - try using Subversion
93        try:
94            # The null stuff is so this section fails quitly.
95            # This could cause the svn info command to fail due to
96            # the redirection being bad on some platforms.
97            # If that occurs then change this code.
98            if sys.platform[0:3] == 'win':
99                fid = os.popen('svn info 2> null')
100            else:
101                fid = os.popen('svn info 2>/dev/null')
102       
103        except:
104            raise Exception(msg)
105        else:
106            #print 'Got version from svn'           
107            version_info = fid.read()
108           
109            if version_info == '':
110                raise Exception(msg)   
111    else:
112        pass
113        #print 'Got version from file'
114
115           
116    for line in version_info.split('\n'):
117        if line.startswith('Revision:'):
118            break
119
120    fields = line.split(':')
121    msg = 'Keyword "Revision" was not found anywhere in text: %s' %version_info
122    assert fields[0].startswith('Revision'), msg           
123
124    try:
125        revision_number = int(fields[1])
126    except:
127        msg = 'Revision number must be an integer. I got %s' %fields[1]
128        msg += 'Check that the command svn is on the system path' 
129        raise Exception(msg)               
130       
131    return revision_number
132
133
134def store_version_info(destination_path='.', verbose=False):
135    """Obtain current version from Subversion and store it.
136   
137    Title: store_version_info()
138
139    Author: Ole Nielsen (Ole.Nielsen@ga.gov.au)
140
141    CreationDate: January 2006
142
143    Description:
144        This function obtains current version from Subversion and stores it
145        is a Python file named 'stored_version_info.py' for use with
146        get_version_info()
147
148        If svn is not available on the system PATH, an Exception is thrown
149    """
150
151    # Note (Ole): This function should not be unit tested as it will only
152    # work when running out of the sandpit. End users downloading the
153    # ANUGA distribution would see a failure.
154    #
155    # FIXME: This function should really only be used by developers (
156    # (e.g. for creating new ANUGA releases), so maybe it should move
157    # to somewhere else.
158   
159    import config
160
161    try:
162        fid = os.popen('svn info')
163    except:
164        msg = 'Command "svn" is not recognised on the system PATH'
165        raise Exception(msg)
166    else:   
167        txt = fid.read()
168        fid.close()
169
170
171        # Determine absolute filename
172        if destination_path[-1] != os.sep:
173            destination_path += os.sep
174           
175        filename = destination_path + config.version_filename
176
177        fid = open(filename, 'w')
178
179        docstring = 'Stored version info.\n\n'
180        docstring += 'This file provides the version for distributions '
181        docstring += 'that are not accessing Subversion directly.\n'
182        docstring += 'The file is automatically generated and should not '
183        docstring += 'be modified manually.\n'
184        fid.write('"""%s"""\n\n' %docstring)
185       
186        fid.write('version_info = """\n%s"""' %txt)
187        fid.close()
188
189
190        if verbose is True:
191            print 'Version info stored to %s' %filename
192
193def safe_crc(string):
194    """64 bit safe crc computation.
195
196       See Guido's 64 bit fix at http://bugs.python.org/issue1202           
197    """
198
199    from zlib import crc32
200    import os
201
202    x = crc32(string)
203       
204    if os.name == 'posix' and os.uname()[4] in ['x86_64', 'ia64']:
205        crcval = x - ((x & 0x80000000) << 1)
206    else:
207        crcval = x
208       
209    return crcval
210
211
212def compute_checksum(filename, max_length=2**20):
213    """Compute the CRC32 checksum for specified file
214
215    Optional parameter max_length sets the maximum number
216    of bytes used to limit time used with large files.
217    Default = 2**20 (1MB)
218    """
219
220    fid = open(filename, 'rb') # Use binary for portability
221    crcval = safe_crc(fid.read(max_length))
222    fid.close()
223
224    return crcval
225
226def get_pathname_from_package(package):
227    """Get pathname of given package (provided as string)
228
229    This is useful for reading files residing in the same directory as
230    a particular module. Typically, this is required in unit tests depending
231    on external files.
232
233    The given module must start from a directory on the pythonpath
234    and be importable using the import statement.
235
236    Example
237    path = get_pathname_from_package('anuga.utilities')
238
239    """
240
241    exec('import %s as x' %package)
242
243    path = x.__path__[0]
244   
245    return path
246
247    # Alternative approach that has been used at times
248    #try:
249    #    # When unit test is run from current dir
250    #    p1 = read_polygon('mainland_only.csv')
251    #except:
252    #    # When unit test is run from ANUGA root dir
253    #    from os.path import join, split
254    #    dir, tail = split(__file__)
255    #    path = join(dir, 'mainland_only.csv')
256    #    p1 = read_polygon(path)
257       
258           
259##
260# @brief Get list of variable names in an expression string.
261# @param source A string containing a python expression.
262# @return A list of variable name strings.
263# @note Throws SyntaxError exception if not a valid expression.
264def get_vars_in_expression(source):
265    '''Get list of variable names in a python expression.'''
266
267    import compiler
268    from compiler.ast import Node
269
270    ##
271    # @brief Internal recursive function.
272    # @param node An AST parse Node.
273    # @param var_list Input list of variables.
274    # @return An updated list of variables.
275    def get_vars_body(node, var_list=[]):
276        if isinstance(node, Node):
277            if node.__class__.__name__ == 'Name':
278                for child in node.getChildren():
279                    if child not in var_list:
280                        var_list.append(child)
281            for child in node.getChildren():
282                if isinstance(child, Node):
283                    for child in node.getChildren():
284                        var_list = get_vars_body(child, var_list)
285                    break
286
287        return var_list
288
289    return get_vars_body(compiler.parse(source))
290
291
292##
293# @brief Get a file from the web.
294# @param file_url URL of the file to fetch.
295# @param file_name Path to file to create in the filesystem.
296# @param auth Auth tuple (httpproxy, proxyuser, proxypass).
297# @param blocksize Read file in this block size.
298# @return 'auth' tuple for subsequent calls, if successful.
299# @note If 'auth' not supplied, will prompt user.
300# @note Will try using environment variable HTTP_PROXY for proxy server.
301# @note Will try using environment variable PROXY_USERNAME for proxy username.
302# @note Will try using environment variable PROXY_PASSWORD for proxy password.
303def get_web_file(file_url, file_name, auth=None, blocksize=1024*1024):
304    '''Get a file from the web.
305
306    file_url:  The URL of the file to get
307    file_name: Local path to save loaded file in
308    auth:      A tuple (httpproxy, proxyuser, proxypass)
309    blocksize: Block size of file reads
310   
311    Will try simple load through urllib first.  Drop down to urllib2
312    if there is a proxy and it requires authentication.
313
314    Environment variable HTTP_PROXY can be used to supply proxy information.
315    PROXY_USERNAME is used to supply the authentication username.
316    PROXY_PASSWORD supplies the password, if you dare!
317    '''
318
319    # Simple fetch, if fails, check for proxy error
320    try:
321        urllib.urlretrieve(file_url, file_name)     # original 'line of code'
322        return None     # no proxy, no auth required
323    except IOError, e:
324        if e[1] != 407:
325            raise       # raise error if *not* proxy auth error
326
327    # We get here if there was a proxy error, get file through the proxy
328
329    # unpack auth info
330    try:
331        (httpproxy, proxyuser, proxypass) = auth
332    except:
333        (httpproxy, proxyuser, proxypass) = (None, None, None)
334
335    # fill in any gaps from the environment
336    if httpproxy is None:
337        httpproxy = os.getenv('HTTP_PROXY')
338    if proxyuser is None:
339        proxyuser = os.getenv('PROXY_USERNAME')
340    if proxypass is None:
341        proxypass = os.getenv('PROXY_PASSWORD')
342
343    # Get auth info from user if still not supplied
344    if httpproxy is None or proxyuser is None or proxypass is None:
345        print '-'*80
346        print ('You need to supply proxy authentication information.  '
347               'Use environment variables')
348        print ('HTTP_PROXY, PROXY_USERNAME and PROXY_PASSWORD to bypass '
349               'entry here:')
350        if httpproxy is None:
351            httpproxy = raw_input('  proxy server: ')
352        if proxyuser is None:
353            proxyuser = raw_input('proxy username: ') 
354        if proxypass is None:
355            proxypass = getpass.getpass('proxy password: ')
356        print '-'*80
357
358    # the proxy URL cannot start with 'http://'
359    httpproxy = httpproxy.lower()
360    if httpproxy.startswith('http://'):
361        httpproxy = httpproxy.replace('http://', '', 1)
362
363    # open remote file
364    proxy = urllib2.ProxyHandler({'http': 'http://' + proxyuser
365                                              + ':' + proxypass
366                                              + '@' + httpproxy})
367    authinfo = urllib2.HTTPBasicAuthHandler()
368    opener = urllib2.build_opener(proxy, authinfo, urllib2.HTTPHandler)
369    urllib2.install_opener(opener)
370    webget = urllib2.urlopen(file_url)
371
372    # transfer file to local filesystem
373    fd = open(file_name, 'wb')
374    while True:
375        data = webget.read(blocksize)
376        if len(data) == 0:
377            break
378        fd.write(data)
379    fd.close
380    webget.close()
381
382    # return successful auth info
383    return (httpproxy, proxyuser, proxypass)
384
385
386##
387# @brief Tar a file (or directory) into a tarfile.
388# @param files A list of files (or directories) to tar.
389# @param tarfile The created tarfile name.
390# @note We use gzip compression.
391def tar_file(files, tarname):
392    '''Compress a file or directory into a tar file.'''
393
394    o = tarfile.open(tarname, 'w:gz')
395    for file in files:
396        o.add(file)
397    o.close()
398
399
400##
401# @brief Untar a file into an optional target directory.
402# @param tarname Name of the file to untar.
403# @param target_dir Directory to untar into.
404def untar_file(tarname, target_dir='.'):
405    '''Uncompress a tar file.'''
406
407    o = tarfile.open(tarname, 'r:gz')
408    members = o.getmembers()
409    for member in members:
410        o.extract(member, target_dir)
411    o.close()
412
413
414##
415# @brief Return a hex digest (MD5) of a given file.
416# @param filename Path to the file of interest.
417# @param blocksize Size of data blocks to read.
418# @return A hex digest string (16 bytes).
419# @note Uses MD5 digest.
420def get_file_hexdigest(filename, blocksize=1024*1024*10):
421    '''Get a hex digest of a file.'''
422   
423    m = md5.new()
424    fd = open(filename, 'r')
425           
426    while True:
427        data = fd.read(blocksize)
428        if len(data) == 0:
429            break
430        m.update(data)
431                                                               
432    fd.close()
433    return m.hexdigest()
434
435    fd = open(filename, 'r')
436
437
438##
439# @brief Create a file containing a hexdigest string of a data file.
440# @param data_file Path to the file to get the hexdigest from.
441# @param digest_file Path to hexdigest file to create.
442# @note Uses MD5 digest.
443def make_digest_file(data_file, digest_file):
444    '''Create a file containing the hex digest string of a data file.'''
445   
446    hexdigest = get_file_hexdigest(data_file)
447    fd = open(digest_file, 'w')
448    fd.write(hexdigest)
449    fd.close()
450
451##
452# @brief Function to return the length of a file.
453# @param in_file Path to file to get length of.
454# @return Number of lines in file.
455# @note Doesn't count '\n' characters.
456# @note Zero byte file, returns 0.
457# @note No \n in file at all, but >0 chars, returns 1.
458def file_length(in_file):
459    '''Function to return the length of a file.'''
460
461    fid = open(in_file)
462    data = fid.readlines()
463    fid.close()
464    return len(data)
465
466
Note: See TracBrowser for help on using the repository browser.