source: anuga_core/source/anuga/utilities/system_tools.py @ 6612

Last change on this file since 6612 was 6612, checked in by rwilson, 15 years ago

Added a 'get a file from the web' function. Part of patong validation.

File size: 10.9 KB
Line 
1"""Implementation of tools to do with system administration made as platform independent as possible.
2
3
4"""
5
6import sys
7import os
8import urllib
9import urllib2
10import getpass
11
12
13def log_to_file(filename, s, verbose=False):
14    """Log string to file name
15    """
16
17    fid = open(filename, 'a')
18    if verbose: print s
19    fid.write(s + '\n')
20    fid.close()
21
22
23def get_user_name():
24    """Get user name provide by operating system
25    """
26
27    if sys.platform == 'win32':
28        #user = os.getenv('USERPROFILE')
29        user = os.getenv('USERNAME')
30    else:
31        user = os.getenv('LOGNAME')
32
33
34    return user   
35
36def get_host_name():
37    """Get host name provide by operating system
38    """
39
40    if sys.platform == 'win32':
41        host = os.getenv('COMPUTERNAME')
42    else:
43        host = os.uname()[1]
44
45
46    return host   
47
48def get_revision_number():
49    """Get the version number of the SVN
50    NOTE: This requires that the command svn is on the system PATH
51    (simply aliasing svn to the binary will not work)
52    """
53
54    # Create dummy info
55    #info = 'Revision: Version info could not be obtained.'
56    #info += 'A command line version of svn must be availbable '
57    #info += 'on the system PATH, access to the subversion '
58    #info += 'repository is necessary and the output must '
59    #info += 'contain a line starting with "Revision:"'
60   
61
62    #FIXME (Ole): Change this so that svn info is attempted first.
63    # If that fails, try to read a stored file with that same info (this would be created by e.g. the release script). Failing that, throw an exception.
64
65    #FIXME (Ole): Move this and store_version_info to utilities
66
67
68    try:
69        from anuga.stored_version_info import version_info
70    except:
71        msg = 'No version info stored and command "svn" is not '
72        msg += 'recognised on the system PATH.\n\n'
73        msg += 'If ANUGA has been installed from a distribution e.g. as '
74        msg += 'obtained from SourceForge,\n'
75        msg += 'the version info should be '
76        msg += 'available in the automatically generated file '
77        msg += 'stored_version_info.py\n'
78        msg += 'in the anuga root directory.\n'
79        msg += 'If run from a Subversion sandpit, '
80        msg += 'ANUGA will try to obtain the version info '
81        msg += 'by using the command: "svn info".\n'
82        msg += 'In this case, make sure svn is accessible on the system path. '
83        msg += 'Simply aliasing svn to the binary will not work. '
84        msg += 'Good luck!'
85
86        # No file available - try using Subversion
87        try:
88            # The null stuff is so this section fails quitly.
89            # This could cause the svn info command to fail due to
90            # the redirection being bad on some platforms.
91            # If that occurs then change this code.
92            if sys.platform[0:3] == 'win':
93                fid = os.popen('svn info 2> null')
94            else:
95                fid = os.popen('svn info 2>/dev/null')
96       
97        except:
98            raise Exception(msg)
99        else:
100            #print 'Got version from svn'           
101            version_info = fid.read()
102           
103            if version_info == '':
104                raise Exception(msg)   
105    else:
106        pass
107        #print 'Got version from file'
108
109           
110    for line in version_info.split('\n'):
111        if line.startswith('Revision:'):
112            break
113
114    fields = line.split(':')
115    msg = 'Keyword "Revision" was not found anywhere in text: %s' %version_info
116    assert fields[0].startswith('Revision'), msg           
117
118    try:
119        revision_number = int(fields[1])
120    except:
121        msg = 'Revision number must be an integer. I got %s' %fields[1]
122        msg += 'Check that the command svn is on the system path' 
123        raise Exception(msg)               
124       
125    return revision_number
126
127
128def store_version_info(destination_path='.', verbose=False):
129    """Obtain current version from Subversion and store it.
130   
131    Title: store_version_info()
132
133    Author: Ole Nielsen (Ole.Nielsen@ga.gov.au)
134
135    CreationDate: January 2006
136
137    Description:
138        This function obtains current version from Subversion and stores it
139        is a Python file named 'stored_version_info.py' for use with
140        get_version_info()
141
142        If svn is not available on the system PATH, an Exception is thrown
143    """
144
145    # Note (Ole): This function should not be unit tested as it will only
146    # work when running out of the sandpit. End users downloading the
147    # ANUGA distribution would see a failure.
148    #
149    # FIXME: This function should really only be used by developers (
150    # (e.g. for creating new ANUGA releases), so maybe it should move
151    # to somewhere else.
152   
153    import config
154
155    try:
156        fid = os.popen('svn info')
157    except:
158        msg = 'Command "svn" is not recognised on the system PATH'
159        raise Exception(msg)
160    else:   
161        txt = fid.read()
162        fid.close()
163
164
165        # Determine absolute filename
166        if destination_path[-1] != os.sep:
167            destination_path += os.sep
168           
169        filename = destination_path + config.version_filename
170
171        fid = open(filename, 'w')
172
173        docstring = 'Stored version info.\n\n'
174        docstring += 'This file provides the version for distributions '
175        docstring += 'that are not accessing Subversion directly.\n'
176        docstring += 'The file is automatically generated and should not '
177        docstring += 'be modified manually.\n'
178        fid.write('"""%s"""\n\n' %docstring)
179       
180        fid.write('version_info = """\n%s"""' %txt)
181        fid.close()
182
183
184        if verbose is True:
185            print 'Version info stored to %s' %filename
186
187def safe_crc(string):
188    """64 bit safe crc computation.
189
190       See Guido's 64 bit fix at http://bugs.python.org/issue1202           
191    """
192
193    from zlib import crc32
194    import os
195
196    x = crc32(string)
197       
198    if os.name == 'posix' and os.uname()[4] in ['x86_64', 'ia64']:
199        crcval = x - ((x & 0x80000000) << 1)
200    else:
201        crcval = x
202       
203    return crcval
204
205
206def compute_checksum(filename, max_length=2**20):
207    """Compute the CRC32 checksum for specified file
208
209    Optional parameter max_length sets the maximum number
210    of bytes used to limit time used with large files.
211    Default = 2**20 (1MB)
212    """
213
214    fid = open(filename, 'rb') # Use binary for portability
215    crcval = safe_crc(fid.read(max_length))
216    fid.close()
217
218    return crcval
219
220def get_pathname_from_package(package):
221    """Get pathname of given package (provided as string)
222
223    This is useful for reading files residing in the same directory as
224    a particular module. Typically, this is required in unit tests depending
225    on external files.
226
227    The given module must start from a directory on the pythonpath
228    and be importable using the import statement.
229
230    Example
231    path = get_pathname_from_package('anuga.utilities')
232
233    """
234
235    exec('import %s as x' %package)
236
237    path = x.__path__[0]
238   
239    return path
240
241    # Alternative approach that has been used at times
242    #try:
243    #    # When unit test is run from current dir
244    #    p1 = read_polygon('mainland_only.csv')
245    #except:
246    #    # When unit test is run from ANUGA root dir
247    #    from os.path import join, split
248    #    dir, tail = split(__file__)
249    #    path = join(dir, 'mainland_only.csv')
250    #    p1 = read_polygon(path)
251       
252           
253##
254# @brief Get list of variable names in an expression string.
255# @param source A string containing a python expression.
256# @return A list of variable name strings.
257# @note Throws SyntaxError exception if not a valid expression.
258def get_vars_in_expression(source):
259    '''Get list of variable names in a python expression.'''
260
261    import compiler
262    from compiler.ast import Node
263
264    ##
265    # @brief Internal recursive function.
266    # @param node An AST parse Node.
267    # @param var_list Input list of variables.
268    # @return An updated list of variables.
269    def get_vars_body(node, var_list=[]):
270        if isinstance(node, Node):
271            if node.__class__.__name__ == 'Name':
272                for child in node.getChildren():
273                    if child not in var_list:
274                        var_list.append(child)
275            for child in node.getChildren():
276                if isinstance(child, Node):
277                    for child in node.getChildren():
278                        var_list = get_vars_body(child, var_list)
279                    break
280
281        return var_list
282
283    return get_vars_body(compiler.parse(source))
284
285
286##
287# @brief Get a file from the web.
288# @param file_url URL of the file to fetch.
289# @param file_name Path to file to create in the filesystem.
290# @param blocksize Read file in this block size.
291# @param auth Auth tuple (httpproxy, proxyuser, proxypass).
292# @return 'auth' tuple for subsequent calls, if successful.
293# @note If 'auth' not supplied, will prompt user.
294# @note Will try using environment variable HTTP_PROXY for proxy server.
295# @note Will try using environment variable PROXY_USERNAME for proxy username.
296# @note Will try using environment variable PROXY_PASSWORD for proxy password.
297def get_web_file(file_url, file_name, blocksize=1024*1024, auth=None):
298    '''Get a file from the web.'''
299
300    # Simple fetch, if fails, check for proxy error
301    try:
302        urllib.urlretrieve(file_url, file_name)
303        return None     # no proxy, no auth required
304    except IOError, e:
305        print str(e)
306        if e[1] != 407:
307            raise       # raise error if *not* proxy auth error
308
309    # We get here if there was a proxy error, get file through the proxy
310
311    # unpack auth info
312    try:
313        (httpproxy, proxyuser, proxypass) = auth
314    except:
315        (httpproxy, proxyuser, proxypass) = (None, None, None)
316
317    # fill in any gaps from the environment
318    if httpproxy is None:
319        httpproxy = os.getenv('HTTP_PROXY')
320    if proxyuser is None:
321        proxyuser = os.getenv('PROXY_USERNAME')
322    if proxypass is None:
323        proxypass = os.getenv('PROXY_PASSWORD')
324
325    # Get auth info from user if still not supplied
326    if httpproxy is None or proxyuser is None or proxypass is None:
327        print 'You need to supply proxy authentication information:'
328        if httpproxy is None:
329            httpproxy = raw_input('  proxy server: ')
330        if proxyuser is None:
331            proxyuser = raw_input('proxy username: ') 
332        if proxypass is None:
333            proxypass = getpass.getpass('proxy password: ')
334
335    # the proxy URL cannot start with 'http://'
336    httpproxy = httpproxy.lower()
337    if httpproxy.startswith('http://'):
338        httpproxy = httpproxy.replace('http://', '', 1)
339
340    # open 'net file
341    proxy = urllib2.ProxyHandler({'http': 'http://'+proxyuser+':'+proxypass+'@'+httpproxy})
342    authinfo = urllib2.HTTPBasicAuthHandler()
343    opener = urllib2.build_opener(proxy, authinfo, urllib2.HTTPHandler)
344    urllib2.install_opener(opener)
345    webget = urllib2.urlopen(file_url)
346
347    # transfer file to local filesystem
348    fd = open(file_name, 'w')
349    while True:
350        data = webget.read(blocksize)
351        if len(data) == 0:
352            break
353        fd.write(data)
354    fd.close
355    webget.close()
356
357    # return successful auth info
358    return (httpproxy, proxyuser, proxypass)
359
360
Note: See TracBrowser for help on using the repository browser.