Changeset 6612


Ignore:
Timestamp:
Mar 25, 2009, 12:11:00 PM (15 years ago)
Author:
rwilson
Message:

Added a 'get a file from the web' function. Part of patong validation.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • anuga_core/source/anuga/utilities/system_tools.py

    r6575 r6612  
    66import sys
    77import os
     8import urllib
     9import urllib2
     10import getpass
     11
    812
    913def log_to_file(filename, s, verbose=False):
     
    274278                        var_list = get_vars_body(child, var_list)
    275279                    break
    276 ##            if any(isinstance(child, Node) for child in node.getChildren()):
    277 ##                for child in node.getChildren():
    278 ##                    var_list = get_vars_body(child, var_list)
    279280
    280281        return var_list
    281282
    282283    return get_vars_body(compiler.parse(source))
     284
     285
     286##
     287# @brief Get a file from the web.
     288# @param file_url URL of the file to fetch.
     289# @param file_name Path to file to create in the filesystem.
     290# @param blocksize Read file in this block size.
     291# @param auth Auth tuple (httpproxy, proxyuser, proxypass).
     292# @return 'auth' tuple for subsequent calls, if successful.
     293# @note If 'auth' not supplied, will prompt user.
     294# @note Will try using environment variable HTTP_PROXY for proxy server.
     295# @note Will try using environment variable PROXY_USERNAME for proxy username.
     296# @note Will try using environment variable PROXY_PASSWORD for proxy password.
     297def get_web_file(file_url, file_name, blocksize=1024*1024, auth=None):
     298    '''Get a file from the web.'''
     299
     300    # Simple fetch, if fails, check for proxy error
     301    try:
     302        urllib.urlretrieve(file_url, file_name)
     303        return None     # no proxy, no auth required
     304    except IOError, e:
     305        print str(e)
     306        if e[1] != 407:
     307            raise       # raise error if *not* proxy auth error
     308
     309    # We get here if there was a proxy error, get file through the proxy
     310
     311    # unpack auth info
     312    try:
     313        (httpproxy, proxyuser, proxypass) = auth
     314    except:
     315        (httpproxy, proxyuser, proxypass) = (None, None, None)
     316
     317    # fill in any gaps from the environment
     318    if httpproxy is None:
     319        httpproxy = os.getenv('HTTP_PROXY')
     320    if proxyuser is None:
     321        proxyuser = os.getenv('PROXY_USERNAME')
     322    if proxypass is None:
     323        proxypass = os.getenv('PROXY_PASSWORD')
     324
     325    # Get auth info from user if still not supplied
     326    if httpproxy is None or proxyuser is None or proxypass is None:
     327        print 'You need to supply proxy authentication information:'
     328        if httpproxy is None:
     329            httpproxy = raw_input('  proxy server: ')
     330        if proxyuser is None:
     331            proxyuser = raw_input('proxy username: ')
     332        if proxypass is None:
     333            proxypass = getpass.getpass('proxy password: ')
     334
     335    # the proxy URL cannot start with 'http://'
     336    httpproxy = httpproxy.lower()
     337    if httpproxy.startswith('http://'):
     338        httpproxy = httpproxy.replace('http://', '', 1)
     339
     340    # open 'net file
     341    proxy = urllib2.ProxyHandler({'http': 'http://'+proxyuser+':'+proxypass+'@'+httpproxy})
     342    authinfo = urllib2.HTTPBasicAuthHandler()
     343    opener = urllib2.build_opener(proxy, authinfo, urllib2.HTTPHandler)
     344    urllib2.install_opener(opener)
     345    webget = urllib2.urlopen(file_url)
     346
     347    # transfer file to local filesystem
     348    fd = open(file_name, 'w')
     349    while True:
     350        data = webget.read(blocksize)
     351        if len(data) == 0:
     352            break
     353        fd.write(data)
     354    fd.close
     355    webget.close()
     356
     357    # return successful auth info
     358    return (httpproxy, proxyuser, proxypass)
     359
     360
Note: See TracChangeset for help on using the changeset viewer.