1 | """Implementation of tools to do with system administration made as platform independent as possible. |
---|
2 | |
---|
3 | |
---|
4 | """ |
---|
5 | |
---|
6 | import sys |
---|
7 | import os |
---|
8 | import urllib |
---|
9 | import urllib2 |
---|
10 | import getpass |
---|
11 | import tarfile |
---|
12 | import md5 |
---|
13 | |
---|
14 | |
---|
15 | def log_to_file(filename, s, verbose=False): |
---|
16 | """Log string to file name |
---|
17 | """ |
---|
18 | |
---|
19 | fid = open(filename, 'a') |
---|
20 | if verbose: print s |
---|
21 | fid.write(s + '\n') |
---|
22 | fid.close() |
---|
23 | |
---|
24 | |
---|
25 | def get_user_name(): |
---|
26 | """Get user name provide by operating system |
---|
27 | """ |
---|
28 | |
---|
29 | if sys.platform == 'win32': |
---|
30 | #user = os.getenv('USERPROFILE') |
---|
31 | user = os.getenv('USERNAME') |
---|
32 | else: |
---|
33 | user = os.getenv('LOGNAME') |
---|
34 | |
---|
35 | |
---|
36 | return user |
---|
37 | |
---|
38 | def get_host_name(): |
---|
39 | """Get host name provide by operating system |
---|
40 | """ |
---|
41 | |
---|
42 | if sys.platform == 'win32': |
---|
43 | host = os.getenv('COMPUTERNAME') |
---|
44 | else: |
---|
45 | host = os.uname()[1] |
---|
46 | |
---|
47 | |
---|
48 | return host |
---|
49 | |
---|
50 | def get_revision_number(): |
---|
51 | """Get the version number of the SVN |
---|
52 | NOTE: This requires that the command svn is on the system PATH |
---|
53 | (simply aliasing svn to the binary will not work) |
---|
54 | """ |
---|
55 | |
---|
56 | # Create dummy info |
---|
57 | #info = 'Revision: Version info could not be obtained.' |
---|
58 | #info += 'A command line version of svn must be availbable ' |
---|
59 | #info += 'on the system PATH, access to the subversion ' |
---|
60 | #info += 'repository is necessary and the output must ' |
---|
61 | #info += 'contain a line starting with "Revision:"' |
---|
62 | |
---|
63 | |
---|
64 | #FIXME (Ole): Change this so that svn info is attempted first. |
---|
65 | # If that fails, try to read a stored file with that same info (this would be created by e.g. the release script). Failing that, throw an exception. |
---|
66 | |
---|
67 | #FIXME (Ole): Move this and store_version_info to utilities |
---|
68 | |
---|
69 | |
---|
70 | try: |
---|
71 | from anuga.stored_version_info import version_info |
---|
72 | except: |
---|
73 | msg = 'No version info stored and command "svn" is not ' |
---|
74 | msg += 'recognised on the system PATH.\n\n' |
---|
75 | msg += 'If ANUGA has been installed from a distribution e.g. as ' |
---|
76 | msg += 'obtained from SourceForge,\n' |
---|
77 | msg += 'the version info should be ' |
---|
78 | msg += 'available in the automatically generated file ' |
---|
79 | msg += 'stored_version_info.py\n' |
---|
80 | msg += 'in the anuga root directory.\n' |
---|
81 | msg += 'If run from a Subversion sandpit, ' |
---|
82 | msg += 'ANUGA will try to obtain the version info ' |
---|
83 | msg += 'by using the command: "svn info".\n' |
---|
84 | msg += 'In this case, make sure the command line client ' |
---|
85 | msg += 'svn is accessible on the system path. ' |
---|
86 | msg += 'Simply aliasing svn to the binary will not work. ' |
---|
87 | msg += 'If you are using Windows, you have to install the file svn.exe ' |
---|
88 | msg += 'which can be obtained from e.g ' |
---|
89 | msg += 'http://www.collab.net/downloads/subversion' |
---|
90 | msg += 'Good luck!' |
---|
91 | |
---|
92 | # No file available - try using Subversion |
---|
93 | try: |
---|
94 | # The null stuff is so this section fails quitly. |
---|
95 | # This could cause the svn info command to fail due to |
---|
96 | # the redirection being bad on some platforms. |
---|
97 | # If that occurs then change this code. |
---|
98 | if sys.platform[0:3] == 'win': |
---|
99 | fid = os.popen('svn info 2> null') |
---|
100 | else: |
---|
101 | fid = os.popen('svn info 2>/dev/null') |
---|
102 | |
---|
103 | except: |
---|
104 | raise Exception(msg) |
---|
105 | else: |
---|
106 | #print 'Got version from svn' |
---|
107 | version_info = fid.read() |
---|
108 | |
---|
109 | if version_info == '': |
---|
110 | raise Exception(msg) |
---|
111 | else: |
---|
112 | pass |
---|
113 | #print 'Got version from file' |
---|
114 | |
---|
115 | |
---|
116 | for line in version_info.split('\n'): |
---|
117 | if line.startswith('Revision:'): |
---|
118 | break |
---|
119 | |
---|
120 | fields = line.split(':') |
---|
121 | msg = 'Keyword "Revision" was not found anywhere in text: %s' %version_info |
---|
122 | assert fields[0].startswith('Revision'), msg |
---|
123 | |
---|
124 | try: |
---|
125 | revision_number = int(fields[1]) |
---|
126 | except: |
---|
127 | msg = 'Revision number must be an integer. I got %s' %fields[1] |
---|
128 | msg += 'Check that the command svn is on the system path' |
---|
129 | raise Exception(msg) |
---|
130 | |
---|
131 | return revision_number |
---|
132 | |
---|
133 | |
---|
134 | def store_version_info(destination_path='.', verbose=False): |
---|
135 | """Obtain current version from Subversion and store it. |
---|
136 | |
---|
137 | Title: store_version_info() |
---|
138 | |
---|
139 | Author: Ole Nielsen (Ole.Nielsen@ga.gov.au) |
---|
140 | |
---|
141 | CreationDate: January 2006 |
---|
142 | |
---|
143 | Description: |
---|
144 | This function obtains current version from Subversion and stores it |
---|
145 | is a Python file named 'stored_version_info.py' for use with |
---|
146 | get_version_info() |
---|
147 | |
---|
148 | If svn is not available on the system PATH, an Exception is thrown |
---|
149 | """ |
---|
150 | |
---|
151 | # Note (Ole): This function should not be unit tested as it will only |
---|
152 | # work when running out of the sandpit. End users downloading the |
---|
153 | # ANUGA distribution would see a failure. |
---|
154 | # |
---|
155 | # FIXME: This function should really only be used by developers ( |
---|
156 | # (e.g. for creating new ANUGA releases), so maybe it should move |
---|
157 | # to somewhere else. |
---|
158 | |
---|
159 | import config |
---|
160 | |
---|
161 | try: |
---|
162 | fid = os.popen('svn info') |
---|
163 | except: |
---|
164 | msg = 'Command "svn" is not recognised on the system PATH' |
---|
165 | raise Exception(msg) |
---|
166 | else: |
---|
167 | txt = fid.read() |
---|
168 | fid.close() |
---|
169 | |
---|
170 | |
---|
171 | # Determine absolute filename |
---|
172 | if destination_path[-1] != os.sep: |
---|
173 | destination_path += os.sep |
---|
174 | |
---|
175 | filename = destination_path + config.version_filename |
---|
176 | |
---|
177 | fid = open(filename, 'w') |
---|
178 | |
---|
179 | docstring = 'Stored version info.\n\n' |
---|
180 | docstring += 'This file provides the version for distributions ' |
---|
181 | docstring += 'that are not accessing Subversion directly.\n' |
---|
182 | docstring += 'The file is automatically generated and should not ' |
---|
183 | docstring += 'be modified manually.\n' |
---|
184 | fid.write('"""%s"""\n\n' %docstring) |
---|
185 | |
---|
186 | fid.write('version_info = """\n%s"""' %txt) |
---|
187 | fid.close() |
---|
188 | |
---|
189 | |
---|
190 | if verbose is True: |
---|
191 | print 'Version info stored to %s' %filename |
---|
192 | |
---|
193 | def safe_crc(string): |
---|
194 | """64 bit safe crc computation. |
---|
195 | |
---|
196 | See Guido's 64 bit fix at http://bugs.python.org/issue1202 |
---|
197 | """ |
---|
198 | |
---|
199 | from zlib import crc32 |
---|
200 | import os |
---|
201 | |
---|
202 | x = crc32(string) |
---|
203 | |
---|
204 | if os.name == 'posix' and os.uname()[4] in ['x86_64', 'ia64']: |
---|
205 | crcval = x - ((x & 0x80000000) << 1) |
---|
206 | else: |
---|
207 | crcval = x |
---|
208 | |
---|
209 | return crcval |
---|
210 | |
---|
211 | |
---|
212 | def compute_checksum(filename, max_length=2**20): |
---|
213 | """Compute the CRC32 checksum for specified file |
---|
214 | |
---|
215 | Optional parameter max_length sets the maximum number |
---|
216 | of bytes used to limit time used with large files. |
---|
217 | Default = 2**20 (1MB) |
---|
218 | """ |
---|
219 | |
---|
220 | fid = open(filename, 'rb') # Use binary for portability |
---|
221 | crcval = safe_crc(fid.read(max_length)) |
---|
222 | fid.close() |
---|
223 | |
---|
224 | return crcval |
---|
225 | |
---|
226 | def get_pathname_from_package(package): |
---|
227 | """Get pathname of given package (provided as string) |
---|
228 | |
---|
229 | This is useful for reading files residing in the same directory as |
---|
230 | a particular module. Typically, this is required in unit tests depending |
---|
231 | on external files. |
---|
232 | |
---|
233 | The given module must start from a directory on the pythonpath |
---|
234 | and be importable using the import statement. |
---|
235 | |
---|
236 | Example |
---|
237 | path = get_pathname_from_package('anuga.utilities') |
---|
238 | |
---|
239 | """ |
---|
240 | |
---|
241 | exec('import %s as x' %package) |
---|
242 | |
---|
243 | path = x.__path__[0] |
---|
244 | |
---|
245 | return path |
---|
246 | |
---|
247 | # Alternative approach that has been used at times |
---|
248 | #try: |
---|
249 | # # When unit test is run from current dir |
---|
250 | # p1 = read_polygon('mainland_only.csv') |
---|
251 | #except: |
---|
252 | # # When unit test is run from ANUGA root dir |
---|
253 | # from os.path import join, split |
---|
254 | # dir, tail = split(__file__) |
---|
255 | # path = join(dir, 'mainland_only.csv') |
---|
256 | # p1 = read_polygon(path) |
---|
257 | |
---|
258 | |
---|
259 | ## |
---|
260 | # @brief Get list of variable names in an expression string. |
---|
261 | # @param source A string containing a python expression. |
---|
262 | # @return A list of variable name strings. |
---|
263 | # @note Throws SyntaxError exception if not a valid expression. |
---|
264 | def get_vars_in_expression(source): |
---|
265 | '''Get list of variable names in a python expression.''' |
---|
266 | |
---|
267 | import compiler |
---|
268 | from compiler.ast import Node |
---|
269 | |
---|
270 | ## |
---|
271 | # @brief Internal recursive function. |
---|
272 | # @param node An AST parse Node. |
---|
273 | # @param var_list Input list of variables. |
---|
274 | # @return An updated list of variables. |
---|
275 | def get_vars_body(node, var_list=[]): |
---|
276 | if isinstance(node, Node): |
---|
277 | if node.__class__.__name__ == 'Name': |
---|
278 | for child in node.getChildren(): |
---|
279 | if child not in var_list: |
---|
280 | var_list.append(child) |
---|
281 | for child in node.getChildren(): |
---|
282 | if isinstance(child, Node): |
---|
283 | for child in node.getChildren(): |
---|
284 | var_list = get_vars_body(child, var_list) |
---|
285 | break |
---|
286 | |
---|
287 | return var_list |
---|
288 | |
---|
289 | return get_vars_body(compiler.parse(source)) |
---|
290 | |
---|
291 | |
---|
292 | ## |
---|
293 | # @brief Get a file from the web. |
---|
294 | # @param file_url URL of the file to fetch. |
---|
295 | # @param file_name Path to file to create in the filesystem. |
---|
296 | # @param auth Auth tuple (httpproxy, proxyuser, proxypass). |
---|
297 | # @param blocksize Read file in this block size. |
---|
298 | # @return 'auth' tuple for subsequent calls, if successful. |
---|
299 | # @note If 'auth' not supplied, will prompt user. |
---|
300 | # @note Will try using environment variable HTTP_PROXY for proxy server. |
---|
301 | # @note Will try using environment variable PROXY_USERNAME for proxy username. |
---|
302 | # @note Will try using environment variable PROXY_PASSWORD for proxy password. |
---|
303 | def get_web_file(file_url, file_name, auth=None, blocksize=1024*1024): |
---|
304 | '''Get a file from the web. |
---|
305 | |
---|
306 | file_url: The URL of the file to get |
---|
307 | file_name: Local path to save loaded file in |
---|
308 | auth: A tuple (httpproxy, proxyuser, proxypass) |
---|
309 | blocksize: Block size of file reads |
---|
310 | |
---|
311 | Will try simple load through urllib first. Drop down to urllib2 |
---|
312 | if there is a proxy and it requires authentication. |
---|
313 | |
---|
314 | Environment variable HTTP_PROXY can be used to supply proxy information. |
---|
315 | PROXY_USERNAME is used to supply the authentication username. |
---|
316 | PROXY_PASSWORD supplies the password, if you dare! |
---|
317 | ''' |
---|
318 | |
---|
319 | # Simple fetch, if fails, check for proxy error |
---|
320 | try: |
---|
321 | urllib.urlretrieve(file_url, file_name) # original 'line of code' |
---|
322 | return None # no proxy, no auth required |
---|
323 | except IOError, e: |
---|
324 | if e[1] != 407: |
---|
325 | raise # raise error if *not* proxy auth error |
---|
326 | |
---|
327 | # We get here if there was a proxy error, get file through the proxy |
---|
328 | |
---|
329 | # unpack auth info |
---|
330 | try: |
---|
331 | (httpproxy, proxyuser, proxypass) = auth |
---|
332 | except: |
---|
333 | (httpproxy, proxyuser, proxypass) = (None, None, None) |
---|
334 | |
---|
335 | # fill in any gaps from the environment |
---|
336 | if httpproxy is None: |
---|
337 | httpproxy = os.getenv('HTTP_PROXY') |
---|
338 | if proxyuser is None: |
---|
339 | proxyuser = os.getenv('PROXY_USERNAME') |
---|
340 | if proxypass is None: |
---|
341 | proxypass = os.getenv('PROXY_PASSWORD') |
---|
342 | |
---|
343 | # Get auth info from user if still not supplied |
---|
344 | if httpproxy is None or proxyuser is None or proxypass is None: |
---|
345 | print '-'*80 |
---|
346 | print ('You need to supply proxy authentication information. ' |
---|
347 | 'Use environment variables\n' |
---|
348 | 'HTTP_PROXY, PROXY_USERNAME and PROXY_PASSWORD to bypass ' |
---|
349 | 'entry here:') |
---|
350 | if httpproxy is None: |
---|
351 | httpproxy = raw_input(' proxy server: ') |
---|
352 | if proxyuser is None: |
---|
353 | proxyuser = raw_input('proxy username: ') |
---|
354 | if proxypass is None: |
---|
355 | proxypass = getpass.getpass('proxy password: ') |
---|
356 | print '-'*80 |
---|
357 | |
---|
358 | # the proxy URL cannot start with 'http://', we add that later |
---|
359 | httpproxy = httpproxy.lower() |
---|
360 | if httpproxy.startswith('http://'): |
---|
361 | httpproxy = httpproxy.replace('http://', '', 1) |
---|
362 | |
---|
363 | # open remote file |
---|
364 | proxy = urllib2.ProxyHandler({'http': 'http://' + proxyuser |
---|
365 | + ':' + proxypass |
---|
366 | + '@' + httpproxy}) |
---|
367 | authinfo = urllib2.HTTPBasicAuthHandler() |
---|
368 | opener = urllib2.build_opener(proxy, authinfo, urllib2.HTTPHandler) |
---|
369 | urllib2.install_opener(opener) |
---|
370 | webget = urllib2.urlopen(file_url) |
---|
371 | |
---|
372 | # transfer file to local filesystem |
---|
373 | fd = open(file_name, 'wb') |
---|
374 | while True: |
---|
375 | data = webget.read(blocksize) |
---|
376 | if len(data) == 0: |
---|
377 | break |
---|
378 | fd.write(data) |
---|
379 | fd.close |
---|
380 | webget.close() |
---|
381 | |
---|
382 | # return successful auth info |
---|
383 | return (httpproxy, proxyuser, proxypass) |
---|
384 | |
---|
385 | |
---|
386 | ## |
---|
387 | # @brief Tar a file (or directory) into a tarfile. |
---|
388 | # @param files A list of files (or directories) to tar. |
---|
389 | # @param tarfile The created tarfile name. |
---|
390 | # @note We use gzip compression. |
---|
391 | def tar_file(files, tarname): |
---|
392 | '''Compress a file or directory into a tar file.''' |
---|
393 | |
---|
394 | o = tarfile.open(tarname, 'w:gz') |
---|
395 | for file in files: |
---|
396 | o.add(file) |
---|
397 | o.close() |
---|
398 | |
---|
399 | |
---|
400 | ## |
---|
401 | # @brief Untar a file into an optional target directory. |
---|
402 | # @param tarname Name of the file to untar. |
---|
403 | # @param target_dir Directory to untar into. |
---|
404 | def untar_file(tarname, target_dir='.'): |
---|
405 | '''Uncompress a tar file.''' |
---|
406 | |
---|
407 | o = tarfile.open(tarname, 'r:gz') |
---|
408 | members = o.getmembers() |
---|
409 | for member in members:Compute normal depth at gauge location usin |
---|
410 | o.extract(member, target_dir) |
---|
411 | o.close() |
---|
412 | |
---|
413 | |
---|
414 | ## |
---|
415 | # @brief Return a hex digest (MD5) of a given file. |
---|
416 | # @param filename Path to the file of interest. |
---|
417 | # @param blocksize Size of data blocks to read. |
---|
418 | # @return A hex digest string (16 bytes). |
---|
419 | # @note Uses MD5 digest. |
---|
420 | def get_file_hexdigest(filename, blocksize=1024*1024*10): |
---|
421 | '''Get a hex digest of a file.''' |
---|
422 | |
---|
423 | m = md5.new() |
---|
424 | fd = open(filename, 'r') |
---|
425 | |
---|
426 | while True: |
---|
427 | data = fd.read(blocksize) |
---|
428 | if len(data) == 0: |
---|
429 | break |
---|
430 | m.update(data) |
---|
431 | |
---|
432 | fd.close() |
---|
433 | return m.hexdigest() |
---|
434 | |
---|
435 | fd = open(filename, 'r') |
---|
436 | |
---|
437 | |
---|
438 | ## |
---|
439 | # @brief Create a file containing a hexdigest string of a data file. |
---|
440 | # @param data_file Path to the file to get the hexdigest from. |
---|
441 | # @param digest_file Path to hexdigest file to create. |
---|
442 | # @note Uses MD5 digest. |
---|
443 | def make_digest_file(data_file, digest_file): |
---|
444 | '''Create a file containing the hex digest string of a data file.''' |
---|
445 | |
---|
446 | hexdigest = get_file_hexdigest(data_file) |
---|
447 | fd = open(digest_file, 'w') |
---|
448 | fd.write(hexdigest) |
---|
449 | fd.close() |
---|
450 | |
---|
451 | ## |
---|
452 | # @brief Function to return the length of a file. |
---|
453 | # @param in_file Path to file to get length of. |
---|
454 | # @return Number of lines in file. |
---|
455 | # @note Doesn't count '\n' characters. |
---|
456 | # @note Zero byte file, returns 0. |
---|
457 | # @note No \n in file at all, but >0 chars, returns 1. |
---|
458 | def file_length(in_file): |
---|
459 | '''Function to return the length of a file.''' |
---|
460 | |
---|
461 | fid = open(in_file) |
---|
462 | data = fid.readlines() |
---|
463 | fid.close() |
---|
464 | return len(data) |
---|
465 | |
---|
466 | |
---|