| 1 | """Track IP of data files included in this distribution. |
|---|
| 2 | """ |
|---|
| 3 | |
|---|
| 4 | from os import remove, walk, sep |
|---|
| 5 | from os.path import join, splitext |
|---|
| 6 | |
|---|
| 7 | from anuga.utilities.xml_tools import parse, pretty_print_tree, get_elements, get_text |
|---|
| 8 | from anuga.utilities.system_tools import compute_checksum |
|---|
| 9 | |
|---|
| 10 | # Audit exceptions |
|---|
| 11 | class NotPublishable(Exception): pass |
|---|
| 12 | class FilenameMismatch(Exception): pass |
|---|
| 13 | class CRCMismatch(Exception): pass |
|---|
| 14 | class Invalid(Exception): pass |
|---|
| 15 | class WrongTags(Exception): pass |
|---|
| 16 | |
|---|
| 17 | audit_exceptions = (NotPublishable, FilenameMismatch, CRCMismatch, Invalid, WrongTags) |
|---|
| 18 | |
|---|
| 19 | def IP_verified(directory, verbose=False): |
|---|
| 20 | """Find and audit potential data files that might violate IP |
|---|
| 21 | |
|---|
| 22 | This is the public function to be used to ascertain that |
|---|
| 23 | all data in the specified directory tree has been audited according |
|---|
| 24 | to the GA data IP tracking process. |
|---|
| 25 | |
|---|
| 26 | if IP_verified is False: |
|---|
| 27 | # Stop and take remedial action |
|---|
| 28 | ... |
|---|
| 29 | else: |
|---|
| 30 | # Proceed boldly with confidence |
|---|
| 31 | |
|---|
| 32 | verbose controls standard output. |
|---|
| 33 | If verbose is False, only diagnostics about failed audits will appear. |
|---|
| 34 | All files that check OK will pass silently. |
|---|
| 35 | |
|---|
| 36 | |
|---|
| 37 | """ |
|---|
| 38 | |
|---|
| 39 | print '---------------------------------------------' |
|---|
| 40 | print 'Files that need to be assessed for IP issues:' |
|---|
| 41 | print '---------------------------------------------' |
|---|
| 42 | |
|---|
| 43 | # Print header |
|---|
| 44 | dirwidth = 72 |
|---|
| 45 | print '---------------------------------------------' |
|---|
| 46 | print 'File'.ljust(dirwidth), 'Status' |
|---|
| 47 | print '---------------------------------------------' |
|---|
| 48 | |
|---|
| 49 | # Identify data files |
|---|
| 50 | all_files_accounted_for = True |
|---|
| 51 | for dirpath, datafile in identify_datafiles(directory): |
|---|
| 52 | |
|---|
| 53 | filename = join(dirpath, datafile) |
|---|
| 54 | basename, ext = splitext(datafile) |
|---|
| 55 | |
|---|
| 56 | # Look for a XML license file with the .lic |
|---|
| 57 | status = 'OK' |
|---|
| 58 | try: |
|---|
| 59 | fid = open(join(dirpath, basename + '.lic')) |
|---|
| 60 | except IOError: |
|---|
| 61 | status = 'NO LICENSE FILE' |
|---|
| 62 | all_files_accounted_for = False |
|---|
| 63 | else: |
|---|
| 64 | try: |
|---|
| 65 | license_file_is_valid(fid, dirpath, verbose=verbose) |
|---|
| 66 | except audit_exceptions, e: |
|---|
| 67 | all_files_accounted_for = False |
|---|
| 68 | status = 'LICENSE FILE NOT VALID' |
|---|
| 69 | status += 'REASON: %s' %e |
|---|
| 70 | |
|---|
| 71 | #doc = parse(fid) |
|---|
| 72 | #pretty_print_tree(doc) |
|---|
| 73 | fid.seek(0) |
|---|
| 74 | status += fid.read() |
|---|
| 75 | |
|---|
| 76 | #else: |
|---|
| 77 | # if verbose: print 'OK' |
|---|
| 78 | |
|---|
| 79 | fid.close() |
|---|
| 80 | |
|---|
| 81 | if status != 'OK' or verbose is True: |
|---|
| 82 | print filename + ' (Checksum=%s): '\ |
|---|
| 83 | %str(compute_checksum(filename)), status |
|---|
| 84 | |
|---|
| 85 | |
|---|
| 86 | |
|---|
| 87 | # Return result |
|---|
| 88 | return all_files_accounted_for |
|---|
| 89 | |
|---|
| 90 | |
|---|
| 91 | def identify_datafiles(root): |
|---|
| 92 | """ Identify files that might contain data |
|---|
| 93 | """ |
|---|
| 94 | |
|---|
| 95 | # Ignore source code files |
|---|
| 96 | extensions_to_ignore = ['.py','.c','.h', '.f'] #, '.gif', '.jpg', '.png'] |
|---|
| 97 | |
|---|
| 98 | # Ignore generated stuff |
|---|
| 99 | extensions_to_ignore += ['.pyc', '.o', '.so', '~'] |
|---|
| 100 | extensions_to_ignore += ['.aux', '.log', '.idx', 'ilg', '.ind', |
|---|
| 101 | '.bbl', '.blg'] |
|---|
| 102 | |
|---|
| 103 | # Ignore license files themselves |
|---|
| 104 | extensions_to_ignore += ['.lic'] |
|---|
| 105 | |
|---|
| 106 | |
|---|
| 107 | # Ignore certain other files |
|---|
| 108 | files_to_ignore = ['README.txt'] |
|---|
| 109 | |
|---|
| 110 | # Ignore directories |
|---|
| 111 | directories_to_ignore = ['anuga_work', 'pymetis', 'obsolete_code', |
|---|
| 112 | 'anuga_parallel', 'anuga_viewer', |
|---|
| 113 | 'planning', 'coding_standards', |
|---|
| 114 | 'experimentation', |
|---|
| 115 | '.svn', 'misc', '.metadata'] |
|---|
| 116 | |
|---|
| 117 | for dirpath, dirnames, filenames in walk(root): |
|---|
| 118 | |
|---|
| 119 | for ignore in directories_to_ignore: |
|---|
| 120 | if ignore in dirnames: |
|---|
| 121 | dirnames.remove(ignore) # don't visit ignored directories |
|---|
| 122 | |
|---|
| 123 | #print 'Searching dir', dirpath |
|---|
| 124 | |
|---|
| 125 | |
|---|
| 126 | for filename in filenames: |
|---|
| 127 | |
|---|
| 128 | |
|---|
| 129 | # Ignore extensions that need no IP check |
|---|
| 130 | ignore = False |
|---|
| 131 | for ext in extensions_to_ignore: |
|---|
| 132 | if filename.endswith(ext): |
|---|
| 133 | ignore = True |
|---|
| 134 | |
|---|
| 135 | if filename in files_to_ignore: |
|---|
| 136 | ignore = True |
|---|
| 137 | |
|---|
| 138 | if ignore is False: |
|---|
| 139 | yield dirpath, filename |
|---|
| 140 | |
|---|
| 141 | |
|---|
| 142 | def license_file_is_valid(fid, dirpath='.', verbose=False): |
|---|
| 143 | """Check that XML license file is valid |
|---|
| 144 | """ |
|---|
| 145 | |
|---|
| 146 | license_filename = fid.name |
|---|
| 147 | doc = parse(fid) |
|---|
| 148 | #print_tree(doc) |
|---|
| 149 | |
|---|
| 150 | # Check that file is valid (e.g. all elements there) |
|---|
| 151 | # FIXME (Ole): Todo |
|---|
| 152 | |
|---|
| 153 | |
|---|
| 154 | if doc.nodeName != '#document': |
|---|
| 155 | msg = 'License file %s does not appear' %license_filename |
|---|
| 156 | msg += 'to be a valid XML document' |
|---|
| 157 | msg += 'The root node has name %s' %doc.nodeName |
|---|
| 158 | msg += 'but it should be %s' %'#document' |
|---|
| 159 | raise Invalid, msg |
|---|
| 160 | |
|---|
| 161 | if len(doc.childNodes) != 1: |
|---|
| 162 | msg = 'License file %s must have only one element' %license_filename |
|---|
| 163 | msg += ' at the root level. It is\n ' |
|---|
| 164 | msg += '<ga_license_file>' |
|---|
| 165 | raise Invalid, msg |
|---|
| 166 | |
|---|
| 167 | |
|---|
| 168 | # Start looking at document in earnest |
|---|
| 169 | root_node = doc.childNodes[0] |
|---|
| 170 | if root_node.nodeName != 'ga_license_file': |
|---|
| 171 | msg = 'License file %s must have two elements' %license_filename |
|---|
| 172 | msg += ' at the root level. They are\n ' |
|---|
| 173 | msg += '<?xml version="1.0" encoding="iso-8859-1"?>\n' |
|---|
| 174 | msg += '<ga_license_file>\n' |
|---|
| 175 | msg += 'The second element was found to be %s' %root_node.nodeName |
|---|
| 176 | raise WrongTags, msg |
|---|
| 177 | |
|---|
| 178 | |
|---|
| 179 | # Validate elements: metadata, datafile, datafile, ... |
|---|
| 180 | elements = get_elements(root_node.childNodes) |
|---|
| 181 | if elements[0].nodeName != 'metadata': |
|---|
| 182 | msg = 'The first element under %s must be "metadata"'\ |
|---|
| 183 | %root_node.nodeName |
|---|
| 184 | msg += 'The element found was %s' %elements[0].nodeName |
|---|
| 185 | raise WrongTags, msg |
|---|
| 186 | |
|---|
| 187 | for node in elements[1:]: |
|---|
| 188 | if node.nodeName != 'datafile': |
|---|
| 189 | msg = 'All elements, except the first, under %s must '\ |
|---|
| 190 | %root_node.nodeName |
|---|
| 191 | msg += 'be "datafile"' |
|---|
| 192 | msg += 'The element found was %s' %node.nodeName |
|---|
| 193 | raise WrongTags, msg |
|---|
| 194 | |
|---|
| 195 | if verbose: print |
|---|
| 196 | # Extract information for source section |
|---|
| 197 | for node in get_elements(elements[0].childNodes): |
|---|
| 198 | if node.nodeName == 'author': |
|---|
| 199 | # Do something |
|---|
| 200 | if verbose: print 'Author: ', get_text(node.childNodes) |
|---|
| 201 | |
|---|
| 202 | if node.nodeName == 'svn_keywords': |
|---|
| 203 | # Do nothing |
|---|
| 204 | pass |
|---|
| 205 | |
|---|
| 206 | # Extract information for datafile sections |
|---|
| 207 | for datanode in elements[1:]: |
|---|
| 208 | if verbose: print |
|---|
| 209 | |
|---|
| 210 | for node in get_elements(datanode.childNodes): |
|---|
| 211 | #print 'Node', node.nodeName, node.childNodes |
|---|
| 212 | #continue |
|---|
| 213 | |
|---|
| 214 | if node.nodeName == 'filename': |
|---|
| 215 | # FIXME Check correctness |
|---|
| 216 | filename = join(dirpath, get_text(node.childNodes)) |
|---|
| 217 | if verbose: print 'Filename: "%s"' %filename |
|---|
| 218 | try: |
|---|
| 219 | fid = open(filename, 'r') |
|---|
| 220 | except: |
|---|
| 221 | msg = 'Specified filename %s could not be opened'\ |
|---|
| 222 | %filename |
|---|
| 223 | raise FilenameMismatch, msg |
|---|
| 224 | |
|---|
| 225 | if node.nodeName == 'checksum': |
|---|
| 226 | # FIXME (Ole): This relies on crc being preceded by filename |
|---|
| 227 | reported_crc = get_text(node.childNodes) |
|---|
| 228 | if verbose: print 'Checksum: "%s"' %reported_crc |
|---|
| 229 | |
|---|
| 230 | file_crc = str(compute_checksum(filename)) |
|---|
| 231 | |
|---|
| 232 | if reported_crc != file_crc: |
|---|
| 233 | msg = 'Bad checksum (CRC).\n' |
|---|
| 234 | msg += ' The CRC reported in license file "%s" is "%s"\n'\ |
|---|
| 235 | %(license_filename, reported_crc) |
|---|
| 236 | msg += ' The CRC computed from file "%s" is "%s"'\ |
|---|
| 237 | %(filename, file_crc) |
|---|
| 238 | raise CRCMismatch, msg |
|---|
| 239 | |
|---|
| 240 | |
|---|
| 241 | if node.nodeName == 'accountable': |
|---|
| 242 | accountable = get_text(node.childNodes) |
|---|
| 243 | if verbose: print 'Accountable: "%s"' %accountable |
|---|
| 244 | if accountable == "": |
|---|
| 245 | msg = 'No accountable person specified' |
|---|
| 246 | raise Exception, msg |
|---|
| 247 | |
|---|
| 248 | if node.nodeName == 'source': |
|---|
| 249 | source = get_text(node.childNodes) |
|---|
| 250 | if verbose: print 'Source: "%s"' %source |
|---|
| 251 | if source == "": |
|---|
| 252 | msg = 'No source specified' |
|---|
| 253 | raise Exception, msg |
|---|
| 254 | |
|---|
| 255 | if node.nodeName == 'IP_owner': |
|---|
| 256 | ip_owner = get_text(node.childNodes) |
|---|
| 257 | if verbose: print 'IP owner: "%s"' %ip_owner |
|---|
| 258 | if ip_owner == "": |
|---|
| 259 | msg = 'No IP owner specified' |
|---|
| 260 | raise Exception, msg |
|---|
| 261 | |
|---|
| 262 | |
|---|
| 263 | if node.nodeName == 'IP_info': |
|---|
| 264 | if verbose: print 'IP info: "%s"' %get_text(node.childNodes) |
|---|
| 265 | |
|---|
| 266 | |
|---|
| 267 | if node.nodeName == 'publishable': |
|---|
| 268 | |
|---|
| 269 | if verbose: print 'Publishable: %s' %fid.name |
|---|
| 270 | value = get_text(node.childNodes) |
|---|
| 271 | if value.upper() != 'YES': |
|---|
| 272 | msg = 'Data file %s is not flagged as publishable'\ |
|---|
| 273 | %fid.name |
|---|
| 274 | raise NotPublishable, msg |
|---|
| 275 | |
|---|
| 276 | |
|---|
| 277 | |
|---|
| 278 | # If we get this far, the license file is OK |
|---|
| 279 | return True |
|---|
