Changeset 4963

Timestamp:

Jan 21, 2008, 6:58:15 PM (17 years ago)

Author:

ole

Message:

Work on IP tracking:
Checksums, removed DOCTYPE for the time being, formatted, and more checks

Location:

anuga_core/source/anuga/utilities

Files:

• data_audit.py (modified) (10 diffs)
• mainland_only.lic (modified) (2 diffs)
• parse.py (modified) (1 diff)
• system_tools.py (modified) (1 diff)
• xml_tools.py (modified) (2 diffs)

anuga_core/source/anuga/utilities/data_audit.py

@@ -5 +5 @@
 from os.path import join, splitext
 
-from anuga.utilities.xml_tools import parse, print_tree, get_elements, get_text
+from anuga.utilities.xml_tools import parse, pretty_print_tree, get_elements, get_text
+from anuga.utilities.system_tools import compute_checksum
 
 # Audit exceptions
 class NotPublishable(Exception): pass
+class FilenameMismatch(Exception): pass
+class CRCMismatch(Exception): pass
 class Invalid(Exception): pass
 class WrongTags(Exception): pass
 
+audit_exceptions = (NotPublishable, FilenameMismatch, CRCMismatch, Invalid, WrongTags)
 
 def IP_verified(directory):
@@ -42 +46 @@
     all_files_accounted_for = True
     for dirpath, datafile in identify_datafiles(directory):
-        print join(dirpath, datafile) + ': ',
+        filename = join(dirpath, datafile)
+        
+        print filename + ' (Checksum=%s): ' %str(compute_checksum(filename)),
 
         basename, ext = splitext(datafile)
@@ -53 +59 @@
             all_files_accounted_for = False
         else:
-            if license_file_is_valid(fid):
+            try:
+                license_file_is_valid(fid, dirpath, verbose=False)
+            except audit_exceptions, e:
+                all_files_accounted_for = False                                
+                print 'LICENSE FILE NOT VALID'
+                print 'REASON:', e
+
+                #doc = parse(fid)
+                #pretty_print_tree(doc)
+                fid.seek(0)
+                print fid.read()
+
+            else:        
                 print 'OK'
-            else:
-                print 'LICENSE FILE NOT VALID'
-                all_files_accounted_for = False
+
             fid.close()
 
@@ -115 +131 @@
 
 
-def license_file_is_valid(fid):
+def license_file_is_valid(fid, dirpath, verbose=False):
     """Check that XML license file is valid
     """
 
+    license_filename = fid.name
     doc = parse(fid)
     #print_tree(doc)
@@ -127 +144 @@
 
     if doc.nodeName != '#document':
-        msg = 'License file %s does not appear' %fid.name
+        msg = 'License file %s does not appear' %license_filename
         msg += 'to be a valid XML document'
         msg += 'The root node has name %s' %doc.nodeName
@@ -133 +150 @@
         raise Invalid, msg        
 
-    if len(doc.childNodes) != 2:
-        msg = 'License file %s must have two elements' %fid.name
-        msg += ' at the root level. They are\n '
-        msg += '<?xml version="1.0" encoding="iso-8859-1"?>\n'
+    if len(doc.childNodes) != 1:
+        msg = 'License file %s must have only one element' %license_filename
+        msg += ' at the root level. It is\n '
         msg += '<ga_license_file>'
         raise Invalid, msg
@@ -142 +158 @@
 
     # Start looking at document in earnest
-    root_node = doc.childNodes[1]
+    root_node = doc.childNodes[0]
     if root_node.nodeName != 'ga_license_file':
-        msg = 'License file %s must have two elements' %fid.name
+        msg = 'License file %s must have two elements' %license_filename
         msg += ' at the root level. They are\n '
         msg += '<?xml version="1.0" encoding="iso-8859-1"?>\n'
@@ -168 +184 @@
             raise WrongTags, msg        
 
-    print 
+    if verbose: print    
     # Extract information for source section
     for node in get_elements(elements[0].childNodes):
         if node.nodeName == 'author':
             # Do something
-            print 'Author is', get_text(node.childNodes)
+            if verbose: print 'Author:   ', get_text(node.childNodes)
 
         if node.nodeName == 'svn_keywords':
@@ -181 +197 @@
     # Extract information for datafile sections
     for datanode in elements[1:]:
-        print    
+        if verbose: print
     
         for node in get_elements(datanode.childNodes):
@@ -189 +205 @@
             if node.nodeName == 'filename':
                 # FIXME Check correctness
-                print 'Filename is "%s"' %get_text(node.childNodes)
+                filename = join(dirpath, get_text(node.childNodes))
+                if verbose: print 'Filename: "%s"' %filename
+                try:
+                    fid = open(filename, 'r')
+                except:
+                    msg = 'Specified filename %s could not be opened'\
+                          %filename
+                    raise FilenameMismatch, msg
+
+            if node.nodeName == 'checksum':
+                # FIXME (Ole): This relies on crc being preceded by filename
+                reported_crc = get_text(node.childNodes)
+                if verbose: print 'Checksum: "%s"' %reported_crc
+
+                file_crc = str(compute_checksum(filename))
+
+                if reported_crc != file_crc:
+                    msg = 'Bad checksum (CRC).\n'
+                    msg += '  The CRC reported in license file "%s" is "%s"\n'\
+                          %(license_filename, reported_crc)
+                    msg += '  The CRC computed from file "%s" is "%s"'\
+                           %(filename, file_crc)
+                    raise CRCMismatch, msg
+                
 
             if node.nodeName == 'accountable':
-                print 'Accountable is "%s"' %get_text(node.childNodes)
+                accountable = get_text(node.childNodes)
+                if verbose: print 'Accountable: "%s"' %accountable
+                if accountable == "":
+                    msg = 'No accountable person specified'
+                    raise Exception, msg
 
             if node.nodeName == 'source':
-                print 'Source is "%s"' %get_text(node.childNodes)
+                source = get_text(node.childNodes)
+                if verbose: print 'Source: "%s"' %source
+                if source == "":
+                    msg = 'No source specified'
+                    raise Exception, msg                
 
             if node.nodeName == 'IP_owner':
-                print 'IP owner is "%s"' %get_text(node.childNodes)
+                ip_owner = get_text(node.childNodes)
+                if verbose: print 'IP owner: "%s"' %ip_owner
+                if ip_owner == "":
+                    msg = 'No IP owner specified'
+                    raise Exception, msg                                
+                
 
             if node.nodeName == 'IP_info':
-                print 'IP info is "%s"' %get_text(node.childNodes)                                
+                if verbose: print 'IP info: "%s"' %get_text(node.childNodes)  
                 
 
             if node.nodeName == 'publishable':
+                
+                if verbose: print 'Publishable: %s' %fid.name                
                 value = get_text(node.childNodes)
                 if value.upper() != 'YES':
                     msg = 'Data file %s is not flagged as publishable'\
                           %fid.name
-                    print msg
-                    #raise NotPublishable, msg
-                else:
-                    print 'Data file %s is flagged publishable' %fid.name                
-
-    #FIXME (Ole): Use hash code for original datafile as an XML element
-    # USE CRC32 in zlib or hash
-    
-    #for node in elements:
-    #    print node
-    #print 
-
-
-
-    # Check that file is deemed publishable
-    items = doc.getElementsByTagName('publishable')
-    for i in items:
-        print i
-        #i.getAttribute()
+                    raise NotPublishable, msg
+
+
+
+    # If we get this far, the license file is OK
+    return True

anuga_core/source/anuga/utilities/mainland_only.lic

@@ -1 +1 @@
 <?xml version="1.0" encoding="iso-8859-1"?>
-
-  <!DOCTYPE ga_license_file [
-      <!ELEMENT ga_license_file (source, datafile+)>
-      <!ELEMENT metadata (author, svn_keywords)>
-      <!ELEMENT svn_keywords (author, date, revision, url, id)>    
-      <!ELEMENT datafile (filename, publishable, accountable, 
-                        owner, location, IP_info)>    
-      <!ELEMENT filename (#PCDATA)>
-      <!ELEMENT publishable (#PCDATA)>    
-      <!ELEMENT accountable (#PCDATA)>        
-      <!ELEMENT source (#PCDATA)>                               
-      <!ELEMENT IP_owner (#PCDATA)>            
-      <!ELEMENT IP_info (#PCDATA)>                
-  ]>
 
   <ga_license_file>
@@ -28 +14 @@
     <datafile>
       <filename>mainland_only.csv</filename>
+      <checksum>-1661725548</checksum>
       <publishable>No</publishable>
       <accountable>Jane Sexton</accountable>
       <source>Unknown</source>
       <IP_owner>Geoscience Australia</IP_owner>
-      <IP_info>This is a polygon tracing the coastline at Dampier WA. The origin and license issues are undecided</IP_info>
+      <IP_info>This is a polygon comprising easting and northing locations 
+      tracing parts of the coastline at Dampier WA as well as a rectangular area inland.
+      This is used to specifically set the onshore initial condition in a tsunami scenario
+      and here, it is used with a unit test in test_polygon.py.
+      
+      The coastline was derived from Maritime Boundaries which is a public dataset. However,
+      rumour has it that some of it was digitised from a Landgate supplied image.
+      
+      The origin and license issues are still undecided</IP_info>
     </datafile>
 
   </ga_license_file>
-  

anuga_core/source/anuga/utilities/parse.py

@@ -22 +22 @@
 
 from data_audit import license_file_is_valid
-license_file_is_valid(fid)
+license_file_is_valid(fid, '.')
 
 

anuga_core/source/anuga/utilities/system_tools.py

@@ -172 +172 @@
             
 
-def compute_checksum(filename):
+def compute_checksum(filename, max_length=2**20):
     """Compute the CRC32 checksum for specified file
+
+    Optional parameter max_length sets the maximum number
+    of bytes used to limit time used with large files.
+    Default = 2**20 (1MB)
     """
 
-    import zlib
-    #FIXME(Ole): Do we need to limit the size?
-
+    #from zlib import crc32
+    from binascii import crc32 #(works as well)
+    
     fid = open(filename)
-
-    crcval = zlib.crc32(fid.read())
-    
+    crcval = crc32(fid.read(max_length))
     return crcval

anuga_core/source/anuga/utilities/xml_tools.py

@@ -22 +22 @@
 
 
+def pretty_print_tree(n, indent=0):
+    print n
+
 def parse(fid):
     """Parse XML file descriptor and return DOM object.
@@ -30 +33 @@
     #doc = minidom.parse(fid, make_parser())
 
+    fid.seek(0)
     doc = minidom.parse(fid)    
     return doc