Changeset 4576

Timestamp:

Jul 2, 2007, 5:25:38 PM (17 years ago)

Author:

ole

Message:

Improved verbose reporting when large data files are
processed block wise. This works best for pts files,
but would be good to do for txt and cvs as well.

Location:

anuga_core/source/anuga

Files:

• fit_interpolate/fit.py (modified) (1 diff)
• geospatial_data/geospatial_data.py (modified) (7 diffs)

anuga_core/source/anuga/fit_interpolate/fit.py

@@ -325 +325 @@
             assert point_origin is None, msg
             filename = point_coordinates_or_filename
-            
-            for i, geo_block in enumerate(Geospatial_data(filename,
-                                              max_read_lines=max_read_lines,
-                                              load_file_now=False,
-                                              verbose=verbose)):
+
+            G_data = Geospatial_data(filename,
+                                     max_read_lines=max_read_lines,
+                                     load_file_now=False,
+                                     verbose=verbose)
+            for i, geo_block in enumerate(G_data):
                 if verbose is True and 0 == i%200: # round every 5 minutes
                     # But this is dependant on the # of Triangles, so it
                     #isn't every 5 minutes.
-                    print 'Block %i' %i
+                        
+                    print 'Processing Block %d' %i
+                    # FIXME (Ole): It would be good to say how many blocks
+                    # there are here. But this is no longer necessary
+                    # for pts files as they are reported in geospatial_data
+                    # I suggest deleting this verbose output and make
+                    # Geospatial_data more informative for txt files.
+            
+
+                    
                 # build the array
                 points = geo_block.get_data_points(absolute=True)

anuga_core/source/anuga/geospatial_data/geospatial_data.py

@@ -165 +165 @@
             # attributes etc are provided!!
             # if file name then all provided info will be removed!
+
+            #if verbose is True:
+            #    if file_name is not None:
+            #        print 'Loading Geospatial data from file: %s' %file_name
+            
             self.import_points_file(file_name, delimiter, verbose)
                 
@@ -172 +177 @@
             self.set_default_attribute_name(default_attribute_name)
 
+
         #Why?    
         #assert self.attributes is None or isinstance(self.attributes, DictType)
         #This is a hassle when blocking, so I've removed it.
-        
+
+
+        if verbose is True:
+            if file_name is not None:
+                print 'Geospatial data created from file: %s' %file_name
+                if load_file_now is False:
+                    print 'Data will be loaded blockwise on demand'
+
+                    if file_name.endswith('csv') or file_name.endswith('txt'):
+                        print 'ASCII formats are not that great for '
+                        print 'blockwise reading. Consider storing this'
+                        print 'data as a pts NetCDF format'
 
     def __len__(self):
@@ -255 +272 @@
         if verbose in [False, True]:
             self.verbose = verbose
-            
-            if verbose is True:
-                print 'Geospatial_data.verbose = True'
         else:
             msg = 'Illegal value: %s' %str(verbose)
@@ -759 +773 @@
         #FIXME - what to do if the file isn't there
 
+        # FIXME (Ole): Shouldn't this go into the constructor?
+        # ... and shouldn't it be called block_size?
+        if self.max_read_lines is None:
+            self.max_read_lines = MAX_READ_LINES
+        
+
         if self.file_name[-4:] == ".xya":
             # FIXME (Ole): shouldn't the xya format be replaced by txt/csv?
@@ -774 +794 @@
             self.fid = NetCDFFile(self.file_name, 'r')
             
-            self.blocking_georef, self.blocking_keys, self.last_row = \
-                     _read_pts_file_header(self.fid, self.verbose)
+            self.blocking_georef, self.blocking_keys, self.number_of_points =\
+                                  _read_pts_file_header(self.fid,
+                                                        self.verbose)
             self.start_row = 0
+            self.last_row = self.number_of_points            
             self.show_verbose = 0
             self.verbose_block_size = (self.last_row + 10)/10
+            self.block_number = 0
+            self.number_of_blocks = self.number_of_points/self.max_read_lines
+            # This computes the number of full blocks. The last block may be
+            # smaller and won't be ircluded in this estimate.
             
             if self.verbose is True:
-                print 'Reading %d points (blocking) from file %s'\
-                      %(self.last_row,
-                        self.file_name)
+                print 'Reading %d points (in ~%d blocks) from file %s. '\
+                      %(self.number_of_points,
+                        self.number_of_blocks,
+                        self.file_name),
+                print 'Each block consists of %d data points'\
+                      %self.max_read_lines
             
         else:
@@ -791 +820 @@
                          _read_csv_file_header(file_pointer)
             self.blocking_georef = None # Used for reconciling zones
-            
-        if self.max_read_lines is None:
-            self.max_read_lines = MAX_READ_LINES
+
         return self
+    
     
     def next(self):
@@ -828 +856 @@
                 fin_row = self.last_row
 
-                
+               
             
             if self.verbose is True:
                 if self.show_verbose >= self.start_row and \
                        self.show_verbose < fin_row:
-                 print 'Doing %d of %d' %(self.start_row, self.last_row+10)
-                 self.show_verbose += max(self.max_read_lines,
-                                          self.verbose_block_size)
-            #call stuff
+
+                    
+                    #print 'Doing %d of %d' %(self.start_row, self.last_row+10)
+
+                    print 'Reading block %d (points %d to %d) out of %d'\
+                          %(self.block_number,
+                            self.start_row,
+                            fin_row,
+                            self.number_of_blocks)
+
+                    
+                    self.show_verbose += max(self.max_read_lines,
+                                             self.verbose_block_size)
+
+                 
+            # Read next block
             pointlist, att_dict, = \
-                   _read_pts_file_blocking(self.fid,
-                                           self.start_row,
-                                           fin_row,
-                                           self.blocking_keys)
+                       _read_pts_file_blocking(self.fid,
+                                               self.start_row,
+                                               fin_row,
+                                               self.blocking_keys)
             
             geo = Geospatial_data(pointlist, att_dict, self.blocking_georef)
             self.start_row = fin_row
+            
+            self.block_number += 1            
             
         else: