Changeset 6906 for misc/tools/write_large_files/rwi_big_file.py

Timestamp:

Apr 26, 2009, 5:35:16 PM (16 years ago)

Author:

rwilson

Message:

Patong beach validation changes plus back-merge changes.

File:

• misc/tools/write_large_files/rwi_big_file.py (modified) (12 diffs)

misc/tools/write_large_files/rwi_big_file.py

@@ -8 +8 @@
 # that is checked when the file is read.
 #
-# So, element E of slice S ov variable V has the float value:
-#     V*1000 + S . E*1.0e-1
+# So, for variable V (numbering from 0), slice S (starting from 0), element
+# E will range from 0 to 10**6-1, and the element value will be:
+#     S*100 + V + E*1.0E-6
 # which will look like:
-#     VVVSSSS.0EEEEEE
+#     SSSSVV.EEEEEE
+# This implies that the maximum number of variables written is 100 (0 to 99).
 ################################################################################
 
@@ -28 +30 @@
 
 # array slice definitions
-XDIM = 1024*1024
+XDIM = 1000*1000
 SLICE_SIZE = XDIM*8     # 8 bytes / float
 MBYTE = 1024*1024
@@ -47 +49 @@
 
 # mask for variable names
-VarnameMask = 'var_%04d'
+VarnameMask = 'var_%02d'
 
 
@@ -87 +89 @@
 
             # create expected slice array
-            slice_array = slice_array_mask*slice_number + var_num*1000
-            for x in xrange(XDIM):
-                slice_array[x] += x*1.0e-7
+            slice_array = slice_array_mask*slice_number*100 + var_num
+            slice_array += ElementIndexArray
 
             if Verbose:
                 print ('File %s, variable %s, reading slice %d: '
                        'var=%.1fMiB, file=%.1fMiB' %
-                       (filename, varname, slice_number, float(var_bytes_read)/MBYTE,
+                       (filename, varname, slice_number,
+                        float(var_bytes_read)/MBYTE,
                         float(file_bytes_read)/MBYTE))
 
             var_array = fid.variables[varname][slice_number,:]
-            if var_array != slice_array:
-                result = False
-                if Verbose:
-                    print 'Read variable %s, slice %d: got unexpected value' % \
-                          (varname, slice_number)
-                    for x in xrange(XDIM):
-                        if var_array[x] != slice_array[x]:
-                            print 'Got %f, expected %f' % (var_array[x], slice_array[x])
-                    sys.exit(0)
+            if not num.allclose(var_array, slice_array):
+##            if num.any(var_array != slice_array):
                 print 'Read variable %s, slice %d: got unexpected value' % \
                       (varname, slice_number)
                 for x in xrange(XDIM):
                     if var_array[x] != slice_array[x]:
-                        print 'Got %f, expected %f' % (var_array[x], slice_array[x])
-                        sys.exit(0)
+                        print 'Index %d, got %f, expected %f' % \
+                              (x, var_array[x], slice_array[x])
+                        return False
 
             slice_number += 1
@@ -127 +123 @@
                (stop_file_read - start_file_read))
 
-    return result
+    return True
 
 
@@ -168 +164 @@
 
             # create unique slice array
-            slice_array = slice_array_mask*slice_number + var_num*1000
-            for x in xrange(XDIM):
-                slice_array[x] += x*1.0e-7
+            slice_array = slice_array_mask*slice_number*100 + var_num
+            slice_array += ElementIndexArray
 
             fid.variables[varname][slice_number,:] = slice_array
@@ -188 +183 @@
 
         slice_number += 1
+
+    fid.close()
 
     if TimeFileWrite:
@@ -209 +206 @@
     print "  and <numvars> is the number of variables of the above size"
     print "                    to write.  If not supplied, 1 is assumed."
+    print "                    There can be at most 100 variables."
     print "  and <opts>    is zero or more of:"
     print "                    -c s    close & open the output file after"
@@ -222 +220 @@
 def main(argv=None):
     global TimeFileWrite, TimeFileRead, CloseAfterSlice, Verbose
+    global ElementIndexArray
 
     if argv is None:
@@ -264 +263 @@
 
     var_size = args[0][:-1]
-    modifier =args[0][-1]
+    modifier = args[0][-1]
 
     if modifier in '0123456789':
@@ -285 +284 @@
             usage()
 
+    if num_vars > 100:
+        usage()
+
+    # initialize the global element index array which contains
+    # 0.EEEEEE in each element.
+    ElementIndexArray = num.ndarray((XDIM,), 'd')
+    for x in xrange(XDIM):
+        ElementIndexArray[x] = x*1.0e-6
+    
     # write the required file
     filename = write_file(var_size, num_vars)
@@ -293 +301 @@
         return 10
 
-    print 'Read/write of NectCDF file was correct'
+#    print 'Read/write of NectCDF file was correct'
     return 0