Changeset 6906 for misc/tools/write_large_files/rwil_big_file.py

Timestamp:

Apr 26, 2009, 5:35:16 PM (16 years ago)

Author:

rwilson

Message:

Patong beach validation changes plus back-merge changes.

File:

• misc/tools/write_large_files/rwil_big_file.py (modified) (12 diffs)

misc/tools/write_large_files/rwil_big_file.py

@@ -8 +8 @@
 # that is checked when the file is read.
 #
-# So, element E of slice S ov variable V has the float value:
-#     V*1000 + S . E*1.0e-1
+# So, for variable V (numbering from 0), slice S (starting from 0), element
+# E will range from 0 to 10**6-1, and the element value will be:
+#     S*100 + V + E*1.0E-6
 # which will look like:
-#     VVVSSSS.0EEEEEE
+#     SSSSVV.EEEEEE
+# This implies that the maximum number of variables written is 100 (0 to 99).
 ################################################################################
 
@@ -28 +30 @@
 
 # array slice definitions
-XDIM = 1024*1024
+XDIM = 1000*1000
 SLICE_SIZE = XDIM*8     # 8 bytes / float
 MBYTE = 1024*1024
@@ -47 +49 @@
 
 # mask for variable names
-VarnameMask = 'var_%04d'
+VarnameMask = 'var_%02d'
 
 
@@ -84 +86 @@
 
             # create expected slice array
-            slice_array = slice_array_mask*slice_number + var_num*1000
-            for x in xrange(XDIM):
-                slice_array[x] += x*1.0e-7
+            slice_array = slice_array_mask*slice_number*100 + var_num
+            slice_array += ElementIndexArray
 
             if Verbose:
@@ -96 +97 @@
 
             var_array = fid.variables[varname][slice_number,:]
-
-            if var_array != slice_array:
+            if not num.allclose(var_array, slice_array):
+##            if num.any(var_array != slice_array):
                 print 'Read variable %s, slice %d: got unexpected value' % \
                       (varname, slice_number)
@@ -160 +161 @@
 
             # create unique slice array
-            slice_array = slice_array_mask*slice_number + var_num*1000
-            for x in xrange(XDIM):
-                slice_array[x] += x*1.0e-7
+            slice_array = slice_array_mask*slice_number*100 + var_num
+            slice_array += ElementIndexArray
 
             fid.variables[varname][slice_number,:] = slice_array
@@ -168 +168 @@
             if CloseAfterSlice:
                 fid.close()
-                fid = nc.NetCDFFile(filename, 'al')
-#                fid = nc.NetCDFFile(filename, 'a')
+#                fid = nc.NetCDFFile(filename, 'al')
+                fid = nc.NetCDFFile(filename, 'a')
 
             file_bytes_written += SLICE_SIZE
@@ -204 +204 @@
     print "  and <numvars> is the number of variables of the above size"
     print "                    to write.  If not supplied, 1 is assumed."
+    print "                    There can be at most 100 variables."
     print "  and <opts>    is zero or more of:"
     print "                    -c s    close & open the output file after"
@@ -217 +218 @@
 def main(argv=None):
     global TimeFileWrite, TimeFileRead, CloseAfterSlice, Verbose
+    global ElementIndexArray
 
     if argv is None:
@@ -259 +261 @@
 
     var_size = args[0][:-1]
-    modifier =args[0][-1]
+    modifier = args[0][-1]
 
     if modifier in '0123456789':
@@ -280 +282 @@
             usage()
 
+    if num_vars > 100:
+        usage()
+
+    # initialize the global element index array which contains
+    # 0.EEEEEE in each element.
+    ElementIndexArray = num.ndarray((XDIM,), 'd')
+    for x in xrange(XDIM):
+        ElementIndexArray[x] = x*1.0e-6
+    
     # write the required file
     filename = write_file(var_size, num_vars)
@@ -288 +299 @@
         return 10
 
-    print 'Read/write of NectCDF file was correct'
+#    print 'Read/write of NectCDF file was correct'
     return 0