Index: /misc/tools/write_large_files/README
===================================================================
--- /misc/tools/write_large_files/README	(revision 6739)
+++ /misc/tools/write_large_files/README	(revision 6739)
@@ -0,0 +1,3 @@
+This directory contains exploratory programs to exercise the writing/reading
+of 'large' and '64bit' NetCDF files.  Some of this code will eventually
+become a test case.
Index: /misc/tools/write_large_files/rw_big_file.py
===================================================================
--- /misc/tools/write_large_files/rw_big_file.py	(revision 6739)
+++ /misc/tools/write_large_files/rw_big_file.py	(revision 6739)
@@ -0,0 +1,343 @@
+#!/usr/bin/env python
+################################################################################
+# A test program to write a large NetCDF file and see if we can read it.
+################################################################################
+
+import sys
+import getopt
+import time
+import Numeric
+import Scientific.IO.NetCDF as nc
+
+######
+# Various constants
+######
+
+# default modifier and number of files
+DEF_MODIFIER = 'M'
+
+# array slice definitions
+XDIM = 1024*1024
+SLICE_SIZE = XDIM*8	# 8 bytes / float
+MBYTE = 1024*1024
+
+# Dictionary for suffix modifiers
+Suffixes = {'M': 1024*1024,
+            'G': 1024*1024*1024
+           }
+
+######
+# Globals
+######
+
+TimeFirstWrite = False
+TimeVarWrite = False
+TimeFileWrite = False
+
+TimeFirstRead = False
+TimeVarRead = False
+TimeFileRead = False
+
+CloseAfterSlice = False
+CloseAfterVar = False
+
+Verbose = False
+
+# mask for variable names
+VarnameMask = 'var_%04d'
+
+
+######
+# mainline code
+######
+
+##
+# @brief Read a NetCDF file - see if it as we wrote it.
+# @param filename The name of the file to read.
+# @param variable_size Size of variable, in bytes.
+# @param num_variables Number of variables.
+def read_file(filename, variable_size, num_variables):
+    # create a data array slice
+    slice_array_mask = Numeric.ones((XDIM,), 'd')
+
+    # if timing file read, get start time
+    if TimeFileRead:
+        start_file_read = time.time()
+
+    fid = nc.NetCDFFile(filename, 'r')
+
+    # num file bytes read counter
+    file_bytes_read = 0
+
+    result = True
+
+    for i in xrange(num_variables):
+        varname = VarnameMask % i
+        var = fid.variables[varname]
+
+        # num variable bytes written counter
+        var_bytes_read = 0
+
+        # create expected slice array
+        slice_array = slice_array_mask * (i + 1)
+        for x in xrange(XDIM):
+            slice_array[x] += x*1.0e-7
+
+        if TimeVarRead:
+            start_var_read = time.time()
+
+        index = 0
+        while var_bytes_read < variable_size:
+            var_bytes_read += SLICE_SIZE
+            file_bytes_read += SLICE_SIZE
+
+            if TimeFirstRead and index == 0:
+                start_first_var_read = time.time()
+
+            if Verbose:
+                print ('File %s, variable %s, reading slice %d: '
+                       'var=%.1fMiB, file=%.1fMiB' %
+                       (filename, varname, index, float(var_bytes_read)/MBYTE,
+                        float(file_bytes_read)/MBYTE))
+
+            var_array = fid.variables[varname][index,:]
+            if var_array != slice_array:
+                result = False
+                if Verbose:
+                    print 'Read variable %s, slice %d: got unexpected value' % \
+                          (varname, index)
+
+            if TimeFirstRead and index == 0:
+                stop_first_var_read = time.time()
+                print ("Time to do first read for variable %s: %.2f sec" %
+                       (varname, (stop_first_var_read - start_first_var_read)))
+
+            index += 1
+
+            if CloseAfterSlice:
+                fid.close()
+                fid = nc.NetCDFFile(filename, 'r')
+        if CloseAfterVar:
+            fid.close()
+            fid = nc.NetCDFFile(filename, 'r')
+
+        if TimeVarRead:
+            stop_var_read = time.time()
+            print ('Time to read variable %s: %.2f sec' %
+                   (varname, (stop_var_read - start_var_read)))
+
+    fid.close()
+
+    if TimeFileRead:
+        stop_file_read = time.time()
+        print ('Time to read file: %.2f sec' %
+               (stop_file_read - start_file_read))
+
+    return result
+
+
+##
+# @brief Write a NetCDF file with set number of variables of a defined size.
+# @param variable_size Size of variable, in bytes.
+# @param num_variables Number of required variables.
+# @return The filename of the created file.
+def write_file(variable_size, num_variables=1):
+    # set file and variable name masks
+    filename = 'test.nc'
+    VarnameMask = 'var_%04d'
+
+    # create a data array slice
+    slice_array_mask = Numeric.ones((XDIM,), 'd')
+
+    # if timing file write, remember start time
+    if TimeFileWrite:
+        start_file_write = time.time()
+
+    fid = nc.NetCDFFile(filename, 'w')
+    fid.createDimension('y', None)
+    fid.createDimension('x', XDIM)
+
+    # num file bytes written counter
+    file_bytes_written = 0
+
+    for i in xrange(num_variables):
+        varname = VarnameMask % i
+        fid.createVariable(varname, 'd', ('y', 'x'))
+
+        # num variable bytes written counter
+        var_bytes_written = 0
+
+        # create unique slice array
+        slice_array = slice_array_mask * (i + 1)
+        for x in xrange(XDIM):
+            slice_array[x] = slice_array[x] + x*1.0e-7
+
+        if TimeVarWrite:
+            start_var_write = time.time()
+
+        index = 0
+        while var_bytes_written < variable_size:
+            var_bytes_written += SLICE_SIZE
+            file_bytes_written += SLICE_SIZE
+
+            if TimeFirstWrite and index == 0:
+                start_first_var_write = time.time()
+
+            if Verbose:
+                print ('File %s, variable %s, writing slice %d: '
+                       'var=%.1fMiB, file=%.1fMiB' %
+                       (filename, varname, index, float(var_bytes_written)/MBYTE,
+                        float(file_bytes_written)/MBYTE))
+
+            fid.variables[varname][index,:] = slice_array
+
+            if TimeFirstWrite and index == 0:
+                stop_first_var_write = time.time()
+                print ("Time to do first write for variable %s: %.2f sec" %
+                       (varname, (stop_first_var_write-start_first_var_write)))
+
+            index += 1
+
+            if CloseAfterSlice:
+                fid.close()
+                fid = nc.NetCDFFile(filename, 'a')
+        if CloseAfterVar:
+            fid.close()
+            fid = nc.NetCDFFile(filename, 'a')
+
+        if TimeVarWrite:
+            stop_var_write = time.time()
+            print ('Time to write variable %s: %.2f sec' %
+                   (varname, (stop_var_write - start_var_write)))
+
+    fid.close()
+
+    if TimeFileWrite:
+        stop_file_write = time.time()
+        print ('Time to write file: %.2f sec' %
+               (stop_file_write - start_file_write))
+
+    return filename
+
+
+##
+# @brief Provide help for the befuddled user.
+# @return Doesn't, calls sys.exit().
+def usage(msg=None):
+    print "Usage: write_large_files <opts> <varsize> [<numvars>]"
+    print ""
+    print "where <varsize> is a number followed by an optional modifier:"
+    print "                    1024M or 4G"
+    print "                the assumed modifier if none is given is '%s'." \
+          % DEF_MODIFIER
+    print "  and <numvars> is the number of variables of the above size"
+    print "                    to write.  If not supplied, 1 is assumed."
+    print "  and <opts>    is zero or more of:"
+    print "                    -c s[lice]  close & open the output file after"
+    print "                                each variable slice is read/written,"
+    print "                    -c v[ar]    close & open the output file after"
+    print "                                each variable is read/written,"
+    print "                    -t rf[irst] time the first var slice read,"
+    print "                    -t rv[ar]   time the complete var read,"
+    print "                    -t rw[hole] time the complete file read,"
+    print "                    -t wf[irst] time the first var slice write,"
+    print "                    -t wv[ar]   time the complete var write,"
+    print "                    -t ww[hole] time the complete file write,"
+
+    if msg:
+        derr, "\n%s" % msg
+
+    sys.exit(10)
+
+def main(argv=None):
+    global TimeFirstWrite, TimeVarWrite, TimeFileWrite
+    global TimeFirstRead, TimeVarRead, TimeFileRead
+    global CloseAfterSlice, CloseAfterVar
+    global Verbose
+
+    if argv is None:
+        argv = sys.argv
+
+    # parse command line args
+    try:
+        opts, args = getopt.getopt(argv[1:], "c:t:v", ["help"])
+    except getopt.error, msg:
+        usage(msg)
+
+    for (opt, optarg) in opts:
+        if opt == '-c':
+            optargchar = optarg[0].lower()
+            if optargchar == 's':
+                CloseAfterSlice = True
+            elif optargchar == 'v':
+                CloseAfterVar = True
+            else:
+                usage("Unrecognized -c suboption: %s" % optarg)
+        elif opt == '-t':
+            optargchar = optarg[0].lower()
+            if optargchar == 'r':
+                optargchar = optarg[1].lower()
+                if optargchar == 'f':
+                    TimeFirstRead = True
+                elif optargchar == 'v':
+                    TimeVarRead = True
+                elif optargchar == 'w':
+                    TimeFileRead = True
+                else:
+                    usage("Unrecognized -t suboption: %s" % optarg)
+            elif optargchar == 'w':
+                optargchar = optarg[1].lower()
+                if optargchar == 'f':
+                    TimeFirstWrite = True
+                elif optargchar == 'v':
+                    TimeVarWrite = True
+                elif optargchar == 'w':
+                    TimeFileWrite = True
+                else:
+                    usage("Unrecognized -t suboption: %s" % optarg)
+            else:
+                usage("Unrecognized -t suboption: %s" % optarg)
+        elif opt == '-v':
+            Verbose = True
+        else:
+            usage("Unrecognized option: %s" % opt)
+
+    if len(args) != 1 and len(args) != 2:
+        usage()
+
+    var_size = args[0][:-1]
+    modifier =args[0][-1]
+
+    if modifier in '0123456789':
+        var_size = args[0]
+        modifier = DEF_MODIFIER
+    modifier = Suffixes.get(modifier, None)
+    if modifier is None:
+        usage()
+        
+    try:
+        var_size = int(var_size) * modifier
+    except:
+        usage()
+
+    num_vars = 1
+    if len(args) == 2:
+        try:
+            num_vars = int(args[1])
+        except:
+            usage()
+
+    # write the required file
+    filename = write_file(var_size, num_vars)
+        
+    # read the file to see if is as expected
+    if not read_file(filename, var_size, num_vars):
+        print "Didn't read data that we read!?"
+        return 10
+
+    print 'Read/write of NectCDF file was correct'
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
Index: /misc/tools/write_large_files/rwi4_big_file.py
===================================================================
--- /misc/tools/write_large_files/rwi4_big_file.py	(revision 6739)
+++ /misc/tools/write_large_files/rwi4_big_file.py	(revision 6739)
@@ -0,0 +1,295 @@
+#!/usr/bin/env python
+################################################################################
+# A test program to write a large NetCDF file and see if we can read it.
+#
+# The file is written with a defined number of variables, each of a required
+# size.  Each variable consists of one or more 'slices', each 1000*1000 cells.
+# Each element of each slice of each variable is written with a *unique* value
+# that is checked when the file is read.
+#
+# So, element E of slice S ov variable V has the float value:
+#     V*1000 + S . E*1.0e-1
+# which will look like:
+#     VVVSSSS.0EEEEEE
+################################################################################
+
+import sys
+import getopt
+import time
+import Numeric
+import Scientific.IO.NetCDF as nc
+
+######
+# Various constants
+######
+
+# default modifier and number of files
+DEF_MODIFIER = 'M'
+
+# array slice definitions
+XDIM = 1024*1024
+SLICE_SIZE = XDIM*8	# 8 bytes / float
+MBYTE = 1024*1024
+
+# Dictionary for suffix modifiers
+Suffixes = {'M': 1024*1024,
+            'G': 1024*1024*1024
+           }
+
+######
+# Globals
+######
+
+TimeFileWrite = False
+TimeFileRead = False
+CloseAfterSlice = False
+Verbose = False
+
+# mask for variable names
+VarnameMask = 'var_%04d'
+
+
+######
+# Mainline code
+######
+
+##
+# @brief Read a NetCDF file - see if it as we wrote it.
+# @param filename The name of the file to read.
+# @param variable_size Size of variable, in bytes.
+# @param num_variables Number of variables.
+def read_file(filename, variable_size, num_variables):
+    # create a data array slice
+    slice_array_mask = Numeric.ones((XDIM,), 'd')
+
+    # if timing file read, get start time
+    if TimeFileRead:
+        start_file_read = time.time()
+
+    fid = nc.NetCDFFile(filename, 'r')
+
+    # num file bytes read counter
+    file_bytes_read = 0
+
+    for var_num in xrange(num_variables):
+        varname = VarnameMask % var_num
+        var = fid.variables[varname]
+
+        # num variable bytes written counter
+        var_bytes_read = 0
+
+        slice_number = 0
+        while var_bytes_read < variable_size:
+            var_bytes_read += SLICE_SIZE
+            file_bytes_read += SLICE_SIZE
+
+            # create expected slice array
+            slice_array = slice_array_mask*slice_number + var_num*1000
+            for x in xrange(XDIM):
+                slice_array[x] += x*1.0e-7
+
+            if Verbose:
+                print ('File %s, variable %s, reading slice %d: '
+                       'var=%.1fMiB, file=%.1fMiB' %
+                       (filename, varname, slice_number,
+                        float(var_bytes_read)/MBYTE,
+                        float(file_bytes_read)/MBYTE))
+
+#            var_array = fid.variables[varname][slice_number,:]
+            var_array = var[slice_number,:]
+            if var_array != slice_array:
+                print 'Read variable %s, slice %d: got unexpected value' % \
+                      (varname, slice_number)
+                for x in xrange(XDIM):
+                    if var_array[x] != slice_array[x]:
+                        print 'Index %d, got %f, expected %f' % \
+                              (x, var_array[x], slice_array[x])
+                        return False
+
+            slice_number += 1
+
+            if CloseAfterSlice:
+                fid.close()
+                fid = nc.NetCDFFile(filename, 'r')
+
+    fid.close()
+
+    if TimeFileRead:
+        stop_file_read = time.time()
+        print ('Time to read file: %.2f sec' %
+               (stop_file_read - start_file_read))
+
+    return True
+
+
+##
+# @brief Write a NetCDF file with set number of variables of a defined size.
+# @param variable_size Size of variable, in bytes.
+# @param num_variables Number of required variables.
+# @return The filename of the created file.
+# @note Variable slice writes are interleaved.
+def write_file(variable_size, num_variables=1):
+    # set file and variable name masks
+    filename = 'test.nc'
+
+    # create a data array slice
+    slice_array_mask = Numeric.ones((XDIM,), 'd')
+
+    # if timing file write, remember start time
+    if TimeFileWrite:
+        start_file_write = time.time()
+
+    fid = nc.NetCDFFile(filename, 'w4')
+    fid.createDimension('y', None)
+    fid.createDimension('x', XDIM)
+
+    # create all required variables
+    for i in xrange(num_variables):
+        varname = VarnameMask % i
+        fid.createVariable(varname, 'd', ('y', 'x'))
+
+    # set counters to zero
+    var_bytes_written = 0
+    file_bytes_written = 0
+    slice_number = 0
+
+    while var_bytes_written < variable_size:
+        var_bytes_written += SLICE_SIZE
+
+        for var_num in xrange(num_variables):
+            varname = VarnameMask % var_num
+
+            # create unique slice array
+            slice_array = slice_array_mask*slice_number + var_num*1000
+            for x in xrange(XDIM):
+                slice_array[x] += x*1.0e-7
+
+            fid.variables[varname][slice_number,:] = slice_array
+
+            if CloseAfterSlice:
+                fid.close()
+                fid = nc.NetCDFFile(filename, 'a4')
+
+            file_bytes_written += SLICE_SIZE
+
+            if Verbose:
+                print ('File %s, variable %s, writing slice %d: '
+                       'var=%.1fMiB, file=%.1fMiB' %
+                       (filename, varname, slice_number,
+                        float(var_bytes_written)/MBYTE,
+                        float(file_bytes_written)/MBYTE))
+
+        slice_number += 1
+
+    fid.close()
+
+    if TimeFileWrite:
+        stop_file_write = time.time()
+        print ('Time to write file: %.2f sec' %
+               (stop_file_write - start_file_write))
+
+    return filename
+
+
+##
+# @brief Provide help for the befuddled user.
+# @return Doesn't, calls sys.exit().
+def usage(msg=None):
+    print "Usage: write_large_files <opts> <varsize> [<numvars>]"
+    print ""
+    print "where <varsize> is a number followed by an optional modifier:"
+    print "                    1024M or 4G"
+    print "                the assumed modifier if none is given is '%s'." \
+          % DEF_MODIFIER
+    print "  and <numvars> is the number of variables of the above size"
+    print "                    to write.  If not supplied, 1 is assumed."
+    print "  and <opts>    is zero or more of:"
+    print "                    -c s    close & open the output file after"
+    print "                            each variable slice is read/written,"
+    print "                    -t rf   time the complete file read,"
+    print "                    -t wf   time the complete file write,"
+
+    if msg:
+        derr, "\n%s" % msg
+
+    sys.exit(10)
+
+def main(argv=None):
+    global TimeFileWrite, TimeFileRead, CloseAfterSlice, Verbose
+
+    if argv is None:
+        argv = sys.argv
+
+    # parse command line args
+    try:
+        opts, args = getopt.getopt(argv[1:], "c:t:v", ["help"])
+    except getopt.error, msg:
+        usage(msg)
+
+    for (opt, optarg) in opts:
+        if opt == '-c':
+            optargchar = optarg[0].lower()
+            if optargchar == 's':
+                CloseAfterSlice = True
+            else:
+                usage("Unrecognized -c suboption: %s" % optarg)
+        elif opt == '-t':
+            optargchar = optarg[0].lower()
+            if optargchar == 'r':
+                optargchar = optarg[1].lower()
+                if optargchar == 'f':
+                    TimeFileRead = True
+                else:
+                    usage("Unrecognized -t suboption: %s" % optarg)
+            elif optargchar == 'w':
+                optargchar = optarg[1].lower()
+                if optargchar == 'f':
+                    TimeFileWrite = True
+                else:
+                    usage("Unrecognized -t suboption: %s" % optarg)
+            else:
+                usage("Unrecognized -t suboption: %s" % optarg)
+        elif opt == '-v':
+            Verbose = True
+        else:
+            usage("Unrecognized option: %s" % opt)
+
+    if len(args) != 1 and len(args) != 2:
+        usage()
+
+    var_size = args[0][:-1]
+    modifier =args[0][-1]
+
+    if modifier in '0123456789':
+        var_size = args[0]
+        modifier = DEF_MODIFIER
+    modifier = Suffixes.get(modifier, None)
+    if modifier is None:
+        usage()
+        
+    try:
+        var_size = int(var_size) * modifier
+    except:
+        usage()
+
+    num_vars = 1
+    if len(args) == 2:
+        try:
+            num_vars = int(args[1])
+        except:
+            usage()
+
+    # write the required file
+    filename = write_file(var_size, num_vars)
+        
+    # read the file to see if is as expected
+    if not read_file(filename, var_size, num_vars):
+        print "Didn't read data that we read!?"
+        return 10
+
+    print 'Read/write of NectCDF file was correct'
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
Index: /misc/tools/write_large_files/rwi_big_file.py
===================================================================
--- /misc/tools/write_large_files/rwi_big_file.py	(revision 6739)
+++ /misc/tools/write_large_files/rwi_big_file.py	(revision 6739)
@@ -0,0 +1,300 @@
+#!/usr/bin/env python
+################################################################################
+# A test program to write a large NetCDF file and see if we can read it.
+#
+# The file is written with a defined number of variables, each of a required
+# size.  Each variable consists of one or more 'slices', each 1000*1000 cells.
+# Each element of each slice of each variable is written with a *unique* value
+# that is checked when the file is read.
+#
+# So, element E of slice S ov variable V has the float value:
+#     V*1000 + S . E*1.0e-1
+# which will look like:
+#     VVVSSSS.0EEEEEE
+################################################################################
+
+import sys
+import getopt
+import time
+import Numeric
+import Scientific.IO.NetCDF as nc
+
+######
+# Various constants
+######
+
+# default modifier and number of files
+DEF_MODIFIER = 'M'
+
+# array slice definitions
+XDIM = 1024*1024
+SLICE_SIZE = XDIM*8	# 8 bytes / float
+MBYTE = 1024*1024
+
+# Dictionary for suffix modifiers
+Suffixes = {'M': 1024*1024,
+            'G': 1024*1024*1024
+           }
+
+######
+# Globals
+######
+
+TimeFileWrite = False
+TimeFileRead = False
+CloseAfterSlice = False
+Verbose = False
+
+# mask for variable names
+VarnameMask = 'var_%04d'
+
+
+######
+# Mainline code
+######
+
+##
+# @brief Read a NetCDF file - see if it as we wrote it.
+# @param filename The name of the file to read.
+# @param variable_size Size of variable, in bytes.
+# @param num_variables Number of variables.
+def read_file(filename, variable_size, num_variables):
+    # create a data array slice
+    slice_array_mask = Numeric.ones((XDIM,), 'd')
+
+    # if timing file read, get start time
+    if TimeFileRead:
+        start_file_read = time.time()
+
+    fid = nc.NetCDFFile(filename, 'r')
+
+    # num file bytes read counter
+    file_bytes_read = 0
+
+    result = True
+
+    for var_num in xrange(num_variables):
+        varname = VarnameMask % var_num
+        var = fid.variables[varname]
+
+        # num variable bytes written counter
+        var_bytes_read = 0
+
+        slice_number = 0
+        while var_bytes_read < variable_size:
+            var_bytes_read += SLICE_SIZE
+            file_bytes_read += SLICE_SIZE
+
+            # create expected slice array
+            slice_array = slice_array_mask*slice_number + var_num*1000
+            for x in xrange(XDIM):
+                slice_array[x] += x*1.0e-7
+
+            if Verbose:
+                print ('File %s, variable %s, reading slice %d: '
+                       'var=%.1fMiB, file=%.1fMiB' %
+                       (filename, varname, slice_number, float(var_bytes_read)/MBYTE,
+                        float(file_bytes_read)/MBYTE))
+
+            var_array = fid.variables[varname][slice_number,:]
+            if var_array != slice_array:
+                result = False
+                if Verbose:
+                    print 'Read variable %s, slice %d: got unexpected value' % \
+                          (varname, slice_number)
+                    for x in xrange(XDIM):
+                        if var_array[x] != slice_array[x]:
+                            print 'Got %f, expected %f' % (var_array[x], slice_array[x])
+                    sys.exit(0)
+                print 'Read variable %s, slice %d: got unexpected value' % \
+                      (varname, slice_number)
+                for x in xrange(XDIM):
+                    if var_array[x] != slice_array[x]:
+                        print 'Got %f, expected %f' % (var_array[x], slice_array[x])
+                        sys.exit(0)
+
+            slice_number += 1
+
+            if CloseAfterSlice:
+                fid.close()
+                fid = nc.NetCDFFile(filename, 'r')
+
+    fid.close()
+
+    if TimeFileRead:
+        stop_file_read = time.time()
+        print ('Time to read file: %.2f sec' %
+               (stop_file_read - start_file_read))
+
+    return result
+
+
+##
+# @brief Write a NetCDF file with set number of variables of a defined size.
+# @param variable_size Size of variable, in bytes.
+# @param num_variables Number of required variables.
+# @return The filename of the created file.
+# @note Variable slice writes are interleaved.
+def write_file(variable_size, num_variables=1):
+    # set file and variable name masks
+    filename = 'test.nc'
+
+    # create a data array slice
+    slice_array_mask = Numeric.ones((XDIM,), 'd')
+
+    # if timing file write, remember start time
+    if TimeFileWrite:
+        start_file_write = time.time()
+
+    fid = nc.NetCDFFile(filename, 'w')
+    fid.createDimension('y', None)
+    fid.createDimension('x', XDIM)
+
+    # create all required variables
+    for i in xrange(num_variables):
+        varname = VarnameMask % i
+        fid.createVariable(varname, 'd', ('y', 'x'))
+
+    # set counters to zero
+    var_bytes_written = 0
+    file_bytes_written = 0
+    slice_number = 0
+
+    while var_bytes_written < variable_size:
+        var_bytes_written += SLICE_SIZE
+
+        for var_num in xrange(num_variables):
+            varname = VarnameMask % var_num
+
+            # create unique slice array
+            slice_array = slice_array_mask*slice_number + var_num*1000
+            for x in xrange(XDIM):
+                slice_array[x] += x*1.0e-7
+
+            fid.variables[varname][slice_number,:] = slice_array
+
+            if CloseAfterSlice:
+                fid.close()
+                fid = nc.NetCDFFile(filename, 'a')
+
+            file_bytes_written += SLICE_SIZE
+
+            if Verbose:
+                print ('File %s, variable %s, writing slice %d: '
+                       'var=%.1fMiB, file=%.1fMiB' %
+                       (filename, varname, slice_number,
+                        float(var_bytes_written)/MBYTE,
+                        float(file_bytes_written)/MBYTE))
+
+        slice_number += 1
+
+    if TimeFileWrite:
+        stop_file_write = time.time()
+        print ('Time to write file: %.2f sec' %
+               (stop_file_write - start_file_write))
+
+    return filename
+
+
+##
+# @brief Provide help for the befuddled user.
+# @return Doesn't, calls sys.exit().
+def usage(msg=None):
+    print "Usage: write_large_files <opts> <varsize> [<numvars>]"
+    print ""
+    print "where <varsize> is a number followed by an optional modifier:"
+    print "                    1024M or 4G"
+    print "                the assumed modifier if none is given is '%s'." \
+          % DEF_MODIFIER
+    print "  and <numvars> is the number of variables of the above size"
+    print "                    to write.  If not supplied, 1 is assumed."
+    print "  and <opts>    is zero or more of:"
+    print "                    -c s    close & open the output file after"
+    print "                            each variable slice is read/written,"
+    print "                    -t rf   time the complete file read,"
+    print "                    -t wf   time the complete file write,"
+
+    if msg:
+        derr, "\n%s" % msg
+
+    sys.exit(10)
+
+def main(argv=None):
+    global TimeFileWrite, TimeFileRead, CloseAfterSlice, Verbose
+
+    if argv is None:
+        argv = sys.argv
+
+    # parse command line args
+    try:
+        opts, args = getopt.getopt(argv[1:], "c:t:v", ["help"])
+    except getopt.error, msg:
+        usage(msg)
+
+    for (opt, optarg) in opts:
+        if opt == '-c':
+            optargchar = optarg[0].lower()
+            if optargchar == 's':
+                CloseAfterSlice = True
+            else:
+                usage("Unrecognized -c suboption: %s" % optarg)
+        elif opt == '-t':
+            optargchar = optarg[0].lower()
+            if optargchar == 'r':
+                optargchar = optarg[1].lower()
+                if optargchar == 'f':
+                    TimeFileRead = True
+                else:
+                    usage("Unrecognized -t suboption: %s" % optarg)
+            elif optargchar == 'w':
+                optargchar = optarg[1].lower()
+                if optargchar == 'f':
+                    TimeFileWrite = True
+                else:
+                    usage("Unrecognized -t suboption: %s" % optarg)
+            else:
+                usage("Unrecognized -t suboption: %s" % optarg)
+        elif opt == '-v':
+            Verbose = True
+        else:
+            usage("Unrecognized option: %s" % opt)
+
+    if len(args) != 1 and len(args) != 2:
+        usage()
+
+    var_size = args[0][:-1]
+    modifier =args[0][-1]
+
+    if modifier in '0123456789':
+        var_size = args[0]
+        modifier = DEF_MODIFIER
+    modifier = Suffixes.get(modifier, None)
+    if modifier is None:
+        usage()
+        
+    try:
+        var_size = int(var_size) * modifier
+    except:
+        usage()
+
+    num_vars = 1
+    if len(args) == 2:
+        try:
+            num_vars = int(args[1])
+        except:
+            usage()
+
+    # write the required file
+    filename = write_file(var_size, num_vars)
+        
+    # read the file to see if is as expected
+    if not read_file(filename, var_size, num_vars):
+        print "Didn't read data that we read!?"
+        return 10
+
+    print 'Read/write of NectCDF file was correct'
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
Index: /misc/tools/write_large_files/rwil_big_file.py
===================================================================
--- /misc/tools/write_large_files/rwil_big_file.py	(revision 6739)
+++ /misc/tools/write_large_files/rwil_big_file.py	(revision 6739)
@@ -0,0 +1,295 @@
+#!/usr/bin/env python
+################################################################################
+# A test program to write a large NetCDF file and see if we can read it.
+#
+# The file is written with a defined number of variables, each of a required
+# size.  Each variable consists of one or more 'slices', each 1000*1000 cells.
+# Each element of each slice of each variable is written with a *unique* value
+# that is checked when the file is read.
+#
+# So, element E of slice S ov variable V has the float value:
+#     V*1000 + S . E*1.0e-1
+# which will look like:
+#     VVVSSSS.0EEEEEE
+################################################################################
+
+import sys
+import getopt
+import time
+import Numeric
+import Scientific.IO.NetCDF as nc
+
+######
+# Various constants
+######
+
+# default modifier and number of files
+DEF_MODIFIER = 'M'
+
+# array slice definitions
+XDIM = 1024*1024
+SLICE_SIZE = XDIM*8	# 8 bytes / float
+MBYTE = 1024*1024
+
+# Dictionary for suffix modifiers
+Suffixes = {'M': 1024*1024,
+            'G': 1024*1024*1024
+           }
+
+######
+# Globals
+######
+
+TimeFileWrite = False
+TimeFileRead = False
+CloseAfterSlice = False
+Verbose = False
+
+# mask for variable names
+VarnameMask = 'var_%04d'
+
+
+######
+# Mainline code
+######
+
+##
+# @brief Read a NetCDF file - see if it as we wrote it.
+# @param filename The name of the file to read.
+# @param variable_size Size of variable, in bytes.
+# @param num_variables Number of variables.
+def read_file(filename, variable_size, num_variables):
+    # create a data array slice
+    slice_array_mask = Numeric.ones((XDIM,), 'd')
+
+    # if timing file read, get start time
+    if TimeFileRead:
+        start_file_read = time.time()
+
+    fid = nc.NetCDFFile(filename, 'r')
+
+    # num file bytes read counter
+    file_bytes_read = 0
+
+    for var_num in xrange(num_variables):
+        varname = VarnameMask % var_num
+
+        # num variable bytes written counter
+        var_bytes_read = 0
+
+        slice_number = 0
+        while var_bytes_read < variable_size:
+            var_bytes_read += SLICE_SIZE
+            file_bytes_read += SLICE_SIZE
+
+            # create expected slice array
+            slice_array = slice_array_mask*slice_number + var_num*1000
+            for x in xrange(XDIM):
+                slice_array[x] += x*1.0e-7
+
+            if Verbose:
+                print ('File %s, variable %s, reading slice %d: '
+                       'var=%.1fMiB, file=%.1fMiB' %
+                       (filename, varname, slice_number,
+                        float(var_bytes_read)/MBYTE,
+                        float(file_bytes_read)/MBYTE))
+
+            var_array = fid.variables[varname][slice_number,:]
+
+            if var_array != slice_array:
+                print 'Read variable %s, slice %d: got unexpected value' % \
+                      (varname, slice_number)
+                for x in xrange(XDIM):
+                    if var_array[x] != slice_array[x]:
+                        print 'Index %d, got %f, expected %f' % \
+                              (x, var_array[x], slice_array[x])
+                        return False
+
+            slice_number += 1
+
+            if CloseAfterSlice:
+                fid.close()
+                fid = nc.NetCDFFile(filename, 'r')
+
+    fid.close()
+
+    if TimeFileRead:
+        stop_file_read = time.time()
+        print ('Time to read file: %.2f sec' %
+               (stop_file_read - start_file_read))
+
+    return True
+
+
+##
+# @brief Write a NetCDF file with set number of variables of a defined size.
+# @param variable_size Size of variable, in bytes.
+# @param num_variables Number of required variables.
+# @return The filename of the created file.
+# @note Variable slice writes are interleaved.
+def write_file(variable_size, num_variables=1):
+    # set file and variable name masks
+    filename = 'test.nc'
+
+    # create a data array slice
+    slice_array_mask = Numeric.ones((XDIM,), 'd')
+
+    # if timing file write, remember start time
+    if TimeFileWrite:
+        start_file_write = time.time()
+
+    fid = nc.NetCDFFile(filename, 'wl')
+    fid.createDimension('y', None)
+    fid.createDimension('x', XDIM)
+
+    # create all required variables
+    for i in xrange(num_variables):
+        varname = VarnameMask % i
+        fid.createVariable(varname, 'd', ('y', 'x'))
+
+    # set counters to zero
+    var_bytes_written = 0
+    file_bytes_written = 0
+    slice_number = 0
+
+    while var_bytes_written < variable_size:
+        var_bytes_written += SLICE_SIZE
+
+        for var_num in xrange(num_variables):
+            varname = VarnameMask % var_num
+
+            # create unique slice array
+            slice_array = slice_array_mask*slice_number + var_num*1000
+            for x in xrange(XDIM):
+                slice_array[x] += x*1.0e-7
+
+            fid.variables[varname][slice_number,:] = slice_array
+
+            if CloseAfterSlice:
+                fid.close()
+                fid = nc.NetCDFFile(filename, 'al')
+#                fid = nc.NetCDFFile(filename, 'a')
+
+            file_bytes_written += SLICE_SIZE
+
+            if Verbose:
+                print ('File %s, variable %s, writing slice %d: '
+                       'var=%.1fMiB, file=%.1fMiB' %
+                       (filename, varname, slice_number,
+                        float(var_bytes_written)/MBYTE,
+                        float(file_bytes_written)/MBYTE))
+
+        slice_number += 1
+
+    fid.close()
+
+    if TimeFileWrite:
+        stop_file_write = time.time()
+        print ('Time to write file: %.2f sec' %
+               (stop_file_write - start_file_write))
+
+    return filename
+
+
+##
+# @brief Provide help for the befuddled user.
+# @return Doesn't, calls sys.exit().
+def usage(msg=None):
+    print "Usage: write_large_files <opts> <varsize> [<numvars>]"
+    print ""
+    print "where <varsize> is a number followed by an optional modifier:"
+    print "                    1024M or 4G"
+    print "                the assumed modifier if none is given is '%s'." \
+          % DEF_MODIFIER
+    print "  and <numvars> is the number of variables of the above size"
+    print "                    to write.  If not supplied, 1 is assumed."
+    print "  and <opts>    is zero or more of:"
+    print "                    -c s    close & open the output file after"
+    print "                            each variable slice is read/written,"
+    print "                    -t rf   time the complete file read,"
+    print "                    -t wf   time the complete file write,"
+
+    if msg:
+        derr, "\n%s" % msg
+
+    sys.exit(10)
+
+def main(argv=None):
+    global TimeFileWrite, TimeFileRead, CloseAfterSlice, Verbose
+
+    if argv is None:
+        argv = sys.argv
+
+    # parse command line args
+    try:
+        opts, args = getopt.getopt(argv[1:], "c:t:v", ["help"])
+    except getopt.error, msg:
+        usage(msg)
+
+    for (opt, optarg) in opts:
+        if opt == '-c':
+            optargchar = optarg[0].lower()
+            if optargchar == 's':
+                CloseAfterSlice = True
+            else:
+                usage("Unrecognized -c suboption: %s" % optarg)
+        elif opt == '-t':
+            optargchar = optarg[0].lower()
+            if optargchar == 'r':
+                optargchar = optarg[1].lower()
+                if optargchar == 'f':
+                    TimeFileRead = True
+                else:
+                    usage("Unrecognized -t suboption: %s" % optarg)
+            elif optargchar == 'w':
+                optargchar = optarg[1].lower()
+                if optargchar == 'f':
+                    TimeFileWrite = True
+                else:
+                    usage("Unrecognized -t suboption: %s" % optarg)
+            else:
+                usage("Unrecognized -t suboption: %s" % optarg)
+        elif opt == '-v':
+            Verbose = True
+        else:
+            usage("Unrecognized option: %s" % opt)
+
+    if len(args) != 1 and len(args) != 2:
+        usage()
+
+    var_size = args[0][:-1]
+    modifier =args[0][-1]
+
+    if modifier in '0123456789':
+        var_size = args[0]
+        modifier = DEF_MODIFIER
+    modifier = Suffixes.get(modifier, None)
+    if modifier is None:
+        usage()
+        
+    try:
+        var_size = int(var_size) * modifier
+    except:
+        usage()
+
+    num_vars = 1
+    if len(args) == 2:
+        try:
+            num_vars = int(args[1])
+        except:
+            usage()
+
+    # write the required file
+    filename = write_file(var_size, num_vars)
+        
+    # read the file to see if is as expected
+    if not read_file(filename, var_size, num_vars):
+        print "Didn't read data that we read!?"
+        return 10
+
+    print 'Read/write of NectCDF file was correct'
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
Index: /misc/tools/write_large_files/write_big_file.py
===================================================================
--- /misc/tools/write_large_files/write_big_file.py	(revision 6739)
+++ /misc/tools/write_large_files/write_big_file.py	(revision 6739)
@@ -0,0 +1,215 @@
+#!/usr/bin/env python
+################################################################################
+# A test program to write large NetCDF files.
+################################################################################
+
+import sys
+import getopt
+import time
+import Numeric
+import Scientific.IO.NetCDF as nc
+
+######
+# Various constants
+######
+
+# default modifier and number of files
+DEF_MODIFIER = 'M'
+
+# array slice definitions
+XDIM = 1024*1024
+SLICE_SIZE = XDIM*4	# 4 bytes / int
+MBYTE = 1024*1024
+
+# Dictionary for suffix modifiers
+Suffixes = {'M': 1024*1024,
+            'G': 1024*1024*1024
+           }
+
+######
+# Globals
+######
+
+TimeFirstWrite = False
+TimeVarWrite = False
+TimeFileWrite = False
+
+CloseAfterSlice = False
+CloseAfterVar = False
+
+Verbose = False
+
+######
+# mainline code
+######
+
+##
+# @brief Write a NetCDF file with set number of variables of a defined size.
+# @param variable_size Size of variable, in bytes.
+# @param num_variables Number of required variables.
+def write_file(variable_size, num_variables=1):
+    # set file and variable name masks
+    filename = 'test.nc'
+    varname_mask = 'var_%04d'
+
+    # create a data array slice
+    slice_array = Numeric.ones((XDIM,), 'i')
+
+    # if timing file write, remember start time
+    if TimeFileWrite:
+        start_file_write = time.time()
+
+    fid = nc.NetCDFFile(filename, 'w')
+    fid.createDimension('y', None)
+    fid.createDimension('x', XDIM)
+
+    # num file bytes written counter
+    file_bytes_written = 0
+
+    for i in xrange(num_variables):
+        varname = varname_mask % i
+        fid.createVariable(varname, 'i', ('y', 'x'))
+
+        # num variable bytes written counter
+        var_bytes_written = 0
+
+        if TimeVarWrite:
+            start_var_write = time.time()
+
+        index = 0
+        while var_bytes_written < variable_size:
+            var_bytes_written += SLICE_SIZE
+            file_bytes_written += SLICE_SIZE
+
+            if TimeFirstWrite and index == 0:
+                start_first_var_write = time.time()
+
+            if Verbose:
+                print ('File %s, variable %s, writing slice %d: '
+                       'var=%.1fMiB, file=%.1fMiB' %
+                       (filename, varname, index, float(var_bytes_written)/MBYTE,
+                        float(file_bytes_written)/MBYTE))
+
+            fid.variables[varname][index,:] = slice_array
+
+            if TimeFirstWrite and index == 0:
+                stop_first_var_write = time.time()
+                print ("Time to do first write for variable %s: %.2f sec" %
+                       (varname,
+                        (stop_first_var_write - start_first_var_write)))
+
+            index += 1
+
+            if CloseAfterSlice:
+                fid.close()
+                fid = nc.NetCDFFile(filename, 'a')
+        if CloseAfterVar:
+            fid.close()
+            fid = nc.NetCDFFile(filename, 'a')
+
+        if TimeVarWrite:
+            stop_var_write = time.time()
+            print ('Time to write variable %s: %.2f sec' %
+                   (varname, (stop_first_var_write - start_first_var_write)))
+
+    fid.close()
+
+    if TimeFileWrite:
+        stop_file_write = time.time()
+        print ('Time to write file: %.2f sec' %
+               (stop_file_write - start_file_write))
+
+##
+# @brief Provide help for the befuddled user.
+# @return Doesn't, calls sys.exit().
+def usage(msg=None):
+    print "Usage: write_large_files <opts> <varsize> [<numvars>]"
+    print ""
+    print "where <varsize> is a number followed by an optional modifier:"
+    print "                    1024M or 4G"
+    print "                the assumed modifier is '%s' if none is given." \
+          % DEF_MODIFIER
+    print "  and <numvars> is the number of variables of the above size"
+    print "                    to write.  If not supplied, 1 is assumed."
+    print "  and <opts>    is zero or more of:"
+    print "                    -c s[lice]  close & open the output file after"
+    print "                                each variable slice is written,"
+    print "                    -c v[ar]    close & open the output file after"
+    print "                                each variable is written,"
+    print "                    -t f[irst]  time the first var slice write,"
+    print "                    -t v[ar]    time the complete var write,"
+    print "                    -t w[hole]  time the complete file write,"
+
+    if msg:
+        print "\n%s" % msg
+
+    sys.exit(10)
+
+def main(argv=None):
+    global TimeFirstWrite, TimeVarWrite, TimeFileWrite
+    global CloseAfterSlice, CloseAfterVar
+    global Verbose
+
+    if argv is None:
+        argv = sys.argv
+
+    # parse command line args
+    try:
+        opts, args = getopt.getopt(argv[1:], "c:ht:v", ["help"])
+    except getopt.error, msg:
+        usage(msg)
+
+    for (opt, optarg) in opts:
+        if opt == '-c':
+            optargchar = optarg[0].lower()
+            if optargchar == 's':
+                CloseAfterSlice = True
+            elif optargchar == 'v':
+                CloseAfterVar = True
+            else:
+                usage("Unrecognized -c suboption: %s" % optarg)
+        elif opt == '-t':
+            optargchar = optarg[0].lower()
+            if optargchar == 'f':
+                TimeFirstWrite = True
+            elif optargchar == 'v':
+                TimeVarWrite = True
+            elif optargchar == 'w':
+                TimeFileWrite = True
+            else:
+                usage("Unrecognized -c suboption: %s" % optarg)
+        elif opt == '-v':
+            Verbose = True
+        else:
+            usage("Unrecognized option: %s" % opt)
+
+    if len(args) != 1 and len(args) != 2:
+        usage()
+
+    var_size = args[0][:-1]
+    modifier =args[0][-1]
+
+    if modifier in '0123456789':
+        var_size = args[0]
+        modifier = DEF_MODIFIER
+    modifier = Suffixes.get(modifier, None)
+    if modifier is None:
+        usage()
+        
+    try:
+        var_size = int(var_size) * modifier
+    except:
+        usage()
+
+    num_vars = 1
+    if len(args) == 2:
+        try:
+            num_vars = int(args[1])
+        except:
+            usage()
+
+    # call production function
+    write_file(var_size, num_vars)
+        
+if __name__ == "__main__":
+    sys.exit(main())
