Changeset 6820


Ignore:
Timestamp:
Apr 16, 2009, 3:44:12 PM (15 years ago)
Author:
rwilson
Message:

Changed num.any(a!=b) into num.allclose(a,b) for Windows. Fine on Linux. Also faster.

Location:
branches/numpy_misc/tools/write_large_files
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • branches/numpy_misc/tools/write_large_files/rwi4_big_file.py

    r6817 r6820  
    88# that is checked when the file is read.
    99#
    10 # So, element E of slice S ov variable V has the float value:
    11 #     V*1000 + S . E*1.0e-1
     10# So, for variable V (numbering from 0), slice S (starting from 0), element
     11# E will range from 0 to 10**6-1, and the element value will be:
     12#     S*100 + V + E*1.0E-6
    1213# which will look like:
    13 #     VVVSSSS.0EEEEEE
     14#     SSSSVV.EEEEEE
     15# This implies that the maximum number of variables written is 100 (0 to 99).
    1416################################################################################
    1517
     
    2830
    2931# array slice definitions
    30 XDIM = 1024*1024
     32XDIM = 1000*1000
    3133SLICE_SIZE = XDIM*8     # 8 bytes / float
    3234MBYTE = 1024*1024
     
    4749
    4850# mask for variable names
    49 VarnameMask = 'var_%04d'
     51VarnameMask = 'var_%02d'
    5052
    5153
     
    7476    for var_num in xrange(num_variables):
    7577        varname = VarnameMask % var_num
    76         var = fid.variables[varname]
    7778
    7879        # num variable bytes written counter
     
    8586
    8687            # create expected slice array
    87             slice_array = slice_array_mask*slice_number + var_num*1000
    88             for x in xrange(XDIM):
    89                 slice_array[x] += x*1.0e-7
     88            slice_array = slice_array_mask*slice_number*100 + var_num
     89            slice_array += ElementIndexArray
    9090
    9191            if Verbose:
     
    9696                        float(file_bytes_read)/MBYTE))
    9797
    98 #            var_array = fid.variables[varname][slice_number,:]
    99             var_array = var[slice_number,:]
    100             if num.any(var_array != slice_array):
     98            var_array = fid.variables[varname][slice_number,:]
     99            if not num.allclose(var_array, slice_array):
     100##            if num.any(var_array != slice_array):
    101101                print 'Read variable %s, slice %d: got unexpected value' % \
    102102                      (varname, slice_number)
     
    161161
    162162            # create unique slice array
    163             slice_array = slice_array_mask*slice_number + var_num*1000
    164             for x in xrange(XDIM):
    165                 slice_array[x] += x*1.0e-7
     163            slice_array = slice_array_mask*slice_number*100 + var_num
     164            slice_array += ElementIndexArray
    166165
    167166            fid.variables[varname][slice_number,:] = slice_array
     
    169168            if CloseAfterSlice:
    170169                fid.close()
    171                 fid = nc.NetCDFFile(filename, 'a4')
     170#                fid = nc.NetCDFFile(filename, 'a4')
     171                fid = nc.NetCDFFile(filename, 'a')
    172172
    173173            file_bytes_written += SLICE_SIZE
     
    204204    print "  and <numvars> is the number of variables of the above size"
    205205    print "                    to write.  If not supplied, 1 is assumed."
     206    print "                    There can be at most 100 variables."
    206207    print "  and <opts>    is zero or more of:"
    207208    print "                    -c s    close & open the output file after"
     
    217218def main(argv=None):
    218219    global TimeFileWrite, TimeFileRead, CloseAfterSlice, Verbose
     220    global ElementIndexArray
    219221
    220222    if argv is None:
     
    259261
    260262    var_size = args[0][:-1]
    261     modifier =args[0][-1]
     263    modifier = args[0][-1]
    262264
    263265    if modifier in '0123456789':
     
    280282            usage()
    281283
     284    if num_vars > 100:
     285        usage()
     286
     287    # initialize the global element index array which contains
     288    # 0.EEEEEE in each element.
     289    ElementIndexArray = num.ndarray((XDIM,), 'd')
     290    for x in xrange(XDIM):
     291        ElementIndexArray[x] = x*1.0e-6
     292   
    282293    # write the required file
    283294    filename = write_file(var_size, num_vars)
     
    288299        return 10
    289300
    290     print 'Read/write of NectCDF file was correct'
     301#    print 'Read/write of NectCDF file was correct'
    291302    return 0
    292303
  • branches/numpy_misc/tools/write_large_files/rwi_big_file.py

    r6817 r6820  
    88# that is checked when the file is read.
    99#
    10 # So, element E of slice S ov variable V has the float value:
    11 #     V*1000 + S . E*1.0e-1
     10# So, for variable V (numbering from 0), slice S (starting from 0), element
     11# E will range from 0 to 10**6-1, and the element value will be:
     12#     S*100 + V + E*1.0E-6
    1213# which will look like:
    13 #     VVVSSSS.0EEEEEE
     14#     SSSSVV.EEEEEE
     15# This implies that the maximum number of variables written is 100 (0 to 99).
    1416################################################################################
    1517
     
    2830
    2931# array slice definitions
    30 XDIM = 1024*1024
     32XDIM = 1000*1000
    3133SLICE_SIZE = XDIM*8     # 8 bytes / float
    3234MBYTE = 1024*1024
     
    4749
    4850# mask for variable names
    49 VarnameMask = 'var_%04d'
     51VarnameMask = 'var_%02d'
    5052
    5153
     
    8789
    8890            # create expected slice array
    89             slice_array = slice_array_mask*slice_number + var_num*1000
    90             for x in xrange(XDIM):
    91                 slice_array[x] += x*1.0e-7
     91            slice_array = slice_array_mask*slice_number*100 + var_num
     92            slice_array += ElementIndexArray
    9293
    9394            if Verbose:
    9495                print ('File %s, variable %s, reading slice %d: '
    9596                       'var=%.1fMiB, file=%.1fMiB' %
    96                        (filename, varname, slice_number, float(var_bytes_read)/MBYTE,
     97                       (filename, varname, slice_number,
     98                        float(var_bytes_read)/MBYTE,
    9799                        float(file_bytes_read)/MBYTE))
    98100
    99101            var_array = fid.variables[varname][slice_number,:]
    100             if num.any(var_array != slice_array):
    101                 result = False
    102                 if Verbose:
    103                     print 'Read variable %s, slice %d: got unexpected value' % \
    104                           (varname, slice_number)
    105                     for x in xrange(XDIM):
    106                         if var_array[x] != slice_array[x]:
    107                             print 'Got %f, expected %f' % (var_array[x], slice_array[x])
    108                     sys.exit(0)
     102            if not num.allclose(var_array, slice_array):
     103##            if num.any(var_array != slice_array):
    109104                print 'Read variable %s, slice %d: got unexpected value' % \
    110105                      (varname, slice_number)
    111106                for x in xrange(XDIM):
    112107                    if var_array[x] != slice_array[x]:
    113                         print 'Got %f, expected %f' % (var_array[x], slice_array[x])
    114                         sys.exit(0)
     108                        print 'Index %d, got %f, expected %f' % \
     109                              (x, var_array[x], slice_array[x])
     110                        return False
    115111
    116112            slice_number += 1
     
    127123               (stop_file_read - start_file_read))
    128124
    129     return result
     125    return True
    130126
    131127
     
    168164
    169165            # create unique slice array
    170             slice_array = slice_array_mask*slice_number + var_num*1000
    171             for x in xrange(XDIM):
    172                 slice_array[x] += x*1.0e-7
     166            slice_array = slice_array_mask*slice_number*100 + var_num
     167            slice_array += ElementIndexArray
    173168
    174169            fid.variables[varname][slice_number,:] = slice_array
     
    188183
    189184        slice_number += 1
     185
     186    fid.close()
    190187
    191188    if TimeFileWrite:
     
    209206    print "  and <numvars> is the number of variables of the above size"
    210207    print "                    to write.  If not supplied, 1 is assumed."
     208    print "                    There can be at most 100 variables."
    211209    print "  and <opts>    is zero or more of:"
    212210    print "                    -c s    close & open the output file after"
     
    222220def main(argv=None):
    223221    global TimeFileWrite, TimeFileRead, CloseAfterSlice, Verbose
     222    global ElementIndexArray
    224223
    225224    if argv is None:
     
    264263
    265264    var_size = args[0][:-1]
    266     modifier =args[0][-1]
     265    modifier = args[0][-1]
    267266
    268267    if modifier in '0123456789':
     
    285284            usage()
    286285
     286    if num_vars > 100:
     287        usage()
     288
     289    # initialize the global element index array which contains
     290    # 0.EEEEEE in each element.
     291    ElementIndexArray = num.ndarray((XDIM,), 'd')
     292    for x in xrange(XDIM):
     293        ElementIndexArray[x] = x*1.0e-6
     294   
    287295    # write the required file
    288296    filename = write_file(var_size, num_vars)
     
    293301        return 10
    294302
    295     print 'Read/write of NectCDF file was correct'
     303#    print 'Read/write of NectCDF file was correct'
    296304    return 0
    297305
  • branches/numpy_misc/tools/write_large_files/rwil_big_file.py

    r6817 r6820  
    88# that is checked when the file is read.
    99#
    10 # So, element E of slice S ov variable V has the float value:
    11 #     V*1000 + S . E*1.0e-1
     10# So, for variable V (numbering from 0), slice S (starting from 0), element
     11# E will range from 0 to 10**6-1, and the element value will be:
     12#     S*100 + V + E*1.0E-6
    1213# which will look like:
    13 #     VVVSSSS.0EEEEEE
     14#     SSSSVV.EEEEEE
     15# This implies that the maximum number of variables written is 100 (0 to 99).
    1416################################################################################
    1517
     
    2830
    2931# array slice definitions
    30 XDIM = 1024*1024
     32XDIM = 1000*1000
    3133SLICE_SIZE = XDIM*8     # 8 bytes / float
    3234MBYTE = 1024*1024
     
    4749
    4850# mask for variable names
    49 VarnameMask = 'var_%04d'
     51VarnameMask = 'var_%02d'
    5052
    5153
     
    8486
    8587            # create expected slice array
    86             slice_array = slice_array_mask*slice_number + var_num*1000
    87             for x in xrange(XDIM):
    88                 slice_array[x] += x*1.0e-7
     88            slice_array = slice_array_mask*slice_number*100 + var_num
     89            slice_array += ElementIndexArray
    8990
    9091            if Verbose:
     
    9697
    9798            var_array = fid.variables[varname][slice_number,:]
    98 
    99             if num.any(var_array != slice_array):
     99            if not num.allclose(var_array, slice_array):
     100##            if num.any(var_array != slice_array):
    100101                print 'Read variable %s, slice %d: got unexpected value' % \
    101102                      (varname, slice_number)
     
    160161
    161162            # create unique slice array
    162             slice_array = slice_array_mask*slice_number + var_num*1000
    163             for x in xrange(XDIM):
    164                 slice_array[x] += x*1.0e-7
     163            slice_array = slice_array_mask*slice_number*100 + var_num
     164            slice_array += ElementIndexArray
    165165
    166166            fid.variables[varname][slice_number,:] = slice_array
     
    168168            if CloseAfterSlice:
    169169                fid.close()
    170                 fid = nc.NetCDFFile(filename, 'al')
    171 #                fid = nc.NetCDFFile(filename, 'a')
     170#                fid = nc.NetCDFFile(filename, 'al')
     171                fid = nc.NetCDFFile(filename, 'a')
    172172
    173173            file_bytes_written += SLICE_SIZE
     
    204204    print "  and <numvars> is the number of variables of the above size"
    205205    print "                    to write.  If not supplied, 1 is assumed."
     206    print "                    There can be at most 100 variables."
    206207    print "  and <opts>    is zero or more of:"
    207208    print "                    -c s    close & open the output file after"
     
    217218def main(argv=None):
    218219    global TimeFileWrite, TimeFileRead, CloseAfterSlice, Verbose
     220    global ElementIndexArray
    219221
    220222    if argv is None:
     
    259261
    260262    var_size = args[0][:-1]
    261     modifier =args[0][-1]
     263    modifier = args[0][-1]
    262264
    263265    if modifier in '0123456789':
     
    280282            usage()
    281283
     284    if num_vars > 100:
     285        usage()
     286
     287    # initialize the global element index array which contains
     288    # 0.EEEEEE in each element.
     289    ElementIndexArray = num.ndarray((XDIM,), 'd')
     290    for x in xrange(XDIM):
     291        ElementIndexArray[x] = x*1.0e-6
     292   
    282293    # write the required file
    283294    filename = write_file(var_size, num_vars)
     
    288299        return 10
    289300
    290     print 'Read/write of NectCDF file was correct'
     301#    print 'Read/write of NectCDF file was correct'
    291302    return 0
    292303
Note: See TracChangeset for help on using the changeset viewer.