- Timestamp:
- Apr 26, 2009, 5:35:16 PM (16 years ago)
- Location:
- misc/tools/write_large_files
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
misc/tools/write_large_files/rwi4_big_file.py
r6739 r6906 8 8 # that is checked when the file is read. 9 9 # 10 # So, element E of slice S ov variable V has the float value: 11 # V*1000 + S . E*1.0e-1 10 # So, for variable V (numbering from 0), slice S (starting from 0), element 11 # E will range from 0 to 10**6-1, and the element value will be: 12 # S*100 + V + E*1.0E-6 12 13 # which will look like: 13 # VVVSSSS.0EEEEEE 14 # SSSSVV.EEEEEE 15 # This implies that the maximum number of variables written is 100 (0 to 99). 14 16 ################################################################################ 15 17 … … 28 30 29 31 # array slice definitions 30 XDIM = 10 24*102432 XDIM = 1000*1000 31 33 SLICE_SIZE = XDIM*8 # 8 bytes / float 32 34 MBYTE = 1024*1024 … … 47 49 48 50 # mask for variable names 49 VarnameMask = 'var_%0 4d'51 VarnameMask = 'var_%02d' 50 52 51 53 … … 74 76 for var_num in xrange(num_variables): 75 77 varname = VarnameMask % var_num 76 var = fid.variables[varname]77 78 78 79 # num variable bytes written counter … … 85 86 86 87 # create expected slice array 87 slice_array = slice_array_mask*slice_number + var_num*1000 88 for x in xrange(XDIM): 89 slice_array[x] += x*1.0e-7 88 slice_array = slice_array_mask*slice_number*100 + var_num 89 slice_array += ElementIndexArray 90 90 91 91 if Verbose: … … 96 96 float(file_bytes_read)/MBYTE)) 97 97 98 #var_array = fid.variables[varname][slice_number,:]99 var_array = var[slice_number,:]100 if var_array != slice_array:98 var_array = fid.variables[varname][slice_number,:] 99 if not num.allclose(var_array, slice_array): 100 ## if num.any(var_array != slice_array): 101 101 print 'Read variable %s, slice %d: got unexpected value' % \ 102 102 (varname, slice_number) … … 161 161 162 162 # create unique slice array 163 slice_array = slice_array_mask*slice_number + var_num*1000 164 for x in xrange(XDIM): 165 slice_array[x] += x*1.0e-7 163 slice_array = slice_array_mask*slice_number*100 + var_num 164 slice_array += ElementIndexArray 166 165 167 166 fid.variables[varname][slice_number,:] = slice_array … … 169 168 if CloseAfterSlice: 170 169 fid.close() 171 fid = nc.NetCDFFile(filename, 'a4') 170 # fid = nc.NetCDFFile(filename, 'a4') 171 fid = nc.NetCDFFile(filename, 'a') 172 172 173 173 file_bytes_written += SLICE_SIZE … … 204 204 print " and <numvars> is the number of variables of the above size" 205 205 print " to write. If not supplied, 1 is assumed." 206 print " There can be at most 100 variables." 206 207 print " and <opts> is zero or more of:" 207 208 print " -c s close & open the output file after" … … 217 218 def main(argv=None): 218 219 global TimeFileWrite, TimeFileRead, CloseAfterSlice, Verbose 220 global ElementIndexArray 219 221 220 222 if argv is None: … … 259 261 260 262 var_size = args[0][:-1] 261 modifier = args[0][-1]263 modifier = args[0][-1] 262 264 263 265 if modifier in '0123456789': … … 280 282 usage() 281 283 284 if num_vars > 100: 285 usage() 286 287 # initialize the global element index array which contains 288 # 0.EEEEEE in each element. 289 ElementIndexArray = num.ndarray((XDIM,), 'd') 290 for x in xrange(XDIM): 291 ElementIndexArray[x] = x*1.0e-6 292 282 293 # write the required file 283 294 filename = write_file(var_size, num_vars) … … 288 299 return 10 289 300 290 print 'Read/write of NectCDF file was correct'301 # print 'Read/write of NectCDF file was correct' 291 302 return 0 292 303 -
misc/tools/write_large_files/rwi_big_file.py
r6739 r6906 8 8 # that is checked when the file is read. 9 9 # 10 # So, element E of slice S ov variable V has the float value: 11 # V*1000 + S . E*1.0e-1 10 # So, for variable V (numbering from 0), slice S (starting from 0), element 11 # E will range from 0 to 10**6-1, and the element value will be: 12 # S*100 + V + E*1.0E-6 12 13 # which will look like: 13 # VVVSSSS.0EEEEEE 14 # SSSSVV.EEEEEE 15 # This implies that the maximum number of variables written is 100 (0 to 99). 14 16 ################################################################################ 15 17 … … 28 30 29 31 # array slice definitions 30 XDIM = 10 24*102432 XDIM = 1000*1000 31 33 SLICE_SIZE = XDIM*8 # 8 bytes / float 32 34 MBYTE = 1024*1024 … … 47 49 48 50 # mask for variable names 49 VarnameMask = 'var_%0 4d'51 VarnameMask = 'var_%02d' 50 52 51 53 … … 87 89 88 90 # create expected slice array 89 slice_array = slice_array_mask*slice_number + var_num*1000 90 for x in xrange(XDIM): 91 slice_array[x] += x*1.0e-7 91 slice_array = slice_array_mask*slice_number*100 + var_num 92 slice_array += ElementIndexArray 92 93 93 94 if Verbose: 94 95 print ('File %s, variable %s, reading slice %d: ' 95 96 'var=%.1fMiB, file=%.1fMiB' % 96 (filename, varname, slice_number, float(var_bytes_read)/MBYTE, 97 (filename, varname, slice_number, 98 float(var_bytes_read)/MBYTE, 97 99 float(file_bytes_read)/MBYTE)) 98 100 99 101 var_array = fid.variables[varname][slice_number,:] 100 if var_array != slice_array: 101 result = False 102 if Verbose: 103 print 'Read variable %s, slice %d: got unexpected value' % \ 104 (varname, slice_number) 105 for x in xrange(XDIM): 106 if var_array[x] != slice_array[x]: 107 print 'Got %f, expected %f' % (var_array[x], slice_array[x]) 108 sys.exit(0) 102 if not num.allclose(var_array, slice_array): 103 ## if num.any(var_array != slice_array): 109 104 print 'Read variable %s, slice %d: got unexpected value' % \ 110 105 (varname, slice_number) 111 106 for x in xrange(XDIM): 112 107 if var_array[x] != slice_array[x]: 113 print 'Got %f, expected %f' % (var_array[x], slice_array[x]) 114 sys.exit(0) 108 print 'Index %d, got %f, expected %f' % \ 109 (x, var_array[x], slice_array[x]) 110 return False 115 111 116 112 slice_number += 1 … … 127 123 (stop_file_read - start_file_read)) 128 124 129 return result125 return True 130 126 131 127 … … 168 164 169 165 # create unique slice array 170 slice_array = slice_array_mask*slice_number + var_num*1000 171 for x in xrange(XDIM): 172 slice_array[x] += x*1.0e-7 166 slice_array = slice_array_mask*slice_number*100 + var_num 167 slice_array += ElementIndexArray 173 168 174 169 fid.variables[varname][slice_number,:] = slice_array … … 188 183 189 184 slice_number += 1 185 186 fid.close() 190 187 191 188 if TimeFileWrite: … … 209 206 print " and <numvars> is the number of variables of the above size" 210 207 print " to write. If not supplied, 1 is assumed." 208 print " There can be at most 100 variables." 211 209 print " and <opts> is zero or more of:" 212 210 print " -c s close & open the output file after" … … 222 220 def main(argv=None): 223 221 global TimeFileWrite, TimeFileRead, CloseAfterSlice, Verbose 222 global ElementIndexArray 224 223 225 224 if argv is None: … … 264 263 265 264 var_size = args[0][:-1] 266 modifier = args[0][-1]265 modifier = args[0][-1] 267 266 268 267 if modifier in '0123456789': … … 285 284 usage() 286 285 286 if num_vars > 100: 287 usage() 288 289 # initialize the global element index array which contains 290 # 0.EEEEEE in each element. 291 ElementIndexArray = num.ndarray((XDIM,), 'd') 292 for x in xrange(XDIM): 293 ElementIndexArray[x] = x*1.0e-6 294 287 295 # write the required file 288 296 filename = write_file(var_size, num_vars) … … 293 301 return 10 294 302 295 print 'Read/write of NectCDF file was correct'303 # print 'Read/write of NectCDF file was correct' 296 304 return 0 297 305 -
misc/tools/write_large_files/rwil_big_file.py
r6739 r6906 8 8 # that is checked when the file is read. 9 9 # 10 # So, element E of slice S ov variable V has the float value: 11 # V*1000 + S . E*1.0e-1 10 # So, for variable V (numbering from 0), slice S (starting from 0), element 11 # E will range from 0 to 10**6-1, and the element value will be: 12 # S*100 + V + E*1.0E-6 12 13 # which will look like: 13 # VVVSSSS.0EEEEEE 14 # SSSSVV.EEEEEE 15 # This implies that the maximum number of variables written is 100 (0 to 99). 14 16 ################################################################################ 15 17 … … 28 30 29 31 # array slice definitions 30 XDIM = 10 24*102432 XDIM = 1000*1000 31 33 SLICE_SIZE = XDIM*8 # 8 bytes / float 32 34 MBYTE = 1024*1024 … … 47 49 48 50 # mask for variable names 49 VarnameMask = 'var_%0 4d'51 VarnameMask = 'var_%02d' 50 52 51 53 … … 84 86 85 87 # create expected slice array 86 slice_array = slice_array_mask*slice_number + var_num*1000 87 for x in xrange(XDIM): 88 slice_array[x] += x*1.0e-7 88 slice_array = slice_array_mask*slice_number*100 + var_num 89 slice_array += ElementIndexArray 89 90 90 91 if Verbose: … … 96 97 97 98 var_array = fid.variables[varname][slice_number,:] 98 99 if var_array != slice_array:99 if not num.allclose(var_array, slice_array): 100 ## if num.any(var_array != slice_array): 100 101 print 'Read variable %s, slice %d: got unexpected value' % \ 101 102 (varname, slice_number) … … 160 161 161 162 # create unique slice array 162 slice_array = slice_array_mask*slice_number + var_num*1000 163 for x in xrange(XDIM): 164 slice_array[x] += x*1.0e-7 163 slice_array = slice_array_mask*slice_number*100 + var_num 164 slice_array += ElementIndexArray 165 165 166 166 fid.variables[varname][slice_number,:] = slice_array … … 168 168 if CloseAfterSlice: 169 169 fid.close() 170 fid = nc.NetCDFFile(filename, 'al')171 #fid = nc.NetCDFFile(filename, 'a')170 # fid = nc.NetCDFFile(filename, 'al') 171 fid = nc.NetCDFFile(filename, 'a') 172 172 173 173 file_bytes_written += SLICE_SIZE … … 204 204 print " and <numvars> is the number of variables of the above size" 205 205 print " to write. If not supplied, 1 is assumed." 206 print " There can be at most 100 variables." 206 207 print " and <opts> is zero or more of:" 207 208 print " -c s close & open the output file after" … … 217 218 def main(argv=None): 218 219 global TimeFileWrite, TimeFileRead, CloseAfterSlice, Verbose 220 global ElementIndexArray 219 221 220 222 if argv is None: … … 259 261 260 262 var_size = args[0][:-1] 261 modifier = args[0][-1]263 modifier = args[0][-1] 262 264 263 265 if modifier in '0123456789': … … 280 282 usage() 281 283 284 if num_vars > 100: 285 usage() 286 287 # initialize the global element index array which contains 288 # 0.EEEEEE in each element. 289 ElementIndexArray = num.ndarray((XDIM,), 'd') 290 for x in xrange(XDIM): 291 ElementIndexArray[x] = x*1.0e-6 292 282 293 # write the required file 283 294 filename = write_file(var_size, num_vars) … … 288 299 return 10 289 300 290 print 'Read/write of NectCDF file was correct'301 # print 'Read/write of NectCDF file was correct' 291 302 return 0 292 303
Note: See TracChangeset
for help on using the changeset viewer.