[7243] | 1 | #!/usr/bin/env python |
---|
| 2 | |
---|
| 3 | ''' |
---|
| 4 | A program to test differences in time and memory usage between Numeric |
---|
| 5 | and numpy code. |
---|
| 6 | |
---|
| 7 | Due to the above, this will run only on a machine that has both of the |
---|
| 8 | above packages installed. |
---|
| 9 | ''' |
---|
| 10 | |
---|
| 11 | |
---|
| 12 | import sys |
---|
| 13 | import os |
---|
| 14 | import time |
---|
| 15 | import gc |
---|
| 16 | import Numeric |
---|
| 17 | import numpy |
---|
| 18 | |
---|
| 19 | |
---|
[7278] | 20 | #ARRAY_SIZE = 1000 * 1000 |
---|
| 21 | ARRAY_SIZE = (40000, 3) |
---|
[7243] | 22 | |
---|
| 23 | |
---|
[7245] | 24 | # Numeric version |
---|
| 25 | def Numeric_ensure_numeric(A, typecode=None): |
---|
| 26 | """Ensure that sequence is a numeric array. |
---|
| 27 | |
---|
| 28 | Inputs: |
---|
| 29 | A: Sequence. If A is already a Numeric array it will be returned |
---|
| 30 | unaltered |
---|
| 31 | If not, an attempt is made to convert it to a Numeric |
---|
| 32 | array |
---|
| 33 | A: Scalar. Return 0-dimensional array of length 1, containing that value |
---|
| 34 | A: String. Array of ASCII values |
---|
| 35 | typecode: Numeric type. If specified, use this in the conversion. |
---|
| 36 | If not, let Numeric decide |
---|
| 37 | |
---|
| 38 | This function is necessary as array(A) can cause memory overflow. |
---|
| 39 | """ |
---|
| 40 | |
---|
| 41 | if typecode is None: |
---|
| 42 | if type(A) == Numeric.ArrayType: |
---|
| 43 | return A |
---|
| 44 | else: |
---|
| 45 | return Numeric.array(A) |
---|
| 46 | else: |
---|
| 47 | if type(A) == Numeric.ArrayType: |
---|
| 48 | if A.typecode() == typecode: |
---|
| 49 | return A |
---|
| 50 | else: |
---|
| 51 | return Numeric.array(A, typecode) |
---|
| 52 | else: |
---|
| 53 | return Numeric.array(A, typecode) |
---|
| 54 | |
---|
| 55 | |
---|
| 56 | # numpy version |
---|
| 57 | def numpy_ensure_numeric(A, typecode=None): |
---|
| 58 | """Ensure that sequence is a numeric array. |
---|
| 59 | |
---|
| 60 | Inputs: |
---|
| 61 | A: Sequence. If A is already a numeric array it will be returned |
---|
| 62 | unaltered |
---|
| 63 | If not, an attempt is made to convert it to a numeric |
---|
| 64 | array |
---|
| 65 | A: Scalar. Return 0-dimensional array containing that value. Note |
---|
| 66 | that a 0-dim array DOES NOT HAVE A LENGTH UNDER numpy. |
---|
| 67 | A: String. Array of ASCII values (numpy can't handle this) |
---|
| 68 | |
---|
| 69 | typecode: numeric type. If specified, use this in the conversion. |
---|
| 70 | If not, let numeric package decide. |
---|
| 71 | typecode will always be one of num.float, num.int, etc. |
---|
| 72 | |
---|
| 73 | Note that num.array(A, dtype) will sometimes copy. Use 'copy=False' to |
---|
| 74 | copy only when required. |
---|
| 75 | |
---|
| 76 | This function is necessary as array(A) can cause memory overflow. |
---|
| 77 | """ |
---|
| 78 | |
---|
| 79 | if typecode is None: |
---|
| 80 | if isinstance(A, numpy.ndarray): |
---|
| 81 | return A |
---|
| 82 | else: |
---|
| 83 | return numpy.array(A) |
---|
| 84 | else: |
---|
| 85 | return numpy.array(A, dtype=typecode, copy=False) |
---|
| 86 | |
---|
| 87 | |
---|
[7243] | 88 | def mem_usage(): |
---|
| 89 | '''Get memory usage (virtual) in KiB.''' |
---|
| 90 | |
---|
| 91 | _scale = {'KB': 1024, 'MB': 1024*1024, 'GB': 1024*1024*1024, |
---|
| 92 | 'kB': 1024, 'mB': 1024*1024, 'gB': 1024*1024*1024} |
---|
| 93 | |
---|
| 94 | if sys.platform != 'win32': |
---|
| 95 | _proc_status = '/proc/%d/status' % os.getpid() |
---|
| 96 | |
---|
| 97 | def _VmB(VmKey): |
---|
| 98 | '''Get number of virtual bytes used.''' |
---|
| 99 | |
---|
| 100 | # get pseudo file /proc/<pid>/status |
---|
| 101 | try: |
---|
| 102 | t = open(_proc_status) |
---|
| 103 | v = t.read() |
---|
| 104 | t.close() |
---|
| 105 | except IOError: |
---|
| 106 | return 0.0 |
---|
| 107 | |
---|
| 108 | # get VmKey line, eg: 'VmRSS: 999 kB\n ... |
---|
| 109 | i = v.index(VmKey) |
---|
| 110 | v = v[i:].split(None, 3) |
---|
| 111 | if len(v) < 3: |
---|
| 112 | return 0.0 |
---|
| 113 | |
---|
| 114 | # convert Vm value to bytes |
---|
| 115 | return float(v[1]) * _scale[v[2]] |
---|
| 116 | |
---|
| 117 | return int(_VmB('VmSize:')/_scale['KB']) |
---|
| 118 | else: |
---|
| 119 | # Windows code from: http://code.activestate.com/recipes/511491/ |
---|
| 120 | try: |
---|
| 121 | import ctypes |
---|
| 122 | import _winreg |
---|
| 123 | except: |
---|
| 124 | log(level, 'Windows resource usage not available') |
---|
| 125 | return |
---|
| 126 | |
---|
| 127 | kernel32 = ctypes.windll.kernel32 |
---|
| 128 | c_ulong = ctypes.c_ulong |
---|
| 129 | c_ulonglong = ctypes.c_ulonglong |
---|
| 130 | class MEMORYSTATUSEX(ctypes.Structure): |
---|
| 131 | _fields_ = [('dwLength', c_ulong), |
---|
| 132 | ('dwMemoryLoad', c_ulong), |
---|
| 133 | ('ullTotalPhys', c_ulonglong), |
---|
| 134 | ('ullAvailPhys', c_ulonglong), |
---|
| 135 | ('ullTotalPageFile', c_ulonglong), |
---|
| 136 | ('ullAvailPageFile', c_ulonglong), |
---|
| 137 | ('ullTotalVirtual', c_ulonglong), |
---|
| 138 | ('ullAvailVirtual', c_ulonglong), |
---|
| 139 | ('ullAvailExtendedVirtual', c_ulonglong) |
---|
| 140 | ] |
---|
| 141 | |
---|
| 142 | memoryStatusEx = MEMORYSTATUSEX() |
---|
| 143 | memoryStatusEx.dwLength = ctypes.sizeof(MEMORYSTATUSEX) |
---|
| 144 | kernel32.GlobalMemoryStatusEx(ctypes.byref(memoryStatusEx)) |
---|
| 145 | |
---|
| 146 | return int(memoryStatusEx.ullTotalPhys/_scale['KB']) |
---|
| 147 | |
---|
| 148 | |
---|
[7278] | 149 | class Array(object): |
---|
| 150 | def __init__(self, module, f): |
---|
| 151 | self.array = module.ones(ARRAY_SIZE, f) |
---|
| 152 | for i in xrange(ARRAY_SIZE[0]): |
---|
| 153 | for j in xrange(ARRAY_SIZE[1]): |
---|
| 154 | self.array[i, j] *= float(i)+j |
---|
| 155 | |
---|
[7245] | 156 | def test_usage(module, f, en, ls): |
---|
| 157 | start_mem = mem_usage() |
---|
[7243] | 158 | |
---|
[7278] | 159 | A = Array(module, f) |
---|
| 160 | B = Array(module, f) |
---|
| 161 | C = Array(module, f) |
---|
| 162 | D = module.ones(ARRAY_SIZE[1], f) |
---|
[7243] | 163 | |
---|
[7278] | 164 | start_time = time.time() |
---|
| 165 | |
---|
[7245] | 166 | # do some numeric calculations |
---|
| 167 | for i in xrange(ls): |
---|
[7278] | 168 | for j in xrange(ARRAY_SIZE[0]): |
---|
| 169 | for k in xrange(ARRAY_SIZE[1]): |
---|
| 170 | D[0] = A.array[j, k] |
---|
| 171 | D[1] = B.array[j, k] |
---|
| 172 | D[2] = C.array[j, k] |
---|
[7243] | 173 | |
---|
[7245] | 174 | stop_mem = mem_usage() |
---|
| 175 | stop_time = time.time() |
---|
[7243] | 176 | |
---|
[7245] | 177 | delta_time = stop_time - start_time |
---|
| 178 | delta_mem = stop_mem - start_mem |
---|
[7243] | 179 | |
---|
[7278] | 180 | del A, B, C, D |
---|
[7245] | 181 | gc.collect() |
---|
[7243] | 182 | |
---|
[7245] | 183 | return (delta_time, delta_mem) |
---|
[7243] | 184 | |
---|
[7278] | 185 | #for loop_size in (10, 100, 1000): |
---|
| 186 | for loop_size in (10, 100): |
---|
[7245] | 187 | # Do Numeric work |
---|
| 188 | (t, m) = test_usage(Numeric, Numeric.Float, Numeric_ensure_numeric, |
---|
| 189 | loop_size) |
---|
| 190 | print('Numeric %4d loops: %5.1f s, %d KiB' % (loop_size, t, m)) |
---|
[7243] | 191 | |
---|
[7278] | 192 | # Do numpy work |
---|
| 193 | (t, m) = test_usage(numpy, numpy.float, numpy_ensure_numeric, loop_size) |
---|
| 194 | print(' numpy %4d loops: %5.1f s, %d KiB' % (loop_size, t, m)) |
---|
| 195 | |
---|