1 | #!/usr/bin/env python |
---|
2 | |
---|
3 | ''' |
---|
4 | A program to test differences in time and memory usage between Numeric |
---|
5 | and numpy code. |
---|
6 | |
---|
7 | Due to the above, this will run only on a machine that has both of the |
---|
8 | above packages installed. |
---|
9 | ''' |
---|
10 | |
---|
11 | |
---|
12 | import sys |
---|
13 | import os |
---|
14 | import time |
---|
15 | import gc |
---|
16 | import Numeric |
---|
17 | import numpy |
---|
18 | |
---|
19 | |
---|
20 | #ARRAY_SIZE = 1000 * 1000 |
---|
21 | ARRAY_SIZE = (40000, 3) |
---|
22 | |
---|
23 | |
---|
24 | # Numeric version |
---|
25 | def Numeric_ensure_numeric(A, typecode=None): |
---|
26 | """Ensure that sequence is a numeric array. |
---|
27 | |
---|
28 | Inputs: |
---|
29 | A: Sequence. If A is already a Numeric array it will be returned |
---|
30 | unaltered |
---|
31 | If not, an attempt is made to convert it to a Numeric |
---|
32 | array |
---|
33 | A: Scalar. Return 0-dimensional array of length 1, containing that value |
---|
34 | A: String. Array of ASCII values |
---|
35 | typecode: Numeric type. If specified, use this in the conversion. |
---|
36 | If not, let Numeric decide |
---|
37 | |
---|
38 | This function is necessary as array(A) can cause memory overflow. |
---|
39 | """ |
---|
40 | |
---|
41 | if typecode is None: |
---|
42 | if type(A) == Numeric.ArrayType: |
---|
43 | return A |
---|
44 | else: |
---|
45 | return Numeric.array(A) |
---|
46 | else: |
---|
47 | if type(A) == Numeric.ArrayType: |
---|
48 | if A.typecode() == typecode: |
---|
49 | return A |
---|
50 | else: |
---|
51 | return Numeric.array(A, typecode) |
---|
52 | else: |
---|
53 | return Numeric.array(A, typecode) |
---|
54 | |
---|
55 | |
---|
56 | # numpy version |
---|
57 | def numpy_ensure_numeric(A, typecode=None): |
---|
58 | """Ensure that sequence is a numeric array. |
---|
59 | |
---|
60 | Inputs: |
---|
61 | A: Sequence. If A is already a numeric array it will be returned |
---|
62 | unaltered |
---|
63 | If not, an attempt is made to convert it to a numeric |
---|
64 | array |
---|
65 | A: Scalar. Return 0-dimensional array containing that value. Note |
---|
66 | that a 0-dim array DOES NOT HAVE A LENGTH UNDER numpy. |
---|
67 | A: String. Array of ASCII values (numpy can't handle this) |
---|
68 | |
---|
69 | typecode: numeric type. If specified, use this in the conversion. |
---|
70 | If not, let numeric package decide. |
---|
71 | typecode will always be one of num.float, num.int, etc. |
---|
72 | |
---|
73 | Note that num.array(A, dtype) will sometimes copy. Use 'copy=False' to |
---|
74 | copy only when required. |
---|
75 | |
---|
76 | This function is necessary as array(A) can cause memory overflow. |
---|
77 | """ |
---|
78 | |
---|
79 | if typecode is None: |
---|
80 | if isinstance(A, numpy.ndarray): |
---|
81 | return A |
---|
82 | else: |
---|
83 | return numpy.array(A) |
---|
84 | else: |
---|
85 | return numpy.array(A, dtype=typecode, copy=False) |
---|
86 | |
---|
87 | |
---|
88 | def mem_usage(): |
---|
89 | '''Get memory usage (virtual) in KiB.''' |
---|
90 | |
---|
91 | _scale = {'KB': 1024, 'MB': 1024*1024, 'GB': 1024*1024*1024, |
---|
92 | 'kB': 1024, 'mB': 1024*1024, 'gB': 1024*1024*1024} |
---|
93 | |
---|
94 | if sys.platform != 'win32': |
---|
95 | _proc_status = '/proc/%d/status' % os.getpid() |
---|
96 | |
---|
97 | def _VmB(VmKey): |
---|
98 | '''Get number of virtual bytes used.''' |
---|
99 | |
---|
100 | # get pseudo file /proc/<pid>/status |
---|
101 | try: |
---|
102 | t = open(_proc_status) |
---|
103 | v = t.read() |
---|
104 | t.close() |
---|
105 | except IOError: |
---|
106 | return 0.0 |
---|
107 | |
---|
108 | # get VmKey line, eg: 'VmRSS: 999 kB\n ... |
---|
109 | i = v.index(VmKey) |
---|
110 | v = v[i:].split(None, 3) |
---|
111 | if len(v) < 3: |
---|
112 | return 0.0 |
---|
113 | |
---|
114 | # convert Vm value to bytes |
---|
115 | return float(v[1]) * _scale[v[2]] |
---|
116 | |
---|
117 | return int(_VmB('VmSize:')/_scale['KB']) |
---|
118 | else: |
---|
119 | # Windows code from: http://code.activestate.com/recipes/511491/ |
---|
120 | try: |
---|
121 | import ctypes |
---|
122 | import _winreg |
---|
123 | except: |
---|
124 | log(level, 'Windows resource usage not available') |
---|
125 | return |
---|
126 | |
---|
127 | kernel32 = ctypes.windll.kernel32 |
---|
128 | c_ulong = ctypes.c_ulong |
---|
129 | c_ulonglong = ctypes.c_ulonglong |
---|
130 | class MEMORYSTATUSEX(ctypes.Structure): |
---|
131 | _fields_ = [('dwLength', c_ulong), |
---|
132 | ('dwMemoryLoad', c_ulong), |
---|
133 | ('ullTotalPhys', c_ulonglong), |
---|
134 | ('ullAvailPhys', c_ulonglong), |
---|
135 | ('ullTotalPageFile', c_ulonglong), |
---|
136 | ('ullAvailPageFile', c_ulonglong), |
---|
137 | ('ullTotalVirtual', c_ulonglong), |
---|
138 | ('ullAvailVirtual', c_ulonglong), |
---|
139 | ('ullAvailExtendedVirtual', c_ulonglong) |
---|
140 | ] |
---|
141 | |
---|
142 | memoryStatusEx = MEMORYSTATUSEX() |
---|
143 | memoryStatusEx.dwLength = ctypes.sizeof(MEMORYSTATUSEX) |
---|
144 | kernel32.GlobalMemoryStatusEx(ctypes.byref(memoryStatusEx)) |
---|
145 | |
---|
146 | return int(memoryStatusEx.ullTotalPhys/_scale['KB']) |
---|
147 | |
---|
148 | |
---|
149 | class Array(object): |
---|
150 | def __init__(self, module, f): |
---|
151 | self.array = module.ones(ARRAY_SIZE, f) |
---|
152 | for i in xrange(ARRAY_SIZE[0]): |
---|
153 | for j in xrange(ARRAY_SIZE[1]): |
---|
154 | self.array[i, j] *= float(i)+j |
---|
155 | |
---|
156 | def test_usage(module, f, en, ls): |
---|
157 | start_mem = mem_usage() |
---|
158 | |
---|
159 | A = Array(module, f) |
---|
160 | B = Array(module, f) |
---|
161 | C = Array(module, f) |
---|
162 | D = module.ones(ARRAY_SIZE[1], f) |
---|
163 | |
---|
164 | start_time = time.time() |
---|
165 | |
---|
166 | # do some numeric calculations |
---|
167 | for i in xrange(ls): |
---|
168 | for j in xrange(ARRAY_SIZE[0]): |
---|
169 | for k in xrange(ARRAY_SIZE[1]): |
---|
170 | D[0] = A.array[j, k] |
---|
171 | D[1] = B.array[j, k] |
---|
172 | D[2] = C.array[j, k] |
---|
173 | |
---|
174 | stop_mem = mem_usage() |
---|
175 | stop_time = time.time() |
---|
176 | |
---|
177 | delta_time = stop_time - start_time |
---|
178 | delta_mem = stop_mem - start_mem |
---|
179 | |
---|
180 | del A, B, C, D |
---|
181 | gc.collect() |
---|
182 | |
---|
183 | return (delta_time, delta_mem) |
---|
184 | |
---|
185 | #for loop_size in (10, 100, 1000): |
---|
186 | for loop_size in (10, 100): |
---|
187 | # Do Numeric work |
---|
188 | (t, m) = test_usage(Numeric, Numeric.Float, Numeric_ensure_numeric, |
---|
189 | loop_size) |
---|
190 | print('Numeric %4d loops: %5.1f s, %d KiB' % (loop_size, t, m)) |
---|
191 | |
---|
192 | # Do numpy work |
---|
193 | (t, m) = test_usage(numpy, numpy.float, numpy_ensure_numeric, loop_size) |
---|
194 | print(' numpy %4d loops: %5.1f s, %d KiB' % (loop_size, t, m)) |
---|
195 | |
---|