1 | #!/usr/bin/env python |
---|
2 | |
---|
3 | ''' |
---|
4 | A program to test differences in time and memory usage between Numeric |
---|
5 | and numpy code. |
---|
6 | |
---|
7 | Due to the above, this will run only on a machine that has both of the |
---|
8 | above packages installed. |
---|
9 | ''' |
---|
10 | |
---|
11 | |
---|
12 | import sys |
---|
13 | import os |
---|
14 | import time |
---|
15 | import gc |
---|
16 | import Numeric |
---|
17 | import numpy |
---|
18 | |
---|
19 | |
---|
20 | ARRAY_SIZE = 1000 * 1000 |
---|
21 | |
---|
22 | |
---|
23 | # Numeric version |
---|
24 | def Numeric_ensure_numeric(A, typecode=None): |
---|
25 | """Ensure that sequence is a numeric array. |
---|
26 | |
---|
27 | Inputs: |
---|
28 | A: Sequence. If A is already a Numeric array it will be returned |
---|
29 | unaltered |
---|
30 | If not, an attempt is made to convert it to a Numeric |
---|
31 | array |
---|
32 | A: Scalar. Return 0-dimensional array of length 1, containing that value |
---|
33 | A: String. Array of ASCII values |
---|
34 | typecode: Numeric type. If specified, use this in the conversion. |
---|
35 | If not, let Numeric decide |
---|
36 | |
---|
37 | This function is necessary as array(A) can cause memory overflow. |
---|
38 | """ |
---|
39 | |
---|
40 | if typecode is None: |
---|
41 | if type(A) == Numeric.ArrayType: |
---|
42 | return A |
---|
43 | else: |
---|
44 | return Numeric.array(A) |
---|
45 | else: |
---|
46 | if type(A) == Numeric.ArrayType: |
---|
47 | if A.typecode() == typecode: |
---|
48 | return A |
---|
49 | else: |
---|
50 | return Numeric.array(A, typecode) |
---|
51 | else: |
---|
52 | return Numeric.array(A, typecode) |
---|
53 | |
---|
54 | |
---|
55 | # numpy version |
---|
56 | def numpy_ensure_numeric(A, typecode=None): |
---|
57 | """Ensure that sequence is a numeric array. |
---|
58 | |
---|
59 | Inputs: |
---|
60 | A: Sequence. If A is already a numeric array it will be returned |
---|
61 | unaltered |
---|
62 | If not, an attempt is made to convert it to a numeric |
---|
63 | array |
---|
64 | A: Scalar. Return 0-dimensional array containing that value. Note |
---|
65 | that a 0-dim array DOES NOT HAVE A LENGTH UNDER numpy. |
---|
66 | A: String. Array of ASCII values (numpy can't handle this) |
---|
67 | |
---|
68 | typecode: numeric type. If specified, use this in the conversion. |
---|
69 | If not, let numeric package decide. |
---|
70 | typecode will always be one of num.float, num.int, etc. |
---|
71 | |
---|
72 | Note that num.array(A, dtype) will sometimes copy. Use 'copy=False' to |
---|
73 | copy only when required. |
---|
74 | |
---|
75 | This function is necessary as array(A) can cause memory overflow. |
---|
76 | """ |
---|
77 | |
---|
78 | # if isinstance(A, basestring): |
---|
79 | # msg = 'Sorry, cannot handle strings in ensure_numeric()' |
---|
80 | # raise Exception, msg |
---|
81 | |
---|
82 | if typecode is None: |
---|
83 | if isinstance(A, numpy.ndarray): |
---|
84 | return A |
---|
85 | else: |
---|
86 | return numpy.array(A) |
---|
87 | else: |
---|
88 | return numpy.array(A, dtype=typecode, copy=False) |
---|
89 | |
---|
90 | |
---|
91 | def mem_usage(): |
---|
92 | '''Get memory usage (virtual) in KiB.''' |
---|
93 | |
---|
94 | _scale = {'KB': 1024, 'MB': 1024*1024, 'GB': 1024*1024*1024, |
---|
95 | 'kB': 1024, 'mB': 1024*1024, 'gB': 1024*1024*1024} |
---|
96 | |
---|
97 | if sys.platform != 'win32': |
---|
98 | _proc_status = '/proc/%d/status' % os.getpid() |
---|
99 | |
---|
100 | def _VmB(VmKey): |
---|
101 | '''Get number of virtual bytes used.''' |
---|
102 | |
---|
103 | # get pseudo file /proc/<pid>/status |
---|
104 | try: |
---|
105 | t = open(_proc_status) |
---|
106 | v = t.read() |
---|
107 | t.close() |
---|
108 | except IOError: |
---|
109 | return 0.0 |
---|
110 | |
---|
111 | # get VmKey line, eg: 'VmRSS: 999 kB\n ... |
---|
112 | i = v.index(VmKey) |
---|
113 | v = v[i:].split(None, 3) |
---|
114 | if len(v) < 3: |
---|
115 | return 0.0 |
---|
116 | |
---|
117 | # convert Vm value to bytes |
---|
118 | return float(v[1]) * _scale[v[2]] |
---|
119 | |
---|
120 | return int(_VmB('VmSize:')/_scale['KB']) |
---|
121 | else: |
---|
122 | # Windows code from: http://code.activestate.com/recipes/511491/ |
---|
123 | try: |
---|
124 | import ctypes |
---|
125 | import _winreg |
---|
126 | except: |
---|
127 | log(level, 'Windows resource usage not available') |
---|
128 | return |
---|
129 | |
---|
130 | kernel32 = ctypes.windll.kernel32 |
---|
131 | c_ulong = ctypes.c_ulong |
---|
132 | c_ulonglong = ctypes.c_ulonglong |
---|
133 | class MEMORYSTATUSEX(ctypes.Structure): |
---|
134 | _fields_ = [('dwLength', c_ulong), |
---|
135 | ('dwMemoryLoad', c_ulong), |
---|
136 | ('ullTotalPhys', c_ulonglong), |
---|
137 | ('ullAvailPhys', c_ulonglong), |
---|
138 | ('ullTotalPageFile', c_ulonglong), |
---|
139 | ('ullAvailPageFile', c_ulonglong), |
---|
140 | ('ullTotalVirtual', c_ulonglong), |
---|
141 | ('ullAvailVirtual', c_ulonglong), |
---|
142 | ('ullAvailExtendedVirtual', c_ulonglong) |
---|
143 | ] |
---|
144 | |
---|
145 | memoryStatusEx = MEMORYSTATUSEX() |
---|
146 | memoryStatusEx.dwLength = ctypes.sizeof(MEMORYSTATUSEX) |
---|
147 | kernel32.GlobalMemoryStatusEx(ctypes.byref(memoryStatusEx)) |
---|
148 | |
---|
149 | return int(memoryStatusEx.ullTotalPhys/_scale['KB']) |
---|
150 | |
---|
151 | |
---|
152 | def test_usage(module, f, en, ls): |
---|
153 | start_time = time.time() |
---|
154 | start_mem = mem_usage() |
---|
155 | |
---|
156 | A = module.ones(ARRAY_SIZE, f) |
---|
157 | B = module.ones(ARRAY_SIZE, f) |
---|
158 | for i in xrange(ls): |
---|
159 | A[i] *= float(i) |
---|
160 | B[i] *= float(i) |
---|
161 | |
---|
162 | # do some numeric calculations |
---|
163 | A = en(A) |
---|
164 | B = en(B) |
---|
165 | C = module.ones(ARRAY_SIZE, f) |
---|
166 | C = en(C) |
---|
167 | for i in xrange(ls): |
---|
168 | C = en(2.6*A + B + C + i) |
---|
169 | |
---|
170 | stop_mem = mem_usage() |
---|
171 | stop_time = time.time() |
---|
172 | |
---|
173 | delta_time = stop_time - start_time |
---|
174 | delta_mem = stop_mem - start_mem |
---|
175 | |
---|
176 | del A, B, C |
---|
177 | gc.collect() |
---|
178 | |
---|
179 | return (delta_time, delta_mem) |
---|
180 | |
---|
181 | for loop_size in (10, 100, 1000): |
---|
182 | # Do numpy work |
---|
183 | (t, m) = test_usage(numpy, numpy.float, numpy_ensure_numeric, loop_size) |
---|
184 | print(' numpy %4d loops: %5.1f s, %d KiB' % (loop_size, t, m)) |
---|
185 | |
---|
186 | # Do Numeric work |
---|
187 | (t, m) = test_usage(Numeric, Numeric.Float, Numeric_ensure_numeric, |
---|
188 | loop_size) |
---|
189 | print('Numeric %4d loops: %5.1f s, %d KiB' % (loop_size, t, m)) |
---|
190 | |
---|