Skip to content

Commit 6b719ba

Browse files
nouiznouiz
authored
Merge pull request lisa-lab#174 from slefrancois/speed_report
Add JUnit writer for speed tests, remove hardcoded reference times
2 parents 99c7aef + f724c2c commit 6b719ba

File tree

1 file changed

+27
-85
lines changed

1 file changed

+27
-85
lines changed

‎code/test.py‎

Lines changed: 27 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -98,43 +98,7 @@ def speed():
9898
do_gpu = True
9999

100100
algo_executed = [s for idx, s in enumerate(algo) if to_exec[idx]]
101-
#Timming expected are from the buildbot that have an i7-920 @
102-
# 2.67GHz with hyperthread enabled for the cpu, 12G of ram. An GeForce GTX
103-
# 580 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD
104-
# 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread.
105-
106-
expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4,
107-
346.9, 355.0, 268.2, 115.8, 16.8, 91.6])
108-
expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 63.1, 71,
109-
191.2, 199.0, 201.9, 107, 12.6, 61.3])
110-
111-
# Number with just 1 decimal are new value that are faster with
112-
# the Theano version 0.5rc2 Other number are older. They are not
113-
# updated, as we where faster in the past!
114-
# TODO: find why and fix this!
115-
116-
# Here is the value for the buildbot on February 3th 2012 with a GTX 285
117-
# sgd, cg mlp conv da
118-
# sda dbn rbm
119-
# gpu times[3.72957802, 9.94316864, 29.1772666, 9.13857198, 25.91144657,
120-
# 18.30802011, 53.38651466, 285.41386175]
121-
# expected [3.076634879, 7.555234910, 18.99226785, 9.58915591, 24.130070450,
122-
# 24.77524018, 92.66246653, 322.340329170]
123-
# sgd, cg mlp conv da
124-
# sda dbn rbm
125-
#expected/get [0.82492841, 0.75984178, 0.65092691, 1.04930573, 0.93125138
126-
# 1.35324519 1.7356905 1.12937868]
127-
128-
expected_times_gpu = numpy.asarray([2.9, 7.55523491, 18.99226785,
129-
5.8, 19.2,
130-
11.2, 7.3, 122, 112.5, 31.1, 8.3])
131-
expected_times_64 = [s for idx, s in enumerate(expected_times_64)
132-
if to_exec[idx]]
133-
expected_times_32 = [s for idx, s in enumerate(expected_times_32)
134-
if to_exec[idx]]
135-
expected_times_gpu = [s for idx, s in enumerate(expected_times_gpu)
136-
if to_exec[idx]]
137-
101+
138102
def time_test(m, l, idx, f, **kwargs):
139103
if not to_exec[idx]:
140104
return
@@ -188,27 +152,29 @@ def do_tests():
188152
saveto='')
189153
return numpy.asarray(l)
190154

155+
# Initialize test count and results dictionnary
156+
test_total = 0
157+
times_dic = {}
158+
191159
#test in float64 in FAST_RUN mode on the cpu
192160
import theano
193161
if do_float64:
194162
theano.config.floatX = 'float64'
195163
theano.config.mode = 'FAST_RUN'
196164
float64_times = do_tests()
165+
times_dic['float64'] = float64_times
166+
test_total += numpy.size(float64_times)
197167
print(algo_executed, file=sys.stderr)
198168
print('float64 times', float64_times, file=sys.stderr)
199-
print('float64 expected', expected_times_64, file=sys.stderr)
200-
print('float64 % expected/get', (
201-
expected_times_64 / float64_times), file=sys.stderr)
202169

203170
#test in float32 in FAST_RUN mode on the cpu
204171
theano.config.floatX = 'float32'
205172
if do_float32:
206173
float32_times = do_tests()
174+
times_dic['float32'] = float32_times
175+
test_total += numpy.size(float32_times)
207176
print(algo_executed, file=sys.stderr)
208177
print('float32 times', float32_times, file=sys.stderr)
209-
print('float32 expected', expected_times_32, file=sys.stderr)
210-
print('float32 % expected/get', (
211-
expected_times_32 / float32_times), file=sys.stderr)
212178

213179
if do_float64:
214180
print('float64/float32', (
@@ -218,29 +184,20 @@ def do_tests():
218184
'in one place'), file=sys.stderr)
219185
print(algo_executed, file=sys.stderr)
220186
print('float64 times', float64_times, file=sys.stderr)
221-
print('float64 expected', expected_times_64, file=sys.stderr)
222-
print('float64 % expected/get', (
223-
expected_times_64 / float64_times), file=sys.stderr)
224187
print('float32 times', float32_times, file=sys.stderr)
225-
print('float32 expected', expected_times_32, file=sys.stderr)
226-
print('float32 % expected/get', (
227-
expected_times_32 / float32_times), file=sys.stderr)
228188

229189
print('float64/float32', (
230190
float64_times / float32_times), file=sys.stderr)
231-
print('expected float64/float32', (
232-
expected_times_64 / float32_times), file=sys.stderr)
233191

234192
#test in float32 in FAST_RUN mode on the gpu
235193
import theano.sandbox.cuda
236194
if do_gpu:
237195
theano.sandbox.cuda.use('gpu')
238196
gpu_times = do_tests()
197+
times_dic['gpu'] = gpu_times
198+
test_total += numpy.size(gpu_times)
239199
print(algo_executed, file=sys.stderr)
240200
print('gpu times', gpu_times, file=sys.stderr)
241-
print('gpu expected', expected_times_gpu, file=sys.stderr)
242-
print('gpu % expected/get', (
243-
expected_times_gpu / gpu_times), file=sys.stderr)
244201

245202
if do_float64:
246203
print('float64/gpu', float64_times / gpu_times, file=sys.stderr)
@@ -252,50 +209,35 @@ def do_tests():
252209
print(algo_executed, file=sys.stderr)
253210
if do_float64:
254211
print('float64 times', float64_times, file=sys.stderr)
255-
print('float64 expected', expected_times_64, file=sys.stderr)
256-
print('float64 % expected/get', (
257-
expected_times_64 / float64_times), file=sys.stderr)
258212
if do_float32:
259213
print('float32 times', float32_times, file=sys.stderr)
260-
print('float32 expected', expected_times_32, file=sys.stderr)
261-
print('float32 % expected/get', (
262-
expected_times_32 / float32_times), file=sys.stderr)
263214
if do_gpu:
264215
print('gpu times', gpu_times, file=sys.stderr)
265-
print('gpu expected', expected_times_gpu, file=sys.stderr)
266-
print('gpu % expected/get', (
267-
expected_times_gpu / gpu_times), file=sys.stderr)
268216

269217
print()
270218
if do_float64 and do_float32:
271219
print('float64/float32', (
272220
float64_times / float32_times), file=sys.stderr)
273-
print('expected float64/float32', (
274-
expected_times_64 / float32_times), file=sys.stderr)
275221
if do_float64 and do_gpu:
276222
print('float64/gpu', float64_times / gpu_times, file=sys.stderr)
277-
print('expected float64/gpu', (
278-
expected_times_64 / gpu_times), file=sys.stderr)
279223
if do_float32 and do_gpu:
280224
print('float32/gpu', float32_times / gpu_times, file=sys.stderr)
281-
print('expected float32/gpu', (
282-
expected_times_32 / gpu_times), file=sys.stderr)
283-
284-
def compare(x, y):
285-
ratio = x / y
286-
# If there is more then 5% difference between the expected
287-
# time and the real time, we consider this an error.
288-
return sum((ratio < 0.95) + (ratio > 1.05))
225+
226+
# Generate JUnit performance report
227+
# Define speedtest file write method
228+
def write_junit(f, algos, times, label):
229+
for algo, time in zip(algos, times):
230+
f.write(' <testcase classname="{label}" name="{algo}" time="{time}">'
231+
.format(label=label, algo=algo, time=time))
232+
f.write(' </testcase>\n')
233+
234+
with open('speedtests_time.xml', 'w') as f:
235+
f.write('<?xml version="1.0" encoding="UTF-8"?>\n')
236+
f.write('<testsuite name="dlt_speedtests" tests="{ntests}">\n'
237+
.format(ntests=test_total))
238+
for label, times in times_dic.items():
239+
write_junit(f, algo_executed, times, label)
240+
f.write('</testsuite>\n')
289241

290-
print(file=sys.stderr)
291-
if do_float64:
292-
err = compare(expected_times_64, float64_times)
293-
print('speed_failure_float64=' + str(err), file=sys.stderr)
294-
if do_float32:
295-
err = compare(expected_times_32, float32_times)
296-
print('speed_failure_float32=' + str(err), file=sys.stderr)
297242
if do_gpu:
298-
err = compare(expected_times_gpu, gpu_times)
299-
print('speed_failure_gpu=' + str(err), file=sys.stderr)
300-
301243
assert not numpy.isnan(gpu_times).any()

0 commit comments

Comments
 (0)