diff --git a/code/test.py b/code/test.py
index 4332e8b0..b3077b7c 100644
--- a/code/test.py
+++ b/code/test.py
@@ -98,43 +98,7 @@ def speed():
do_gpu = True
algo_executed = [s for idx, s in enumerate(algo) if to_exec[idx]]
- #Timming expected are from the buildbot that have an i7-920 @
- # 2.67GHz with hyperthread enabled for the cpu, 12G of ram. An GeForce GTX
- # 580 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD
- # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread.
-
- expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4,
- 346.9, 355.0, 268.2, 115.8, 16.8, 91.6])
- expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 63.1, 71,
- 191.2, 199.0, 201.9, 107, 12.6, 61.3])
-
- # Number with just 1 decimal are new value that are faster with
- # the Theano version 0.5rc2 Other number are older. They are not
- # updated, as we where faster in the past!
- # TODO: find why and fix this!
-
-# Here is the value for the buildbot on February 3th 2012 with a GTX 285
-# sgd, cg mlp conv da
-# sda dbn rbm
-# gpu times[3.72957802, 9.94316864, 29.1772666, 9.13857198, 25.91144657,
-# 18.30802011, 53.38651466, 285.41386175]
-# expected [3.076634879, 7.555234910, 18.99226785, 9.58915591, 24.130070450,
-# 24.77524018, 92.66246653, 322.340329170]
-# sgd, cg mlp conv da
-# sda dbn rbm
-#expected/get [0.82492841, 0.75984178, 0.65092691, 1.04930573, 0.93125138
-# 1.35324519 1.7356905 1.12937868]
-
- expected_times_gpu = numpy.asarray([2.9, 7.55523491, 18.99226785,
- 5.8, 19.2,
- 11.2, 7.3, 122, 112.5, 31.1, 8.3])
- expected_times_64 = [s for idx, s in enumerate(expected_times_64)
- if to_exec[idx]]
- expected_times_32 = [s for idx, s in enumerate(expected_times_32)
- if to_exec[idx]]
- expected_times_gpu = [s for idx, s in enumerate(expected_times_gpu)
- if to_exec[idx]]
-
+
def time_test(m, l, idx, f, **kwargs):
if not to_exec[idx]:
return
@@ -188,27 +152,29 @@ def do_tests():
saveto='')
return numpy.asarray(l)
+ # Initialize test count and results dictionnary
+ test_total = 0
+ times_dic = {}
+
#test in float64 in FAST_RUN mode on the cpu
import theano
if do_float64:
theano.config.floatX = 'float64'
theano.config.mode = 'FAST_RUN'
float64_times = do_tests()
+ times_dic['float64'] = float64_times
+ test_total += numpy.size(float64_times)
print(algo_executed, file=sys.stderr)
print('float64 times', float64_times, file=sys.stderr)
- print('float64 expected', expected_times_64, file=sys.stderr)
- print('float64 % expected/get', (
- expected_times_64 / float64_times), file=sys.stderr)
#test in float32 in FAST_RUN mode on the cpu
theano.config.floatX = 'float32'
if do_float32:
float32_times = do_tests()
+ times_dic['float32'] = float32_times
+ test_total += numpy.size(float32_times)
print(algo_executed, file=sys.stderr)
print('float32 times', float32_times, file=sys.stderr)
- print('float32 expected', expected_times_32, file=sys.stderr)
- print('float32 % expected/get', (
- expected_times_32 / float32_times), file=sys.stderr)
if do_float64:
print('float64/float32', (
@@ -218,29 +184,20 @@ def do_tests():
'in one place'), file=sys.stderr)
print(algo_executed, file=sys.stderr)
print('float64 times', float64_times, file=sys.stderr)
- print('float64 expected', expected_times_64, file=sys.stderr)
- print('float64 % expected/get', (
- expected_times_64 / float64_times), file=sys.stderr)
print('float32 times', float32_times, file=sys.stderr)
- print('float32 expected', expected_times_32, file=sys.stderr)
- print('float32 % expected/get', (
- expected_times_32 / float32_times), file=sys.stderr)
print('float64/float32', (
float64_times / float32_times), file=sys.stderr)
- print('expected float64/float32', (
- expected_times_64 / float32_times), file=sys.stderr)
#test in float32 in FAST_RUN mode on the gpu
import theano.sandbox.cuda
if do_gpu:
theano.sandbox.cuda.use('gpu')
gpu_times = do_tests()
+ times_dic['gpu'] = gpu_times
+ test_total += numpy.size(gpu_times)
print(algo_executed, file=sys.stderr)
print('gpu times', gpu_times, file=sys.stderr)
- print('gpu expected', expected_times_gpu, file=sys.stderr)
- print('gpu % expected/get', (
- expected_times_gpu / gpu_times), file=sys.stderr)
if do_float64:
print('float64/gpu', float64_times / gpu_times, file=sys.stderr)
@@ -252,50 +209,35 @@ def do_tests():
print(algo_executed, file=sys.stderr)
if do_float64:
print('float64 times', float64_times, file=sys.stderr)
- print('float64 expected', expected_times_64, file=sys.stderr)
- print('float64 % expected/get', (
- expected_times_64 / float64_times), file=sys.stderr)
if do_float32:
print('float32 times', float32_times, file=sys.stderr)
- print('float32 expected', expected_times_32, file=sys.stderr)
- print('float32 % expected/get', (
- expected_times_32 / float32_times), file=sys.stderr)
if do_gpu:
print('gpu times', gpu_times, file=sys.stderr)
- print('gpu expected', expected_times_gpu, file=sys.stderr)
- print('gpu % expected/get', (
- expected_times_gpu / gpu_times), file=sys.stderr)
print()
if do_float64 and do_float32:
print('float64/float32', (
float64_times / float32_times), file=sys.stderr)
- print('expected float64/float32', (
- expected_times_64 / float32_times), file=sys.stderr)
if do_float64 and do_gpu:
print('float64/gpu', float64_times / gpu_times, file=sys.stderr)
- print('expected float64/gpu', (
- expected_times_64 / gpu_times), file=sys.stderr)
if do_float32 and do_gpu:
print('float32/gpu', float32_times / gpu_times, file=sys.stderr)
- print('expected float32/gpu', (
- expected_times_32 / gpu_times), file=sys.stderr)
-
- def compare(x, y):
- ratio = x / y
- # If there is more then 5% difference between the expected
- # time and the real time, we consider this an error.
- return sum((ratio < 0.95) + (ratio > 1.05))
+
+ # Generate JUnit performance report
+ # Define speedtest file write method
+ def write_junit(f, algos, times, label):
+ for algo, time in zip(algos, times):
+ f.write(' '
+ .format(label=label, algo=algo, time=time))
+ f.write(' \n')
+
+ with open('speedtests_time.xml', 'w') as f:
+ f.write('\n')
+ f.write('\n'
+ .format(ntests=test_total))
+ for label, times in times_dic.items():
+ write_junit(f, algo_executed, times, label)
+ f.write('\n')
- print(file=sys.stderr)
- if do_float64:
- err = compare(expected_times_64, float64_times)
- print('speed_failure_float64=' + str(err), file=sys.stderr)
- if do_float32:
- err = compare(expected_times_32, float32_times)
- print('speed_failure_float32=' + str(err), file=sys.stderr)
if do_gpu:
- err = compare(expected_times_gpu, gpu_times)
- print('speed_failure_gpu=' + str(err), file=sys.stderr)
-
assert not numpy.isnan(gpu_times).any()