Skip to content

Add JUnit writer for speed tests, remove hardcoded reference times #174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 11, 2016
Merged

Add JUnit writer for speed tests, remove hardcoded reference times #174

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
82c85e2
Add JUnit writer for speed tests, remove hardcoded reference times
slefrancois Oct 6, 2016
f14107d
single performance file open, correct only access times variables if …
slefrancois Oct 7, 2016
f724c2c
move assert gpu_times not nan
slefrancois Oct 7, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 27 additions & 85 deletions code/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,43 +98,7 @@ def speed():
do_gpu = True

algo_executed = [s for idx, s in enumerate(algo) if to_exec[idx]]
#Timming expected are from the buildbot that have an i7-920 @
# 2.67GHz with hyperthread enabled for the cpu, 12G of ram. An GeForce GTX
# 580 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD
# 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread.

expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4,
346.9, 355.0, 268.2, 115.8, 16.8, 91.6])
expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 63.1, 71,
191.2, 199.0, 201.9, 107, 12.6, 61.3])

# Number with just 1 decimal are new value that are faster with
# the Theano version 0.5rc2 Other number are older. They are not
# updated, as we where faster in the past!
# TODO: find why and fix this!

# Here is the value for the buildbot on February 3th 2012 with a GTX 285
# sgd, cg mlp conv da
# sda dbn rbm
# gpu times[3.72957802, 9.94316864, 29.1772666, 9.13857198, 25.91144657,
# 18.30802011, 53.38651466, 285.41386175]
# expected [3.076634879, 7.555234910, 18.99226785, 9.58915591, 24.130070450,
# 24.77524018, 92.66246653, 322.340329170]
# sgd, cg mlp conv da
# sda dbn rbm
#expected/get [0.82492841, 0.75984178, 0.65092691, 1.04930573, 0.93125138
# 1.35324519 1.7356905 1.12937868]

expected_times_gpu = numpy.asarray([2.9, 7.55523491, 18.99226785,
5.8, 19.2,
11.2, 7.3, 122, 112.5, 31.1, 8.3])
expected_times_64 = [s for idx, s in enumerate(expected_times_64)
if to_exec[idx]]
expected_times_32 = [s for idx, s in enumerate(expected_times_32)
if to_exec[idx]]
expected_times_gpu = [s for idx, s in enumerate(expected_times_gpu)
if to_exec[idx]]


def time_test(m, l, idx, f, **kwargs):
if not to_exec[idx]:
return
Expand Down Expand Up @@ -188,27 +152,29 @@ def do_tests():
saveto='')
return numpy.asarray(l)

# Initialize test count and results dictionnary
test_total = 0
times_dic = {}

#test in float64 in FAST_RUN mode on the cpu
import theano
if do_float64:
theano.config.floatX = 'float64'
theano.config.mode = 'FAST_RUN'
float64_times = do_tests()
times_dic['float64'] = float64_times
test_total += numpy.size(float64_times)
print(algo_executed, file=sys.stderr)
print('float64 times', float64_times, file=sys.stderr)
print('float64 expected', expected_times_64, file=sys.stderr)
print('float64 % expected/get', (
expected_times_64 / float64_times), file=sys.stderr)

#test in float32 in FAST_RUN mode on the cpu
theano.config.floatX = 'float32'
if do_float32:
float32_times = do_tests()
times_dic['float32'] = float32_times
test_total += numpy.size(float32_times)
print(algo_executed, file=sys.stderr)
print('float32 times', float32_times, file=sys.stderr)
print('float32 expected', expected_times_32, file=sys.stderr)
print('float32 % expected/get', (
expected_times_32 / float32_times), file=sys.stderr)

if do_float64:
print('float64/float32', (
Expand All @@ -218,29 +184,20 @@ def do_tests():
'in one place'), file=sys.stderr)
print(algo_executed, file=sys.stderr)
print('float64 times', float64_times, file=sys.stderr)
print('float64 expected', expected_times_64, file=sys.stderr)
print('float64 % expected/get', (
expected_times_64 / float64_times), file=sys.stderr)
print('float32 times', float32_times, file=sys.stderr)
print('float32 expected', expected_times_32, file=sys.stderr)
print('float32 % expected/get', (
expected_times_32 / float32_times), file=sys.stderr)

print('float64/float32', (
float64_times / float32_times), file=sys.stderr)
print('expected float64/float32', (
expected_times_64 / float32_times), file=sys.stderr)

#test in float32 in FAST_RUN mode on the gpu
import theano.sandbox.cuda
if do_gpu:
theano.sandbox.cuda.use('gpu')
gpu_times = do_tests()
times_dic['gpu'] = gpu_times
test_total += numpy.size(gpu_times)
print(algo_executed, file=sys.stderr)
print('gpu times', gpu_times, file=sys.stderr)
print('gpu expected', expected_times_gpu, file=sys.stderr)
print('gpu % expected/get', (
expected_times_gpu / gpu_times), file=sys.stderr)

if do_float64:
print('float64/gpu', float64_times / gpu_times, file=sys.stderr)
Expand All @@ -252,50 +209,35 @@ def do_tests():
print(algo_executed, file=sys.stderr)
if do_float64:
print('float64 times', float64_times, file=sys.stderr)
print('float64 expected', expected_times_64, file=sys.stderr)
print('float64 % expected/get', (
expected_times_64 / float64_times), file=sys.stderr)
if do_float32:
print('float32 times', float32_times, file=sys.stderr)
print('float32 expected', expected_times_32, file=sys.stderr)
print('float32 % expected/get', (
expected_times_32 / float32_times), file=sys.stderr)
if do_gpu:
print('gpu times', gpu_times, file=sys.stderr)
print('gpu expected', expected_times_gpu, file=sys.stderr)
print('gpu % expected/get', (
expected_times_gpu / gpu_times), file=sys.stderr)

print()
if do_float64 and do_float32:
print('float64/float32', (
float64_times / float32_times), file=sys.stderr)
print('expected float64/float32', (
expected_times_64 / float32_times), file=sys.stderr)
if do_float64 and do_gpu:
print('float64/gpu', float64_times / gpu_times, file=sys.stderr)
print('expected float64/gpu', (
expected_times_64 / gpu_times), file=sys.stderr)
if do_float32 and do_gpu:
print('float32/gpu', float32_times / gpu_times, file=sys.stderr)
print('expected float32/gpu', (
expected_times_32 / gpu_times), file=sys.stderr)

def compare(x, y):
ratio = x / y
# If there is more then 5% difference between the expected
# time and the real time, we consider this an error.
return sum((ratio < 0.95) + (ratio > 1.05))

# Generate JUnit performance report
# Define speedtest file write method
def write_junit(f, algos, times, label):
for algo, time in zip(algos, times):
f.write(' <testcase classname="{label}" name="{algo}" time="{time}">'
.format(label=label, algo=algo, time=time))
f.write(' </testcase>\n')

with open('speedtests_time.xml', 'w') as f:
f.write('<?xml version="1.0" encoding="UTF-8"?>\n')
f.write('<testsuite name="dlt_speedtests" tests="{ntests}">\n'
.format(ntests=test_total))
for label, times in times_dic.items():
write_junit(f, algo_executed, times, label)
f.write('</testsuite>\n')

print(file=sys.stderr)
if do_float64:
err = compare(expected_times_64, float64_times)
print('speed_failure_float64=' + str(err), file=sys.stderr)
if do_float32:
err = compare(expected_times_32, float32_times)
print('speed_failure_float32=' + str(err), file=sys.stderr)
if do_gpu:
err = compare(expected_times_gpu, gpu_times)
print('speed_failure_gpu=' + str(err), file=sys.stderr)

assert not numpy.isnan(gpu_times).any()