Skip to content

Add JUnit writer for speed tests, remove hardcoded reference times #174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 11, 2016
Merged

Add JUnit writer for speed tests, remove hardcoded reference times #174

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
82c85e2
Add JUnit writer for speed tests, remove hardcoded reference times
slefrancois Oct 6, 2016
f14107d
single performance file open, correct only access times variables if …
slefrancois Oct 7, 2016
f724c2c
move assert gpu_times not nan
slefrancois Oct 7, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add JUnit writer for speed tests, remove hardcoded reference times
  • Loading branch information
@slefrancois
slefrancois committed Oct 6, 2016
commit 82c85e2a6f74a92736c2afdd805710dedfcc4f4f
117 changes: 29 additions & 88 deletions code/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,43 +98,7 @@ def speed():
do_gpu = True

algo_executed = [s for idx, s in enumerate(algo) if to_exec[idx]]
#Timming expected are from the buildbot that have an i7-920 @
# 2.67GHz with hyperthread enabled for the cpu, 12G of ram. An GeForce GTX
# 580 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD
# 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread.

expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4,
346.9, 355.0, 268.2, 115.8, 16.8, 91.6])
expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 63.1, 71,
191.2, 199.0, 201.9, 107, 12.6, 61.3])

# Number with just 1 decimal are new value that are faster with
# the Theano version 0.5rc2 Other number are older. They are not
# updated, as we where faster in the past!
# TODO: find why and fix this!

# Here is the value for the buildbot on February 3th 2012 with a GTX 285
# sgd, cg mlp conv da
# sda dbn rbm
# gpu times[3.72957802, 9.94316864, 29.1772666, 9.13857198, 25.91144657,
# 18.30802011, 53.38651466, 285.41386175]
# expected [3.076634879, 7.555234910, 18.99226785, 9.58915591, 24.130070450,
# 24.77524018, 92.66246653, 322.340329170]
# sgd, cg mlp conv da
# sda dbn rbm
#expected/get [0.82492841, 0.75984178, 0.65092691, 1.04930573, 0.93125138
# 1.35324519 1.7356905 1.12937868]

expected_times_gpu = numpy.asarray([2.9, 7.55523491, 18.99226785,
5.8, 19.2,
11.2, 7.3, 122, 112.5, 31.1, 8.3])
expected_times_64 = [s for idx, s in enumerate(expected_times_64)
if to_exec[idx]]
expected_times_32 = [s for idx, s in enumerate(expected_times_32)
if to_exec[idx]]
expected_times_gpu = [s for idx, s in enumerate(expected_times_gpu)
if to_exec[idx]]


def time_test(m, l, idx, f, **kwargs):
if not to_exec[idx]:
return
Expand Down Expand Up @@ -196,19 +160,13 @@ def do_tests():
float64_times = do_tests()
print(algo_executed, file=sys.stderr)
print('float64 times', float64_times, file=sys.stderr)
print('float64 expected', expected_times_64, file=sys.stderr)
print('float64 % expected/get', (
expected_times_64 / float64_times), file=sys.stderr)

#test in float32 in FAST_RUN mode on the cpu
theano.config.floatX = 'float32'
if do_float32:
float32_times = do_tests()
print(algo_executed, file=sys.stderr)
print('float32 times', float32_times, file=sys.stderr)
print('float32 expected', expected_times_32, file=sys.stderr)
print('float32 % expected/get', (
expected_times_32 / float32_times), file=sys.stderr)

if do_float64:
print('float64/float32', (
Expand All @@ -218,18 +176,10 @@ def do_tests():
'in one place'), file=sys.stderr)
print(algo_executed, file=sys.stderr)
print('float64 times', float64_times, file=sys.stderr)
print('float64 expected', expected_times_64, file=sys.stderr)
print('float64 % expected/get', (
expected_times_64 / float64_times), file=sys.stderr)
print('float32 times', float32_times, file=sys.stderr)
print('float32 expected', expected_times_32, file=sys.stderr)
print('float32 % expected/get', (
expected_times_32 / float32_times), file=sys.stderr)

print('float64/float32', (
float64_times / float32_times), file=sys.stderr)
print('expected float64/float32', (
expected_times_64 / float32_times), file=sys.stderr)

#test in float32 in FAST_RUN mode on the gpu
import theano.sandbox.cuda
Expand All @@ -238,9 +188,6 @@ def do_tests():
gpu_times = do_tests()
print(algo_executed, file=sys.stderr)
print('gpu times', gpu_times, file=sys.stderr)
print('gpu expected', expected_times_gpu, file=sys.stderr)
print('gpu % expected/get', (
expected_times_gpu / gpu_times), file=sys.stderr)

if do_float64:
print('float64/gpu', float64_times / gpu_times, file=sys.stderr)
Expand All @@ -252,50 +199,44 @@ def do_tests():
print(algo_executed, file=sys.stderr)
if do_float64:
print('float64 times', float64_times, file=sys.stderr)
print('float64 expected', expected_times_64, file=sys.stderr)
print('float64 % expected/get', (
expected_times_64 / float64_times), file=sys.stderr)
if do_float32:
print('float32 times', float32_times, file=sys.stderr)
print('float32 expected', expected_times_32, file=sys.stderr)
print('float32 % expected/get', (
expected_times_32 / float32_times), file=sys.stderr)
if do_gpu:
print('gpu times', gpu_times, file=sys.stderr)
print('gpu expected', expected_times_gpu, file=sys.stderr)
print('gpu % expected/get', (
expected_times_gpu / gpu_times), file=sys.stderr)

print()
if do_float64 and do_float32:
print('float64/float32', (
float64_times / float32_times), file=sys.stderr)
print('expected float64/float32', (
expected_times_64 / float32_times), file=sys.stderr)
if do_float64 and do_gpu:
print('float64/gpu', float64_times / gpu_times, file=sys.stderr)
print('expected float64/gpu', (
expected_times_64 / gpu_times), file=sys.stderr)
if do_float32 and do_gpu:
print('float32/gpu', float32_times / gpu_times, file=sys.stderr)
print('expected float32/gpu', (
expected_times_32 / gpu_times), file=sys.stderr)

def compare(x, y):
ratio = x / y
# If there is more then 5% difference between the expected
# time and the real time, we consider this an error.
return sum((ratio < 0.95) + (ratio > 1.05))

print(file=sys.stderr)
if do_float64:
err = compare(expected_times_64, float64_times)
print('speed_failure_float64=' + str(err), file=sys.stderr)
if do_float32:
err = compare(expected_times_32, float32_times)
print('speed_failure_float32=' + str(err), file=sys.stderr)
if do_gpu:
err = compare(expected_times_gpu, gpu_times)
print('speed_failure_gpu=' + str(err), file=sys.stderr)

assert not numpy.isnan(gpu_times).any()

# Write JUnit xml for speed test performance report

speed_file = 'speedtests_time.xml'

# Define speed test file write method
def write_junit(filename, algos, times, label):
with open(filename, 'a') as f:
for algo, time in zip(algos, times):
f.write(' <testcase classname="{label}" name="{algo}" time="{time}">'
.format(label=label, algo=algo, time=time))
f.write(' </testcase>\n')

test_total = numpy.size(float64_times) \
+ numpy.size(float32_times) \
+ numpy.size(gpu_times)

with open(speed_file, 'w') as f:
f.write('<?xml version="1.0" encoding="UTF-8"?>\n')
f.write('<testsuite name="theano_speedtests" tests="{ntests}">\n'
.format(ntests=numpy.size(test_total)))

write_junit(speed_file, algo_executed, float64_times, label='float64')
Copy link
Member

@nouiz nouiz Oct 7, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

to make the code more clear, you can do just one "with" and pass f here instead of speed_file.

Copy link
Contributor Author

@slefrancois slefrancois Oct 11, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in new commits, also corrected some errors that would pop up if some tests flags were set to false, it would still to access the undeclared results variables.

write_junit(speed_file, algo_executed, float32_times, label='float32')
write_junit(speed_file, algo_executed, gpu_times, label='gpu')

with open(speed_file, 'a') as f:
f.write('</testsuite>\n')