From 82c85e2a6f74a92736c2afdd805710dedfcc4f4f Mon Sep 17 00:00:00 2001 From: slefrancois Date: Thu, 6 Oct 2016 14:36:46 -0400 Subject: [PATCH 1/3] Add JUnit writer for speed tests, remove hardcoded reference times --- code/test.py | 117 +++++++++++++-------------------------------------- 1 file changed, 29 insertions(+), 88 deletions(-) diff --git a/code/test.py b/code/test.py index 4332e8b0..60c0af02 100644 --- a/code/test.py +++ b/code/test.py @@ -98,43 +98,7 @@ def speed(): do_gpu = True algo_executed = [s for idx, s in enumerate(algo) if to_exec[idx]] - #Timming expected are from the buildbot that have an i7-920 @ - # 2.67GHz with hyperthread enabled for the cpu, 12G of ram. An GeForce GTX - # 580 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD - # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. - - expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4, - 346.9, 355.0, 268.2, 115.8, 16.8, 91.6]) - expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 63.1, 71, - 191.2, 199.0, 201.9, 107, 12.6, 61.3]) - - # Number with just 1 decimal are new value that are faster with - # the Theano version 0.5rc2 Other number are older. They are not - # updated, as we where faster in the past! - # TODO: find why and fix this! - -# Here is the value for the buildbot on February 3th 2012 with a GTX 285 -# sgd, cg mlp conv da -# sda dbn rbm -# gpu times[3.72957802, 9.94316864, 29.1772666, 9.13857198, 25.91144657, -# 18.30802011, 53.38651466, 285.41386175] -# expected [3.076634879, 7.555234910, 18.99226785, 9.58915591, 24.130070450, -# 24.77524018, 92.66246653, 322.340329170] -# sgd, cg mlp conv da -# sda dbn rbm -#expected/get [0.82492841, 0.75984178, 0.65092691, 1.04930573, 0.93125138 -# 1.35324519 1.7356905 1.12937868] - - expected_times_gpu = numpy.asarray([2.9, 7.55523491, 18.99226785, - 5.8, 19.2, - 11.2, 7.3, 122, 112.5, 31.1, 8.3]) - expected_times_64 = [s for idx, s in enumerate(expected_times_64) - if to_exec[idx]] - expected_times_32 = [s for idx, s in enumerate(expected_times_32) - if to_exec[idx]] - expected_times_gpu = [s for idx, s in enumerate(expected_times_gpu) - if to_exec[idx]] - + def time_test(m, l, idx, f, **kwargs): if not to_exec[idx]: return @@ -196,9 +160,6 @@ def do_tests(): float64_times = do_tests() print(algo_executed, file=sys.stderr) print('float64 times', float64_times, file=sys.stderr) - print('float64 expected', expected_times_64, file=sys.stderr) - print('float64 % expected/get', ( - expected_times_64 / float64_times), file=sys.stderr) #test in float32 in FAST_RUN mode on the cpu theano.config.floatX = 'float32' @@ -206,9 +167,6 @@ def do_tests(): float32_times = do_tests() print(algo_executed, file=sys.stderr) print('float32 times', float32_times, file=sys.stderr) - print('float32 expected', expected_times_32, file=sys.stderr) - print('float32 % expected/get', ( - expected_times_32 / float32_times), file=sys.stderr) if do_float64: print('float64/float32', ( @@ -218,18 +176,10 @@ def do_tests(): 'in one place'), file=sys.stderr) print(algo_executed, file=sys.stderr) print('float64 times', float64_times, file=sys.stderr) - print('float64 expected', expected_times_64, file=sys.stderr) - print('float64 % expected/get', ( - expected_times_64 / float64_times), file=sys.stderr) print('float32 times', float32_times, file=sys.stderr) - print('float32 expected', expected_times_32, file=sys.stderr) - print('float32 % expected/get', ( - expected_times_32 / float32_times), file=sys.stderr) print('float64/float32', ( float64_times / float32_times), file=sys.stderr) - print('expected float64/float32', ( - expected_times_64 / float32_times), file=sys.stderr) #test in float32 in FAST_RUN mode on the gpu import theano.sandbox.cuda @@ -238,9 +188,6 @@ def do_tests(): gpu_times = do_tests() print(algo_executed, file=sys.stderr) print('gpu times', gpu_times, file=sys.stderr) - print('gpu expected', expected_times_gpu, file=sys.stderr) - print('gpu % expected/get', ( - expected_times_gpu / gpu_times), file=sys.stderr) if do_float64: print('float64/gpu', float64_times / gpu_times, file=sys.stderr) @@ -252,50 +199,44 @@ def do_tests(): print(algo_executed, file=sys.stderr) if do_float64: print('float64 times', float64_times, file=sys.stderr) - print('float64 expected', expected_times_64, file=sys.stderr) - print('float64 % expected/get', ( - expected_times_64 / float64_times), file=sys.stderr) if do_float32: print('float32 times', float32_times, file=sys.stderr) - print('float32 expected', expected_times_32, file=sys.stderr) - print('float32 % expected/get', ( - expected_times_32 / float32_times), file=sys.stderr) if do_gpu: print('gpu times', gpu_times, file=sys.stderr) - print('gpu expected', expected_times_gpu, file=sys.stderr) - print('gpu % expected/get', ( - expected_times_gpu / gpu_times), file=sys.stderr) print() if do_float64 and do_float32: print('float64/float32', ( float64_times / float32_times), file=sys.stderr) - print('expected float64/float32', ( - expected_times_64 / float32_times), file=sys.stderr) if do_float64 and do_gpu: print('float64/gpu', float64_times / gpu_times, file=sys.stderr) - print('expected float64/gpu', ( - expected_times_64 / gpu_times), file=sys.stderr) if do_float32 and do_gpu: print('float32/gpu', float32_times / gpu_times, file=sys.stderr) - print('expected float32/gpu', ( - expected_times_32 / gpu_times), file=sys.stderr) - - def compare(x, y): - ratio = x / y - # If there is more then 5% difference between the expected - # time and the real time, we consider this an error. - return sum((ratio < 0.95) + (ratio > 1.05)) - - print(file=sys.stderr) - if do_float64: - err = compare(expected_times_64, float64_times) - print('speed_failure_float64=' + str(err), file=sys.stderr) - if do_float32: - err = compare(expected_times_32, float32_times) - print('speed_failure_float32=' + str(err), file=sys.stderr) - if do_gpu: - err = compare(expected_times_gpu, gpu_times) - print('speed_failure_gpu=' + str(err), file=sys.stderr) - - assert not numpy.isnan(gpu_times).any() + + # Write JUnit xml for speed test performance report + + speed_file = 'speedtests_time.xml' + + # Define speed test file write method + def write_junit(filename, algos, times, label): + with open(filename, 'a') as f: + for algo, time in zip(algos, times): + f.write(' ' + .format(label=label, algo=algo, time=time)) + f.write(' \n') + + test_total = numpy.size(float64_times) \ + + numpy.size(float32_times) \ + + numpy.size(gpu_times) + + with open(speed_file, 'w') as f: + f.write('\n') + f.write('\n' + .format(ntests=numpy.size(test_total))) + + write_junit(speed_file, algo_executed, float64_times, label='float64') + write_junit(speed_file, algo_executed, float32_times, label='float32') + write_junit(speed_file, algo_executed, gpu_times, label='gpu') + + with open(speed_file, 'a') as f: + f.write('\n') From f14107d31d5cb05f192129a95d3f272acf4dbc09 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Fri, 7 Oct 2016 17:28:37 -0400 Subject: [PATCH 2/3] single performance file open, correct only access times variables if tests ran --- code/test.py | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/code/test.py b/code/test.py index 60c0af02..22a59655 100644 --- a/code/test.py +++ b/code/test.py @@ -152,12 +152,18 @@ def do_tests(): saveto='') return numpy.asarray(l) + # Initialize test count and results dictionnary + test_total = 0 + times_dic = {} + #test in float64 in FAST_RUN mode on the cpu import theano if do_float64: theano.config.floatX = 'float64' theano.config.mode = 'FAST_RUN' float64_times = do_tests() + times_dic['float64'] = float64_times + test_total += numpy.size(float64_times) print(algo_executed, file=sys.stderr) print('float64 times', float64_times, file=sys.stderr) @@ -165,6 +171,8 @@ def do_tests(): theano.config.floatX = 'float32' if do_float32: float32_times = do_tests() + times_dic['float32'] = float32_times + test_total += numpy.size(float32_times) print(algo_executed, file=sys.stderr) print('float32 times', float32_times, file=sys.stderr) @@ -186,6 +194,8 @@ def do_tests(): if do_gpu: theano.sandbox.cuda.use('gpu') gpu_times = do_tests() + times_dic['gpu'] = gpu_times + test_total += numpy.size(gpu_times) print(algo_executed, file=sys.stderr) print('gpu times', gpu_times, file=sys.stderr) @@ -213,30 +223,18 @@ def do_tests(): if do_float32 and do_gpu: print('float32/gpu', float32_times / gpu_times, file=sys.stderr) - # Write JUnit xml for speed test performance report - - speed_file = 'speedtests_time.xml' - - # Define speed test file write method - def write_junit(filename, algos, times, label): - with open(filename, 'a') as f: - for algo, time in zip(algos, times): - f.write(' ' - .format(label=label, algo=algo, time=time)) - f.write(' \n') - - test_total = numpy.size(float64_times) \ - + numpy.size(float32_times) \ - + numpy.size(gpu_times) - - with open(speed_file, 'w') as f: + # Generate JUnit performance report + # Define speedtest file write method + def write_junit(f, algos, times, label): + for algo, time in zip(algos, times): + f.write(' ' + .format(label=label, algo=algo, time=time)) + f.write(' \n') + + with open('speedtests_time.xml', 'w') as f: f.write('\n') - f.write('\n' - .format(ntests=numpy.size(test_total))) - - write_junit(speed_file, algo_executed, float64_times, label='float64') - write_junit(speed_file, algo_executed, float32_times, label='float32') - write_junit(speed_file, algo_executed, gpu_times, label='gpu') - - with open(speed_file, 'a') as f: + f.write('\n' + .format(ntests=test_total)) + for label, times in times_dic.items(): + write_junit(f, algo_executed, times, label) f.write('\n') From f724c2c6054c736c548196d2a7a000ec307e0b0d Mon Sep 17 00:00:00 2001 From: slefrancois Date: Fri, 7 Oct 2016 17:36:44 -0400 Subject: [PATCH 3/3] move assert gpu_times not nan --- code/test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/code/test.py b/code/test.py index 22a59655..b3077b7c 100644 --- a/code/test.py +++ b/code/test.py @@ -238,3 +238,6 @@ def write_junit(f, algos, times, label): for label, times in times_dic.items(): write_junit(f, algo_executed, times, label) f.write('\n') + + if do_gpu: + assert not numpy.isnan(gpu_times).any()