enhance OpenBLAS easyblock to support running LAPACK test suite + checking how many tests fail

easybuild/easyblocks/o/openblas.py

@@ -3,21 +3,39 @@
 
 @author: Andrew Edmondson (University of Birmingham)
 @author: Alex Domingo (Vrije Universiteit Brussel)
+@author: Kenneth Hoste (Ghent University)
 """
 import os
+import re
 from distutils.version import LooseVersion
 from easybuild.easyblocks.generic.configuremake import ConfigureMake
+from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.systemtools import POWER, get_cpu_architecture, get_shared_lib_ext
-from easybuild.tools.build_log import print_warning
+from easybuild.tools.build_log import EasyBuildError, print_warning
 from easybuild.tools.config import ERROR
 from easybuild.tools.run import run_cmd, check_log_for_errors
 
+LAPACK_TEST_TARGET = 'lapack-test'
 TARGET = 'TARGET'
 
 
 class EB_OpenBLAS(ConfigureMake):
     """Support for building/installing OpenBLAS."""
 
+    @staticmethod
+    def extra_options():
+        """Custom easyconfig parameters for OpenBLAS easyblock."""
+        extra_vars = {
+            'max_failing_lapack_tests_num_errors': [0, "Maximum number of LAPACK tests failing "
+                                                    "due to numerical errors", CUSTOM],
+            'max_failing_lapack_tests_other_errors': [0, "Maximum number of LAPACK tests failing "
+                                                      "due to non-numerical errors", CUSTOM],
+            'run_lapack_tests': [False, "Run LAPACK tests during test step, "
+                                        "and check whether failing tests exceeds threshold", CUSTOM],
+        }
+
+        return ConfigureMake.extra_options(extra_vars)
+
     def configure_step(self):
         """ set up some options - but no configure command to run"""
 
@@ -72,10 +90,47 @@ def build_step(self):
         cmd = ' '.join([self.cfg['prebuildopts'], makecmd, ' '.join(build_parts), self.cfg['buildopts']])
         run_cmd(cmd, log_all=True, simple=True)
 
+    def check_lapack_test_results(self, test_output):
+        """Check output of OpenBLAS' LAPACK test suite ('make lapack-test')."""
+
+        # example:
+        #                         -->   LAPACK TESTING SUMMARY  <--
+        # SUMMARY                 nb test run     numerical error         other error
+        # ================        ===========     =================       ================
+        # ...
+        # --> ALL PRECISIONS      4116982         4172    (0.101%)        0       (0.000%)
+        test_summary_pattern = r'\s+'.join([
+            r"^--> ALL PRECISIONS",
+            r"(?P<test_cnt>[0-9]+)",
+            r"(?P<test_fail_num_error>[0-9]+)\s+\([0-9.]+\%\)",
+            r"(?P<test_fail_other_error>[0-9]+)\s+\([0-9.]+\%\)",
+        ])
+        regex = re.compile(test_summary_pattern, re.M)
+        res = regex.search(test_output)
+        if res:
+            (tot_cnt, fail_cnt_num_errors, fail_cnt_other_errors) = [int(x) for x in res.groups()]
+            msg = "%d LAPACK tests run - %d failed due to numerical errors - %d failed due to other errors"
+            self.log.info(msg, tot_cnt, fail_cnt_num_errors, fail_cnt_other_errors)
+
+            if fail_cnt_other_errors > self.cfg['max_failing_lapack_tests_other_errors']:
+                raise EasyBuildError("Too many LAPACK tests failed due to non-numerical errors: %d (> %d)",
+                                     fail_cnt_other_errors, self.cfg['max_failing_lapack_tests_other_errors'])
+
+            if fail_cnt_num_errors > self.cfg['max_failing_lapack_tests_num_errors']:
+                raise EasyBuildError("Too many LAPACK tests failed due to numerical errors: %d (> %d)",
+                                     fail_cnt_num_errors, self.cfg['max_failing_lapack_tests_num_errors'])
+        else:
+            raise EasyBuildError("Failed to find test summary using pattern '%s' in test output: %s",
+                                 test_summary_pattern, test_output)
+
     def test_step(self):
         """ Mandatory test step plus optional runtest"""
 
         run_tests = ['tests']
+
+        if self.cfg['run_lapack_tests']:
+            run_tests += [LAPACK_TEST_TARGET]
+
         if self.cfg['runtest']:
             run_tests += [self.cfg['runtest']]
 
@@ -86,6 +141,10 @@ def test_step(self):
             # Raise an error if any test failed
             check_log_for_errors(out, [('FATAL ERROR', ERROR)])
 
+            # check number of failing LAPACK tests more closely
+            if runtest == LAPACK_TEST_TARGET:
+                self.check_lapack_test_results(out)
+
     def sanity_check_step(self):
         """ Custom sanity check for OpenBLAS """
         custom_paths = {