Improved support for HIP and other tweaks.

Author: v-dobrev

machine-configs/corona.sh

@@ -41,14 +41,39 @@ function setup_hip()
     hip_lib=${hip_home}/lib64
 }
 
+function setup_gcc8()
+{
+    # Load additional modules from 'opt'; needed for the 'rocm' module
+    module load opt
+    # ROCm 3.8.0 is using /opt/rh/devtoolset-8 for linking with libstdc++, so
+    # we use the module gcc/8.3.1 which seems to be the same as the version in
+    # /opt/rh/devtoolset-8
+    module load gcc/8.3.1
+    module load mvapich2/2.3
+
+    # ROCm and HIP
+    module load rocm/3.8.0
+
+    MPICC=mpicc
+    MPICXX=mpicxx
+    CFLAGS="-O3"
+    # CXX11FLAG="--std=c++11 -fno-gpu-rdc"
+    # Used e.g. by libCEED:
+    NATIVE_CFLAG="-march=native"
+
+    hip_path=$(dirname $(which hipcc))
+    hip_home=$(dirname ${hip_path})
+    hip_lib=${hip_home}/lib64
+}
+
 function set_mpi_options()
 {
     echo "${cyan}MPI setup${none}"
     num_proc_node=${num_proc_run}
     compose_mpi_run_command
 }
 
-valid_compilers="hip"
+valid_compilers="hip gcc8"
 
 num_proc_detect="$(getconf _NPROCESSORS_ONLN)"
 num_proc_build=${num_proc_build:-$num_proc_detect}

package-builders/cuda.sh

@@ -39,7 +39,7 @@ pkg="CUDA"
 function build_package()
 {
    if [[ -n "$CUDA_ENABLED" ]]; then
-      echo "CUDA is enabled."
+      echo "CUDA is enabled. [ cuda_home = $cuda_home ]"
    else
       echo "Error: cannot enable CUDA: 'cuda_home' is not configured. Stop."
       return 1

package-builders/hip.sh

@@ -40,7 +40,7 @@ function build_package()
 {
     echo "${cyan}HIP build_package${none}"
     if [[ -n "$HIP_ENABLED" ]]; then
-        echo "HIP is enabled."
+        echo "HIP is enabled. [ hip_home = $hip_home ]"
     else
         echo "Error: cannot enable HIP: 'hip_home' is not configured. Stop."
         return 1

package-builders/libceed.sh

@@ -81,10 +81,19 @@ function libceed_build()
       OPT_FLAGS+=" $NATIVE_CFLAG"
    fi
    # Check for optional packages used by backends
-   # Note: If OCCA already is built in OUT_DIR, libCEED will find it in ../occa
-   #       and use it; in particular, the next INFO message will be incorrect in
-   #       this case.
-   local OCCA_MAKE_OPTS=()
+   local CUDA_MAKE_OPTS=("CUDA_DIR=/disable-cuda")
+   if [[ -n "$CUDA_ENABLED" ]]; then
+      CUDA_MAKE_OPTS=("CUDA_DIR=${cuda_home}")
+   else
+      echo "${magenta}INFO: Building $pkg without CUDA ...${none}"
+   fi
+   local HIP_MAKE_OPTS=("HIP_DIR=/disable-hip")
+   if [[ -n "$HIP_ENABLED" ]]; then
+      HIP_MAKE_OPTS=("HIP_DIR=${hip_home}")
+   else
+      echo "${magenta}INFO: Building $pkg without HIP ...${none}"
+   fi
+   local OCCA_MAKE_OPTS=("OCCA_DIR=/disable-occa")
    if [[ -n "$OCCA_DIR" ]]; then
       OCCA_MAKE_OPTS=("OCCA_DIR=${OCCA_DIR}")
    else
@@ -100,14 +109,16 @@ function libceed_build()
          CXX="$MPICXX" \
          FC="$MPIF77" \
          OPT="$OPT_FLAGS" \
+         "${CUDA_MAKE_OPTS[@]}" \
+         "${HIP_MAKE_OPTS[@]}" \
          "${OCCA_MAKE_OPTS[@]}"
    } &> "${pkg_bld_dir}_build.log" || {
       echo " ... building $pkg FAILED, see log for details."
       return 1
    }
    echo "Build successful."
    print_variables "$pkg_var_prefix" \
-      LIBCEED_BRANCH OCCA_DIR \
+      LIBCEED_BRANCH CUDA_ENABLED cuda_home HIP_ENABLED hip_home OCCA_DIR \
       > "${pkg_bld_dir}_build_successful"
 }
 

package-builders/mfem.sh

@@ -171,8 +171,9 @@ function mfem_build()
    echo "Build successful."
    print_variables "$pkg_var_prefix" \
       MFEM_BRANCH MFEM_DEBUG \
-      HYPRE_DIR METIS_DIR METIS_VERSION CUDA_ENABLED cuda_home OCCA_DIR \
-      RAJA_DIR OMP_ENABLED omp_flag LIBCEED_DIR SUNDIALS_DIR \
+      HYPRE_DIR METIS_DIR METIS_VERSION CUDA_ENABLED cuda_home \
+      HIP_ENABLED hip_home OCCA_DIR RAJA_DIR OMP_ENABLED omp_flag \
+      LIBCEED_DIR SUNDIALS_DIR \
       > "${pkg_bld_dir}_build_successful"
 }
 

package-builders/occa.sh

@@ -80,6 +80,14 @@ function occa_build()
       export OCCA_CUDA_ENABLED=0
       echo "${magenta}INFO: Building $pkg without CUDA ...${none}"
    fi
+   if [[ -n "$HIP_ENABLED" ]]; then
+      add_to_path after OCCA_INCLUDE_PATH "${hip_home}/hip/include"
+      add_to_path after OCCA_INCLUDE_PATH "${hip_home}/opencl/include"
+      export OCCA_INCLUDE_PATH
+   else
+      export OCCA_HIP_ENABLED=0
+      echo "${magenta}INFO: Building $pkg without HIP ...${none}"
+   fi
    echo "Building $pkg, sending output to ${pkg_bld_dir}_build.log ..." && {
       cd "$pkg_bld_dir" && \
       make \
@@ -93,7 +101,7 @@ function occa_build()
    }
    echo "Build successful."
    print_variables "$pkg_var_prefix" \
-      OCCA_BRANCH CUDA_ENABLED cuda_home \
+      OCCA_BRANCH CUDA_ENABLED cuda_home HIP_ENABLED hip_home \
       > "${pkg_bld_dir}_build_successful"
 }
 

package-builders/openmp.sh

@@ -39,7 +39,7 @@ pkg="OpenMP"
 function build_package()
 {
    if [[ -n "$OMP_ENABLED" ]]; then
-      echo "OpenMP is enabled."
+      echo "OpenMP is enabled. [ omp_flag = $omp_flag ]"
    else
       echo "Error: cannot enable OpenMP: 'omp_flag' is not configured. Stop."
       return 1