Add gpuCI GPU testing, automatic GPU detection, and support for more compilers:

* CMake: Add support for detecting the compute archs of the GPUs in your system
  at configure time.
* gpuCI: Add a GPU node configuration that builds and tests as little as possible.
* gpuCI: Cleanup logic for different build and test configurations.
* gpuCI: Fix an unfortunate typo in `determine_build_parallelism.bash` which
  led to the parallelism level not being set.
* gpuCI: Add support for NVC++.
* gpuCI: Update to CUDA 11.1 and Ubuntu 20.04.
* gpuCI: Add NVC++ and ICC configurations to the CPU axis file.
* gpuCI: Add a GPU axis file.
* gpuCI: Increase the desired memory per build thread to 4GB.
* gpuCI: Add a -j switch which controls build parallelism to `ci/local/build.bash`.
* gpuCI: Add support for CMake build types.

Author: brycelelbach

ci/axis/cpu.yml

@@ -1,9 +1,27 @@
+# Copyright (c) 2018-2020 NVIDIA Corporation
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+# Released under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+
+SDK_TYPE:
+  - nvcr.io/nvidia/cuda
+  - nvcr.io/nvidia/nvhpc
+
+SDK_VER:
+  - 11.1-devel
+  - 20.9-devel
+
+OS_TYPE:
+  - ubuntu
+
 OS_VER:
-  - ubuntu18.04
+  - 20.04-devel
 
 CXX_TYPE:
-  - gcc
+  - nvcxx
   - clang
+  - gcc
+  - icc
 
 CXX_VER:
   - 5
@@ -12,17 +30,61 @@ CXX_VER:
   - 8
   - 9
   - 10
+  - 20.9
+  - latest
 
 exclude:
-  # Unsupported compiler version
+  # Excludes by `SDK_TYPE`.
+  - CXX_TYPE: clang
+    SDK_TYPE: nvcr.io/nvidia/nvhpc
+  - CXX_TYPE: icc
+    SDK_TYPE: nvcr.io/nvidia/nvhpc
+  # Excludes by `SDK_VER`.
+  - SDK_TYPE: nvcr.io/nvidia/cuda
+    SDK_VER: 20.9-devel
+  - SDK_TYPE: nvcr.io/nvidia/nvhpc
+    SDK_VER: 11.1-devel
+  # Excludes by `CXX_VER`.
+  - CXX_TYPE: nvcxx
+    CXX_VER: 5
+  - CXX_TYPE: nvcxx
+    CXX_VER: 6
+  - CXX_TYPE: nvcxx
+    CXX_VER: 7
+  - CXX_TYPE: nvcxx
+    CXX_VER: 8
+  - CXX_TYPE: nvcxx
+    CXX_VER: 9
+  - CXX_TYPE: nvcxx
+    CXX_VER: 10
+  - CXX_TYPE: nvcxx
+    CXX_VER: latest
+  - CXX_TYPE: gcc
+    CXX_VER: 20.9
+  - CXX_TYPE: gcc
+    CXX_VER: latest
   - CXX_TYPE: clang
     CXX_VER: 5
-  # This config is broken in the docker image: https://github.com/NVIDIA/cccl/issues/6
   - CXX_TYPE: clang
     CXX_VER: 6
-  # Needs newer nvcc in image, https://github.com/NVIDIA/cccl/issues/7
   - CXX_TYPE: clang
     CXX_VER: 10
-  # Config broken in image: https://github.com/NVIDIA/cccl/issues/8
-  - CXX_TYPE: gcc
+  - CXX_TYPE: clang
+    CXX_VER: 20.9
+  - CXX_TYPE: clang
+    CXX_VER: latest
+  - CXX_TYPE: icc
+    CXX_VER: 5
+  - CXX_TYPE: icc
+    CXX_VER: 6
+  - CXX_TYPE: icc
+    CXX_VER: 7
+  - CXX_TYPE: icc
+    CXX_VER: 8
+  - CXX_TYPE: icc
+    CXX_VER: 9
+  - CXX_TYPE: icc
     CXX_VER: 10
+  - CXX_TYPE: icc
+    CXX_VER: 20.9
+

ci/axis/gpu.yml

@@ -0,0 +1,23 @@
+# Copyright (c) 2018-2020 NVIDIA Corporation
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+# Released under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+
+SDK_TYPE:
+  - nvcr.io/nvidia/cuda
+
+SDK_VER:
+  - 11.1-devel
+
+OS_TYPE:
+  - ubuntu
+
+OS_VER:
+  - 20.04-devel
+
+CXX_TYPE:
+  - gcc
+
+CXX_VER:
+  - 7
+

ci/common/build.bash

@@ -20,6 +20,8 @@ function logger() {
 # VARIABLES - Set up bash and environmental variables.
 ################################################################################
 
+SCRIPT_PATH=$(cd $(dirname ${0}); pwd -P)
+
 # Get the variables the Docker container set up for us: ${CXX}, ${CUDACXX}, etc.
 source /etc/cccl.bashrc
 
@@ -34,62 +36,122 @@ cd ${WORKSPACE}
 mkdir -p build
 cd build
 
+if [[ ! -z "${PARALLEL_LEVEL}" ]]; then
+  DETERMINE_PARALLELISM_FLAGS="-j ${PARALLEL_LEVEL}"
+fi
+
+if [[ ! -z "${CMAKE_BUILD_TYPE}" ]]; then
+  CMAKE_BUILD_TYPE="Release"
+fi
+
 # The Docker image sets up `${CXX}` and `${CUDACXX}`.
-CMAKE_FLAGS="-G Ninja -DCMAKE_CXX_COMPILER='${CXX}' -DCMAKE_CUDA_COMPILER='${CUDACXX}'"
-
-if [ "${BUILD_MODE}" == "branch" ]; then
-  # Post-commit build.
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_INCLUDE_CUB_CMAKE=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_MULTICONFIG=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_WORKLOAD=LARGE"
+CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_CUDA_COMPILER='${CUDACXX}'"
+
+# NVC++ requires special configuration; if we're using the NVHPC SDK, we assume
+# we're using NVC++.
+if [[ ! -z "${HPCSDK_VERSION}" ]]; then
+  # NVC++ isn't properly detected by CMake, so we have to tell CMake to ignore
+  # detection and explicit provide the compiler ID. Ninja currently isn't
+  # supported, so we just use makefiles.
+  CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_CUDA_COMPILER_FORCED=ON -DCMAKE_CUDA_COMPILER_ID='NVCXX'"
+  # NVC++ currently uses a lot of memory.
+  PARALLEL_LEVEL=1
 else
-  # Pre-commit build.
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_50=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_60=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_70=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_80=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_INCLUDE_CUB_CMAKE=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_MULTICONFIG=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
-  CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_WORKLOAD=SMALL"
+  # If we're using NVCC, we need to set the host compiler.
+  CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_CXX_COMPILER='${CXX}' -G Ninja"
 fi
 
-CMAKE_BUILD_FLAGS="-j${PARALLEL_LEVEL}"
+# COVERAGE_PLAN options:
+# * Exhaustive
+# * Thorough
+# * Minimal
+if [[ ! -z "${COVERAGE_PLAN}" ]]; then
+  if [[ "${BUILD_TYPE}" == "cpu" ]] && [[ "${BUILD_MODE}" == "branch" ]]; then
+    # Post-commit CPU CI builds.
+    COVERAGE_PLAN="Exhaustive"
+  elif [[ "${BUILD_TYPE}" == "cpu" ]]; then
+    # Pre-commit CPU CI builds.
+    COVERAGE_PLAN="Thorough"
+  elif [[ "${BUILD_TYPE}" == "gpu" ]]; then
+    # Pre- and post-commit GPU CI builds.
+    COVERAGE_PLAN="Minimal"
+  fi
+fi
 
-if [ ! -z "${@}" ]; then
-  CMAKE_BUILD_FLAGS="${CMAKE_BUILD_FLAGS} -- ${@}"
+case "${COVERAGE_PLAN}" in
+  EXHAUSTIVE)
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_INCLUDE_CUB_CMAKE=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_MULTICONFIG=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_WORKLOAD=LARGE"
+    ;;
+  THOROUGH)
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_INCLUDE_CUB_CMAKE=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_MULTICONFIG=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_WORKLOAD=SMALL"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_AUTO_DETECT_COMPUTE_ARCHS=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_50=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_60=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_70=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_80=ON"
+    ;;
+  MINIMAL)
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_INCLUDE_CUB_CMAKE=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DCUB_ENABLE_THOROUGH_TESTING=OFF"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DCUB_ENABLE_BENCHMARK_TESTING=OFF"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DCUB_ENABLE_MINIMAL_TESTING=ON"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_HOST_SYSTEM=CPP"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_DEVICE_SYSTEM=CUDA"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_CPP_DIALECT=14"
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_AUTO_DETECT_COMPUTE_ARCHS=ON"
+    ;;
+esac
+
+if [[ ! -z "${@}" ]]; then
+  CMAKE_BUILD_FLAGS="-- ${@}"
 fi
 
 CTEST_FLAGS=""
 
-if [ "${BUILD_TYPE}" == "cpu" ]; then
+if [[ "${BUILD_TYPE}" == "cpu" ]]; then
   CTEST_FLAGS="${CTEST_FLAGS} -E ^cub|^thrust.*cuda"
 fi
 
-if [ ! -z "${@}" ]; then
+if [[ ! -z "${@}" ]]; then
   CTEST_FLAGS="${CTEST_FLAGS} -R ^${@}$"
 fi
 
+# Export variables so they'll show up in the logs when we report the environment.
+export COVERAGE_PLAN
+export CMAKE_FLAGS
+export CMAKE_BUILD_FLAGS
+export CTEST_FLAGS
+
 ################################################################################
 # ENVIRONMENT - Configure and print out information about the environment.
 ################################################################################
 
+logger "Determine system topology..."
+
+# Set `${PARALLEL_LEVEL}` if it is unset; otherwise, this just reports the
+# system topology.
+source ${SCRIPT_PATH}/determine_build_parallelism.bash ${DETERMINE_PARALLELISM_FLAGS}
+
 logger "Get environment..."
 env
 
@@ -102,15 +164,18 @@ ${CUDACXX} --version
 ################################################################################
 
 logger "Configure Thrust and CUB..."
+echo cmake .. ${CMAKE_FLAGS}
 cmake .. ${CMAKE_FLAGS}
 
 logger "Build Thrust and CUB..."
-cmake --build . ${CMAKE_BUILD_FLAGS}
+echo cmake --build . ${CMAKE_BUILD_FLAGS} -j ${PARALLEL_LEVEL}
+cmake --build . ${CMAKE_BUILD_FLAGS} -j ${PARALLEL_LEVEL}
 
 ################################################################################
 # TEST - Run Thrust and CUB examples and tests.
 ################################################################################
 
 logger "Test Thrust and CUB..."
+echo ctest ${CTEST_FLAGS}
 ctest ${CTEST_FLAGS}