Skip to content
This repository was archived by the owner on Mar 21, 2024. It is now read-only.

Commit 4f4381f

Browse files
brycelelbachbrycelelbach
committed
Add gpuCI GPU testing, automatic GPU detection, and support for more compilers:
* CMake: Add support for detecting the compute archs of the GPUs in your system at configure time. * CMake: Update FindTBB.cmake to support the newer oneTBB. * gpuCI: Add a GPU node configuration that builds and tests as little as possible. * gpuCI: Cleanup logic for different build and test configurations. * gpuCI: Fix an unfortunate typo in `determine_build_parallelism.bash` which led to the parallelism level not being set. * gpuCI: Add support for NVC++. * gpuCI: Update to CUDA 11.1 and Ubuntu 20.04. * gpuCI: Add NVC++ and ICC configurations to the CPU axis file. * gpuCI: Add a GPU axis file. * gpuCI: Increase the desired memory per build thread to 4GB. * gpuCI: Add a -j switch which controls build parallelism to `ci/local/build.bash`. * gpuCI: Add support for CMake build types.
1 parent 41dc1dd commit 4f4381f

10 files changed

Lines changed: 480 additions & 125 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,11 @@ set(CMAKE_CXX_EXTENSIONS OFF)
8282
set(THRUST_LIBRARY_OUTPUT_DIR "${CMAKE_BINARY_DIR}/lib")
8383
set(THRUST_EXECUTABLE_OUTPUT_DIR "${CMAKE_BINARY_DIR}/bin")
8484

85+
message(STATUS "CMAKE_LIBRARY_ARCHITECTURE_REGEX: ${CMAKE_LIBRARY_ARCHITECTURE_REGEX}")
86+
message(STATUS "CMAKE_LIBRARY_ARCHITECTURE: ${CMAKE_LIBRARY_ARCHITECTURE}")
87+
message(STATUS "CMAKE_CXX_LIBRARY_ARCHITECTURE: ${CMAKE_CXX_LIBRARY_ARCHITECTURE}")
88+
message(STATUS "CMAKE_CUDA_LIBRARY_ARCHITECTURE: ${CMAKE_CUDA_LIBRARY_ARCHITECTURE}")
89+
8590
thrust_configure_multiconfig()
8691
thrust_build_target_list()
8792

‎ci/axis/cpu.yml‎

Lines changed: 73 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,27 @@
1+
# Copyright (c) 2018-2020 NVIDIA Corporation
2+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
3+
# Released under the Apache License v2.0 with LLVM Exceptions.
4+
# See https://llvm.org/LICENSE.txt for license information.
5+
6+
SDK_TYPE:
7+
- cuda
8+
- nvhpc
9+
10+
SDK_VER:
11+
- 11.1-devel
12+
- 20.9-devel
13+
14+
OS_TYPE:
15+
- ubuntu
16+
117
OS_VER:
2-
- ubuntu18.04
18+
- 20.04
319

420
CXX_TYPE:
5-
- gcc
21+
- nvcxx
622
- clang
23+
- gcc
24+
- icc
725

826
CXX_VER:
927
- 5
@@ -12,17 +30,65 @@ CXX_VER:
1230
- 8
1331
- 9
1432
- 10
33+
- 20.9
34+
- latest
1535

1636
exclude:
17-
# Unsupported compiler version
37+
# Excludes by `SDK_TYPE`.
38+
- CXX_TYPE: gcc
39+
SDK_TYPE: nvhpc
40+
- CXX_TYPE: clang
41+
SDK_TYPE: nvhpc
42+
- CXX_TYPE: icc
43+
SDK_TYPE: nvhpc
44+
- CXX_TYPE: nvcxx
45+
SDK_TYPE: cuda
46+
# Excludes by `SDK_VER`.
47+
- SDK_TYPE: cuda
48+
SDK_VER: 20.9-devel
49+
- SDK_TYPE: nvhpc
50+
SDK_VER: 11.1-devel
51+
# Excludes by `CXX_VER`.
52+
- CXX_TYPE: nvcxx
53+
CXX_VER: 5
54+
- CXX_TYPE: nvcxx
55+
CXX_VER: 6
56+
- CXX_TYPE: nvcxx
57+
CXX_VER: 7
58+
- CXX_TYPE: nvcxx
59+
CXX_VER: 8
60+
- CXX_TYPE: nvcxx
61+
CXX_VER: 9
62+
- CXX_TYPE: nvcxx
63+
CXX_VER: 10
64+
- CXX_TYPE: nvcxx
65+
CXX_VER: latest
66+
- CXX_TYPE: gcc
67+
CXX_VER: 20.9
68+
- CXX_TYPE: gcc
69+
CXX_VER: latest
1870
- CXX_TYPE: clang
1971
CXX_VER: 5
20-
# This config is broken in the docker image: https://github.com/NVIDIA/cccl/issues/6
2172
- CXX_TYPE: clang
2273
CXX_VER: 6
23-
# Needs newer nvcc in image, https://github.com/NVIDIA/cccl/issues/7
2474
- CXX_TYPE: clang
2575
CXX_VER: 10
26-
# Config broken in image: https://github.com/NVIDIA/cccl/issues/8
27-
- CXX_TYPE: gcc
76+
- CXX_TYPE: clang
77+
CXX_VER: 20.9
78+
- CXX_TYPE: clang
79+
CXX_VER: latest
80+
- CXX_TYPE: icc
81+
CXX_VER: 5
82+
- CXX_TYPE: icc
83+
CXX_VER: 6
84+
- CXX_TYPE: icc
85+
CXX_VER: 7
86+
- CXX_TYPE: icc
87+
CXX_VER: 8
88+
- CXX_TYPE: icc
89+
CXX_VER: 9
90+
- CXX_TYPE: icc
2891
CXX_VER: 10
92+
- CXX_TYPE: icc
93+
CXX_VER: 20.9
94+

‎ci/axis/gpu.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright (c) 2018-2020 NVIDIA Corporation
2+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
3+
# Released under the Apache License v2.0 with LLVM Exceptions.
4+
# See https://llvm.org/LICENSE.txt for license information.
5+
6+
SDK_TYPE:
7+
- cuda
8+
9+
SDK_VER:
10+
- 11.1-devel
11+
12+
OS_TYPE:
13+
- ubuntu
14+
15+
OS_VER:
16+
- 20.04
17+
18+
CXX_TYPE:
19+
- gcc
20+
21+
CXX_VER:
22+
- 7
23+

‎ci/common/build.bash‎

Lines changed: 150 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,20 @@
1111

1212
set -e
1313

14-
# Logger function for build status output
15-
function logger() {
14+
# log args...
15+
# Prints out a message with a newline before and after.
16+
function log() {
1617
echo -e "\n>>>> ${@}\n"
1718
}
1819

20+
# append variable value
21+
# Appends `value` to `variable`, adding a space before `value` if `variable`
22+
# is not empty.
23+
function append {
24+
tmp="${!1:+${!1} }${2}"
25+
eval "${1}=\${tmp}"
26+
}
27+
1928
################################################################################
2029
# VARIABLES - Set up bash and environmental variables.
2130
################################################################################
@@ -34,83 +43,172 @@ cd ${WORKSPACE}
3443
mkdir -p build
3544
cd build
3645

46+
if [[ -z "${CMAKE_BUILD_TYPE}" ]]; then
47+
CMAKE_BUILD_TYPE="Release"
48+
fi
49+
50+
CMAKE_BUILD_FLAGS="--"
51+
3752
# The Docker image sets up `${CXX}` and `${CUDACXX}`.
38-
CMAKE_FLAGS="-G Ninja -DCMAKE_CXX_COMPILER='${CXX}' -DCMAKE_CUDA_COMPILER='${CUDACXX}'"
39-
40-
if [ "${BUILD_MODE}" == "branch" ]; then
41-
# Post-commit build.
42-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_INCLUDE_CUB_CMAKE=ON"
43-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_MULTICONFIG=ON"
44-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
45-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
46-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
47-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
48-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
49-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
50-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
51-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
52-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_WORKLOAD=LARGE"
53+
append CMAKE_FLAGS "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
54+
append CMAKE_FLAGS "-DCMAKE_CUDA_COMPILER='${CUDACXX}'"
55+
56+
if [[ "${CXX_TYPE}" == "nvcxx" ]]; then
57+
# NVC++ isn't properly detected by CMake, so we have to tell CMake to ignore
58+
# detection and explicit provide the compiler ID. Ninja currently isn't
59+
# supported, so we just use makefiles.
60+
append CMAKE_FLAGS "-DCMAKE_CUDA_COMPILER_FORCED=ON"
61+
append CMAKE_FLAGS "-DCMAKE_CUDA_COMPILER_ID=NVCXX"
62+
# Don't stop on build failures.
63+
append CMAKE_BUILD_FLAGS "-k"
64+
# NVC++ currently uses a lot of memory.
65+
PARALLEL_LEVEL=1
5366
else
54-
# Pre-commit build.
55-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON"
56-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_50=ON"
57-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_60=ON"
58-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_70=ON"
59-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_80=ON"
60-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_INCLUDE_CUB_CMAKE=ON"
61-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_MULTICONFIG=ON"
62-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
63-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
64-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
65-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
66-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
67-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
68-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
69-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
70-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_WORKLOAD=SMALL"
67+
if [[ "${CXX_TYPE}" == "icc" ]]; then
68+
# Only the latest version of the Intel C++ compiler, which NVCC doesn't
69+
# officially support yet, is freely available.
70+
append CMAKE_FLAGS "-DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler"
71+
fi
72+
# We're using NVCC so we need to set the host compiler.
73+
append CMAKE_FLAGS "-DCMAKE_CXX_COMPILER='${CXX}'"
74+
append CMAKE_FLAGS "-G Ninja"
75+
# Don't stop on build failures.
76+
append CMAKE_BUILD_FLAGS "-k0"
7177
fi
7278

73-
CMAKE_BUILD_FLAGS="-j${PARALLEL_LEVEL}"
79+
if [[ -n "${PARALLEL_LEVEL}" ]]; then
80+
DETERMINE_PARALLELISM_FLAGS="-j ${PARALLEL_LEVEL}"
81+
fi
82+
83+
# COVERAGE_PLAN options:
84+
# * Exhaustive
85+
# * Thorough
86+
# * Minimal
87+
if [[ -z "${COVERAGE_PLAN}" ]]; then
88+
if [[ "${BUILD_TYPE}" == "cpu" ]] && [[ "${BUILD_MODE}" == "branch" ]]; then
89+
# Post-commit CPU CI builds.
90+
COVERAGE_PLAN="Exhaustive"
91+
elif [[ "${BUILD_TYPE}" == "cpu" ]]; then
92+
# Pre-commit CPU CI builds.
93+
COVERAGE_PLAN="Thorough"
94+
elif [[ "${BUILD_TYPE}" == "gpu" ]]; then
95+
# Pre- and post-commit GPU CI builds.
96+
COVERAGE_PLAN="Minimal"
97+
fi
98+
fi
7499

75-
if [ ! -z "${@}" ]; then
76-
CMAKE_BUILD_FLAGS="${CMAKE_BUILD_FLAGS} -- ${@}"
100+
case "${COVERAGE_PLAN}" in
101+
Exhaustive)
102+
append CMAKE_FLAGS "-DTHRUST_INCLUDE_CUB_CMAKE=ON"
103+
append CMAKE_FLAGS "-DTHRUST_ENABLE_MULTICONFIG=ON"
104+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
105+
append CMAKE_FLAGS "-DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
106+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
107+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
108+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
109+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
110+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
111+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
112+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_WORKLOAD=LARGE"
113+
;;
114+
Thorough)
115+
append CMAKE_FLAGS "-DTHRUST_INCLUDE_CUB_CMAKE=ON"
116+
append CMAKE_FLAGS "-DTHRUST_ENABLE_MULTICONFIG=ON"
117+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
118+
append CMAKE_FLAGS "-DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
119+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
120+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
121+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
122+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
123+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
124+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
125+
append CMAKE_FLAGS "-DTHRUST_MULTICONFIG_WORKLOAD=SMALL"
126+
append CMAKE_FLAGS "-DTHRUST_AUTO_DETECT_COMPUTE_ARCHS=ON"
127+
if [[ "${CXX_TYPE}" != "nvcxx" ]]; then
128+
# NVC++ can currently only target one CUDA architecture at a time.
129+
append CMAKE_FLAGS "-DTHRUST_ENABLE_COMPUTE_50=ON"
130+
append CMAKE_FLAGS "-DTHRUST_ENABLE_COMPUTE_60=ON"
131+
append CMAKE_FLAGS "-DTHRUST_ENABLE_COMPUTE_70=ON"
132+
fi
133+
append CMAKE_FLAGS "-DTHRUST_ENABLE_COMPUTE_80=ON"
134+
;;
135+
Minimal)
136+
append CMAKE_FLAGS "-DTHRUST_INCLUDE_CUB_CMAKE=ON"
137+
append CMAKE_FLAGS "-DCUB_ENABLE_THOROUGH_TESTING=OFF"
138+
append CMAKE_FLAGS "-DCUB_ENABLE_BENCHMARK_TESTING=OFF"
139+
append CMAKE_FLAGS "-DCUB_ENABLE_MINIMAL_TESTING=ON"
140+
append CMAKE_FLAGS "-DTHRUST_HOST_SYSTEM=CPP"
141+
append CMAKE_FLAGS "-DTHRUST_DEVICE_SYSTEM=CUDA"
142+
append CMAKE_FLAGS "-DTHRUST_CPP_DIALECT=14"
143+
append CMAKE_FLAGS "-DTHRUST_AUTO_DETECT_COMPUTE_ARCHS=ON"
144+
;;
145+
esac
146+
147+
if [[ -n "${@}" ]]; then
148+
append CMAKE_BUILD_FLAGS "${@}"
77149
fi
78150

79-
CTEST_FLAGS=""
151+
append CTEST_FLAGS "--output-on-failure"
80152

81-
if [ "${BUILD_TYPE}" == "cpu" ]; then
82-
CTEST_FLAGS="${CTEST_FLAGS} -E ^cub|^thrust.*cuda"
153+
if [[ "${BUILD_TYPE}" == "cpu" ]]; then
154+
append CTEST_FLAGS "-E ^cub|^thrust.*cuda"
83155
fi
84156

85-
if [ ! -z "${@}" ]; then
86-
CTEST_FLAGS="${CTEST_FLAGS} -R ^${@}$"
157+
if [[ -n "${@}" ]]; then
158+
for arg in "${@}"
159+
do
160+
append CTEST_FLAGS "-R ^${arg}$"
161+
done
87162
fi
88163

164+
# Export variables so they'll show up in the logs when we report the environment.
165+
export COVERAGE_PLAN
166+
export CMAKE_FLAGS
167+
export CMAKE_BUILD_FLAGS
168+
export CTEST_FLAGS
169+
89170
################################################################################
90171
# ENVIRONMENT - Configure and print out information about the environment.
91172
################################################################################
92173

93-
logger "Get environment..."
174+
log "Determine system topology..."
175+
# Set `${PARALLEL_LEVEL}` if it is unset; otherwise, this just reports the
176+
# system topology.
177+
source ${WORKSPACE}/ci/common/determine_build_parallelism.bash ${DETERMINE_PARALLELISM_FLAGS}
178+
179+
log "Get environment..."
94180
env
95181

96-
logger "Check versions..."
97-
${CXX} --version
98-
${CUDACXX} --version
182+
log "Check versions..."
183+
CXX_VERSION_OUTPUT="$(${CXX} --version 2>&1)"
184+
echo -e "${CXX_VERSION_OUTPUT}\n"
185+
CUDACXX_VERSION_OUTPUT="$(${CUDACXX} --version 2>&1)"
186+
echo -e "${CUDACXX_VERSION_OUTPUT}\n"
187+
NVIDIA_SMI_OUTPUT="$(nvidia-smi 2>&1)"
188+
echo -e "${NVIDIA_SMI_OUTPUT}\n"
99189

100190
################################################################################
101191
# BUILD - Build Thrust and CUB examples and tests.
102192
################################################################################
103193

104-
logger "Configure Thrust and CUB..."
105-
cmake .. ${CMAKE_FLAGS}
194+
log "Configure Thrust and CUB..."
195+
echo cmake .. ${CMAKE_FLAGS}
196+
TIMEFORMAT=$'\nConfigure Time: %lR'
197+
time cmake .. ${CMAKE_FLAGS}
106198

107-
logger "Build Thrust and CUB..."
108-
cmake --build . ${CMAKE_BUILD_FLAGS}
199+
log "Build Thrust and CUB..."
200+
# ${PARALLEL_LEVEL} needs to be passed after we run
201+
# determine_build_parallelism.bash, so it can't be part of ${CMAKE_BUILD_FLAGS}.
202+
echo cmake --build . ${CMAKE_BUILD_FLAGS} -j ${PARALLEL_LEVEL}
203+
TIMEFORMAT=$'\nBuild Time: %lR'
204+
time cmake --build . ${CMAKE_BUILD_FLAGS} -j ${PARALLEL_LEVEL}
109205

110206
################################################################################
111207
# TEST - Run Thrust and CUB examples and tests.
112208
################################################################################
113209

114-
logger "Test Thrust and CUB..."
115-
ctest ${CTEST_FLAGS}
210+
log "Test Thrust and CUB..."
211+
echo ctest ${CTEST_FLAGS}
212+
TIMEFORMAT=$'\nTest Time: %lR'
213+
time ctest ${CTEST_FLAGS}
116214

0 commit comments

Comments
 (0)