Skip to content
This repository was archived by the owner on Mar 21, 2024. It is now read-only.

Commit 90be0d3

Browse files
brycelelbachbrycelelbach
committed
Add gpuCI GPU testing, automatic GPU detection, and support for more compilers:
* CMake: Add support for detecting the compute archs of the GPUs in your system at configure time. * gpuCI: Add a GPU node configuration that builds and tests as little as possible. * gpuCI: Cleanup logic for different build and test configurations. * gpuCI: Fix an unfortunate typo in `determine_build_parallelism.bash` which led to the parallelism level not being set. * gpuCI: Add support for NVC++. * gpuCI: Update to CUDA 11.1 and Ubuntu 20.04. * gpuCI: Add NVC++ and ICC configurations to the CPU axis file. * gpuCI: Add a GPU axis file. * gpuCI: Increase the desired memory per build thread to 4GB. * gpuCI: Add a -j switch which controls build parallelism to `ci/local/build.bash`. * gpuCI: Add support for CMake build types.
1 parent b829f4a commit 90be0d3

7 files changed

Lines changed: 413 additions & 106 deletions

File tree

‎ci/axis/cpu.yml‎

Lines changed: 73 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,27 @@
1+
# Copyright (c) 2018-2020 NVIDIA Corporation
2+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
3+
# Released under the Apache License v2.0 with LLVM Exceptions.
4+
# See https://llvm.org/LICENSE.txt for license information.
5+
6+
SDK_TYPE:
7+
- cuda
8+
- nvhpc
9+
10+
SDK_VER:
11+
- 11.1-devel
12+
- 20.9-devel
13+
14+
OS_TYPE:
15+
- ubuntu
16+
117
OS_VER:
2-
- ubuntu18.04
18+
- 20.04
319

420
CXX_TYPE:
5-
- gcc
21+
- nvcxx
622
- clang
23+
- gcc
24+
- icc
725

826
CXX_VER:
927
- 5
@@ -12,17 +30,65 @@ CXX_VER:
1230
- 8
1331
- 9
1432
- 10
33+
- 20.9
34+
- latest
1535

1636
exclude:
17-
# Unsupported compiler version
37+
# Excludes by `SDK_TYPE`.
38+
- CXX_TYPE: gcc
39+
SDK_TYPE: nvhpc
40+
- CXX_TYPE: clang
41+
SDK_TYPE: nvhpc
42+
- CXX_TYPE: icc
43+
SDK_TYPE: nvhpc
44+
- CXX_TYPE: nvcxx
45+
SDK_TYPE: cuda
46+
# Excludes by `SDK_VER`.
47+
- SDK_TYPE: cuda
48+
SDK_VER: 20.9-devel
49+
- SDK_TYPE: nvhpc
50+
SDK_VER: 11.1-devel
51+
# Excludes by `CXX_VER`.
52+
- CXX_TYPE: nvcxx
53+
CXX_VER: 5
54+
- CXX_TYPE: nvcxx
55+
CXX_VER: 6
56+
- CXX_TYPE: nvcxx
57+
CXX_VER: 7
58+
- CXX_TYPE: nvcxx
59+
CXX_VER: 8
60+
- CXX_TYPE: nvcxx
61+
CXX_VER: 9
62+
- CXX_TYPE: nvcxx
63+
CXX_VER: 10
64+
- CXX_TYPE: nvcxx
65+
CXX_VER: latest
66+
- CXX_TYPE: gcc
67+
CXX_VER: 20.9
68+
- CXX_TYPE: gcc
69+
CXX_VER: latest
1870
- CXX_TYPE: clang
1971
CXX_VER: 5
20-
# This config is broken in the docker image: https://github.com/NVIDIA/cccl/issues/6
2172
- CXX_TYPE: clang
2273
CXX_VER: 6
23-
# Needs newer nvcc in image, https://github.com/NVIDIA/cccl/issues/7
2474
- CXX_TYPE: clang
2575
CXX_VER: 10
26-
# Config broken in image: https://github.com/NVIDIA/cccl/issues/8
27-
- CXX_TYPE: gcc
76+
- CXX_TYPE: clang
77+
CXX_VER: 20.9
78+
- CXX_TYPE: clang
79+
CXX_VER: latest
80+
- CXX_TYPE: icc
81+
CXX_VER: 5
82+
- CXX_TYPE: icc
83+
CXX_VER: 6
84+
- CXX_TYPE: icc
85+
CXX_VER: 7
86+
- CXX_TYPE: icc
87+
CXX_VER: 8
88+
- CXX_TYPE: icc
89+
CXX_VER: 9
90+
- CXX_TYPE: icc
2891
CXX_VER: 10
92+
- CXX_TYPE: icc
93+
CXX_VER: 20.9
94+

‎ci/axis/gpu.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright (c) 2018-2020 NVIDIA Corporation
2+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
3+
# Released under the Apache License v2.0 with LLVM Exceptions.
4+
# See https://llvm.org/LICENSE.txt for license information.
5+
6+
SDK_TYPE:
7+
- cuda
8+
9+
SDK_VER:
10+
- 11.1-devel
11+
12+
OS_TYPE:
13+
- ubuntu
14+
15+
OS_VER:
16+
- 20.04
17+
18+
CXX_TYPE:
19+
- gcc
20+
21+
CXX_VER:
22+
- 7
23+

‎ci/common/build.bash‎

Lines changed: 121 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -34,62 +34,137 @@ cd ${WORKSPACE}
3434
mkdir -p build
3535
cd build
3636

37+
if [[ -z "${CMAKE_BUILD_TYPE}" ]]; then
38+
CMAKE_BUILD_TYPE="Release"
39+
fi
40+
41+
CMAKE_BUILD_FLAGS="--"
42+
3743
# The Docker image sets up `${CXX}` and `${CUDACXX}`.
38-
CMAKE_FLAGS="-G Ninja -DCMAKE_CXX_COMPILER='${CXX}' -DCMAKE_CUDA_COMPILER='${CUDACXX}'"
39-
40-
if [ "${BUILD_MODE}" == "branch" ]; then
41-
# Post-commit build.
42-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_INCLUDE_CUB_CMAKE=ON"
43-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_MULTICONFIG=ON"
44-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
45-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
46-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
47-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
48-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
49-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
50-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
51-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
52-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_WORKLOAD=LARGE"
44+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
45+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DCMAKE_CUDA_COMPILER='${CUDACXX}'"
46+
47+
if [[ "${CXX_TYPE}" == "nvcxx" ]]; then
48+
# gpuCI overrides ${PATH}, so we need to reload the nvhpc module.
49+
source /usr/share/lmod/lmod/init/bash
50+
module load nvhpc
51+
# NVC++ isn't properly detected by CMake, so we have to tell CMake to ignore
52+
# detection and explicit provide the compiler ID. Ninja currently isn't
53+
# supported, so we just use makefiles.
54+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DCMAKE_CUDA_COMPILER_FORCED=ON"
55+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DCMAKE_CUDA_COMPILER_ID=NVCXX"
56+
# Don't stop on build failures.
57+
CMAKE_BUILD_FLAGS="${CMAKE_BUILD_FLAGS:+${CMAKE_BUILD_FLAGS} }-k"
58+
# NVC++ currently uses a lot of memory.
59+
PARALLEL_LEVEL=1
5360
else
54-
# Pre-commit build.
55-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_DISABLE_ARCH_BY_DEFAULT=ON"
56-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_50=ON"
57-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_60=ON"
58-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_70=ON"
59-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_COMPUTE_80=ON"
60-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_INCLUDE_CUB_CMAKE=ON"
61-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_ENABLE_MULTICONFIG=ON"
62-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
63-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
64-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
65-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
66-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
67-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
68-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
69-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
70-
CMAKE_FLAGS="${CMAKE_FLAGS} -DTHRUST_MULTICONFIG_WORKLOAD=SMALL"
61+
if [[ "${CXX_TYPE}" == "icc" ]]; then
62+
# Only the latest version of the Intel C++ compiler, which NVCC doesn't
63+
# officially support yet, is freely available.
64+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler"
65+
fi
66+
# We're using NVCC so we need to set the host compiler.
67+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DCMAKE_CXX_COMPILER='${CXX}'"
68+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-G Ninja"
69+
# Don't stop on build failures.
70+
CMAKE_BUILD_FLAGS="${CMAKE_BUILD_FLAGS:+${CMAKE_BUILD_FLAGS} }-k0"
7171
fi
7272

73-
CMAKE_BUILD_FLAGS="-j${PARALLEL_LEVEL}"
73+
if [[ -n "${PARALLEL_LEVEL}" ]]; then
74+
DETERMINE_PARALLELISM_FLAGS="-j ${PARALLEL_LEVEL}"
75+
fi
7476

75-
if [ ! -z "${@}" ]; then
76-
CMAKE_BUILD_FLAGS="${CMAKE_BUILD_FLAGS} -- ${@}"
77+
# COVERAGE_PLAN options:
78+
# * Exhaustive
79+
# * Thorough
80+
# * Minimal
81+
if [[ -z "${COVERAGE_PLAN}" ]]; then
82+
if [[ "${BUILD_TYPE}" == "cpu" ]] && [[ "${BUILD_MODE}" == "branch" ]]; then
83+
# Post-commit CPU CI builds.
84+
COVERAGE_PLAN="Exhaustive"
85+
elif [[ "${BUILD_TYPE}" == "cpu" ]]; then
86+
# Pre-commit CPU CI builds.
87+
COVERAGE_PLAN="Thorough"
88+
elif [[ "${BUILD_TYPE}" == "gpu" ]]; then
89+
# Pre- and post-commit GPU CI builds.
90+
COVERAGE_PLAN="Minimal"
91+
fi
7792
fi
7893

79-
CTEST_FLAGS=""
94+
case "${COVERAGE_PLAN}" in
95+
Exhaustive)
96+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_INCLUDE_CUB_CMAKE=ON"
97+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_ENABLE_MULTICONFIG=ON"
98+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
99+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
100+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
101+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
102+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
103+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
104+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
105+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
106+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_WORKLOAD=LARGE"
107+
;;
108+
Thorough)
109+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_INCLUDE_CUB_CMAKE=ON"
110+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_ENABLE_MULTICONFIG=ON"
111+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP11=ON"
112+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_IGNORE_DEPRECATED_CPP_11=ON"
113+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP14=ON"
114+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_DIALECT_CPP17=OFF"
115+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CPP=ON"
116+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_TBB=ON"
117+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_OMP=ON"
118+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_ENABLE_SYSTEM_CUDA=ON"
119+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_MULTICONFIG_WORKLOAD=SMALL"
120+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_AUTO_DETECT_COMPUTE_ARCHS=ON"
121+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_ENABLE_COMPUTE_50=ON"
122+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_ENABLE_COMPUTE_60=ON"
123+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_ENABLE_COMPUTE_70=ON"
124+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_ENABLE_COMPUTE_80=ON"
125+
;;
126+
Minimal)
127+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_INCLUDE_CUB_CMAKE=ON"
128+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DCUB_ENABLE_THOROUGH_TESTING=OFF"
129+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DCUB_ENABLE_BENCHMARK_TESTING=OFF"
130+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DCUB_ENABLE_MINIMAL_TESTING=ON"
131+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_HOST_SYSTEM=CPP"
132+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_DEVICE_SYSTEM=CUDA"
133+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_CPP_DIALECT=14"
134+
CMAKE_FLAGS="${CMAKE_FLAGS:+${CMAKE_FLAGS} }-DTHRUST_AUTO_DETECT_COMPUTE_ARCHS=ON"
135+
;;
136+
esac
137+
138+
if [[ -n "${@}" ]]; then
139+
CMAKE_BUILD_FLAGS="${CMAKE_BUILD_FLAGS:+${CMAKE_BUILD_FLAGS} }${@}"
140+
fi
80141

81-
if [ "${BUILD_TYPE}" == "cpu" ]; then
82-
CTEST_FLAGS="${CTEST_FLAGS} -E ^cub|^thrust.*cuda"
142+
CTEST_FLAGS="${CTEST_FLAGS:+${CTEST_FLAGS} }--output-on-failure"
143+
144+
if [[ "${BUILD_TYPE}" == "cpu" ]]; then
145+
CTEST_FLAGS="${CTEST_FLAGS:+${CTEST_FLAGS} }-E ^cub|^thrust.*cuda"
83146
fi
84147

85-
if [ ! -z "${@}" ]; then
86-
CTEST_FLAGS="${CTEST_FLAGS} -R ^${@}$"
148+
if [[ -n "${@}" ]]; then
149+
CTEST_FLAGS="${CTEST_FLAGS:+${CTEST_FLAGS} }-R ^${@}$"
87150
fi
88151

152+
# Export variables so they'll show up in the logs when we report the environment.
153+
export COVERAGE_PLAN
154+
export CMAKE_FLAGS
155+
export CMAKE_BUILD_FLAGS
156+
export CTEST_FLAGS
157+
89158
################################################################################
90159
# ENVIRONMENT - Configure and print out information about the environment.
91160
################################################################################
92161

162+
logger "Determine system topology..."
163+
164+
# Set `${PARALLEL_LEVEL}` if it is unset; otherwise, this just reports the
165+
# system topology.
166+
source ${WORKSPACE}/ci/common/determine_build_parallelism.bash ${DETERMINE_PARALLELISM_FLAGS}
167+
93168
logger "Get environment..."
94169
env
95170

@@ -102,15 +177,20 @@ ${CUDACXX} --version
102177
################################################################################
103178

104179
logger "Configure Thrust and CUB..."
180+
echo cmake .. ${CMAKE_FLAGS}
105181
cmake .. ${CMAKE_FLAGS}
106182

107183
logger "Build Thrust and CUB..."
108-
cmake --build . ${CMAKE_BUILD_FLAGS}
184+
# ${PARALLEL_LEVEL} needs to be passed after we run
185+
# determine_build_parallelism.bash, so it can't be part of ${CMAKE_BUILD_FLAGS}.
186+
echo cmake --build . ${CMAKE_BUILD_FLAGS} -j ${PARALLEL_LEVEL}
187+
cmake --build . ${CMAKE_BUILD_FLAGS} -j ${PARALLEL_LEVEL}
109188

110189
################################################################################
111190
# TEST - Run Thrust and CUB examples and tests.
112191
################################################################################
113192

114193
logger "Test Thrust and CUB..."
194+
echo ctest ${CTEST_FLAGS}
115195
ctest ${CTEST_FLAGS}
116196

0 commit comments

Comments
 (0)