diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index b56b74c9a29..c3220e2c144 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -85,137 +85,36 @@ jobs: with: parent-workflow: cpp_extra - docker: - needs: check-labels - name: ${{ matrix.title }} - runs-on: ${{ matrix.runs-on }} - if: >- - needs.check-labels.outputs.force == 'true' || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - include: - - image: alpine-linux-cpp - runs-on: ubuntu-latest - title: AMD64 Alpine Linux - - image: conda-cpp - run-options: >- - -e ARROW_USE_MESON=ON - runs-on: ubuntu-latest - title: AMD64 Ubuntu Meson - # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved - - continue-on-error: true - envs: - - DEBIAN=13 - image: debian-cpp - run-options: >- - -e CMAKE_CXX_STANDARD=23 - runs-on: ubuntu-latest - title: AMD64 Debian C++23 - env: - ARCHERY_DEBUG: 1 - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" + linux-runner: + name: Test instance from runs-on + runs-on: runs-on=${{ github.run_id }}/runner=2cpu-linux-x64 steps: - - name: Checkout Arrow - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@v5 - with: - path: .docker - key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} - restore-keys: extra-${{ matrix.image }}- - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: 3 - - name: Setup Archery - run: python3 -m pip install -e dev/archery[docker] - - name: Execute Docker Build - continue-on-error: ${{ matrix.continue-on-error || false }} - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - ENVS: ${{ toJSON(matrix.envs) }} - run: | - # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes - sudo sysctl -w vm.mmap_rnd_bits=28 - source ci/scripts/util_enable_core_dumps.sh - if [ "${ENVS}" != "null" ]; then - echo "${ENVS}" | jq -r '.[]' | while read env; do - echo "${env}" >> .env - done - fi - archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - msvc-arm64: - needs: check-labels - if: >- - needs.check-labels.outputs.force == 'true' || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - name: ARM64 Windows 11 MSVC - uses: ./.github/workflows/cpp_windows.yml - with: - arch: arm64 - os: windows-11-arm - simd-level: NONE + - run: echo "Hello from x64!" - jni-linux: + cuda-cpp: needs: check-labels - name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }} - runs-on: ${{ matrix.platform.runs-on }} if: >- needs.check-labels.outputs.force == 'true' || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - timeout-minutes: 240 - permissions: - # This is for using GitHub Packages for vcpkg cache - packages: write - strategy: - fail-fast: false - matrix: - platform: - - arch: "amd64" - runs-on: ubuntu-latest - - arch: "arm64v8" - runs-on: ubuntu-24.04-arm + name: CUDA 13.0.2 C++ on Ubuntu 24.04 + runs-on: "runs-on=${{ github.run_id }}/family=g4dn.xlarge/image=ubuntu24-gpu-x64" env: - ARCH: ${{ matrix.platform.arch }} - REPO: ghcr.io/${{ github.repository }}-dev + CUDA: "13.0.2" + UBUNTU: "24.04" steps: + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - name: Ensure Docker is available with GPU support + run: docker run --rm --runtime=nvidia --gpus all ubuntu nvidia-smi - name: Checkout Arrow uses: actions/checkout@v6 with: fetch-depth: 0 submodules: recursive - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@v5 - with: - path: .docker - key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }} - restore-keys: jni-${{ matrix.platform.runs-on }}- - name: Setup Python uses: actions/setup-python@v6 with: @@ -227,10 +126,10 @@ jobs: ARCHERY_DOCKER_USER: ${{ github.actor }} ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite" run: | source ci/scripts/util_enable_core_dumps.sh - archery docker run cpp-jni + archery docker run \ + ubuntu-cuda-cpp - name: Docker Push if: >- success() && @@ -240,209 +139,11 @@ jobs: ARCHERY_DOCKER_USER: ${{ github.actor }} ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} continue-on-error: true - run: archery docker push cpp-jni - - jni-macos: - needs: check-labels - name: JNI macOS - runs-on: macos-14 - if: >- - needs.check-labels.outputs.force == 'true' || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - timeout-minutes: 45 - env: - MACOSX_DEPLOYMENT_TARGET: "14.0" - steps: - - name: Checkout Arrow - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - name: Install dependencies - run: | - brew bundle --file=cpp/Brewfile - # We want to link aws-sdk-cpp statically but Homebrew's - # aws-sdk-cpp provides only shared library. If we have - # Homebrew's aws-sdk-cpp, our build mix Homebrew's - # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's - # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. - brew uninstall aws-sdk-cpp - # We want to use bundled RE2 for static linking. If - # Homebrew's RE2 is installed, its header file may be used. - # We uninstall Homebrew's RE2 to ensure using bundled RE2. - brew uninstall grpc || : # gRPC depends on RE2 - brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too - brew uninstall re2 - # We want to use bundled Protobuf for static linking. If - # Homebrew's Protobuf is installed, its library file may be - # used on test We uninstall Homebrew's Protobuf to ensure using - # bundled Protobuf. - brew uninstall protobuf - - name: Prepare ccache - run: | - echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} - - name: Cache ccache - uses: actions/cache@v5 - with: - path: ccache - key: jni-macos-${{ hashFiles('cpp/**') }} - restore-keys: jni-macos- - - name: CMake - run: | - cmake \ - -S cpp \ - -B cpp.build \ - --preset=ninja-release-jni-macos \ - -DARROW_BUILD_TESTS=ON \ - -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - - name: Build - run: | - cmake --build cpp.build - - name: Install - run: | - cmake --install cpp.build - - name: Test - env: - ARROW_TEST_DATA: ${{ github.workspace }}/testing/data - PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data - run: | - # MinIO is required - exclude_tests="arrow-s3fs-test" - # unstable - exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test" - exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test" - ctest \ - --exclude-regex "${exclude_tests}" \ - --label-regex unittest \ - --output-on-failure \ - --parallel "$(sysctl -n hw.ncpu)" \ - --test-dir "cpp.build" \ - --timeout 300 - - name: Build example - run: | - cmake \ - -S cpp/examples/minimal_build/ \ - -B cpp/examples/minimal_build.build \ - -GNinja \ - -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install - cmake --build cpp/examples/minimal_build.build - cd cpp/examples/minimal_build - ../minimal_build.build/arrow-example - - odbc: - needs: check-labels - name: ODBC - runs-on: windows-2022 - if: >- - needs.check-labels.outputs.force == 'true' || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || - contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++') - timeout-minutes: 240 - env: - ARROW_BUILD_SHARED: ON - ARROW_BUILD_STATIC: OFF - ARROW_BUILD_TESTS: ON - ARROW_BUILD_TYPE: release - ARROW_DEPENDENCY_SOURCE: VCPKG - ARROW_FLIGHT_SQL_ODBC: ON - ARROW_SIMD_LEVEL: AVX2 - CMAKE_CXX_STANDARD: "17" - CMAKE_GENERATOR: Ninja - CMAKE_INSTALL_PREFIX: /usr - VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite' - VCPKG_DEFAULT_TRIPLET: x64-windows - steps: - - name: Disable Crash Dialogs - run: | - reg add ` - "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` - /v DontShowUI ` - /t REG_DWORD ` - /d 1 ` - /f - - name: Checkout Arrow - uses: actions/checkout@v6 - with: - fetch-depth: 0 - submodules: recursive - - name: Download Timezone Database - shell: bash - run: ci/scripts/download_tz_database.sh - - name: Install cmake - shell: bash - run: | - ci/scripts/install_cmake.sh 4.1.2 /usr - - name: Install ccache - shell: bash - run: | - ci/scripts/install_ccache.sh 4.12.1 /usr - - name: Setup ccache - shell: bash - run: | - ci/scripts/ccache_setup.sh - - name: ccache info - id: ccache-info - shell: bash - run: | - echo "cache-dir=$(ccache --get-config cache_dir)" >> $GITHUB_OUTPUT - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ${{ steps.ccache-info.outputs.cache-dir }} - key: cpp-odbc-ccache-windows-x64-${{ hashFiles('cpp/**') }} - restore-keys: cpp-odbc-ccache-windows-x64- - - name: Checkout vcpkg - uses: actions/checkout@v6 - with: - fetch-depth: 0 - path: vcpkg - repository: microsoft/vcpkg - - name: Bootstrap vcpkg - run: | - vcpkg\bootstrap-vcpkg.bat - $VCPKG_ROOT = $(Resolve-Path -LiteralPath "vcpkg").ToString() - Write-Output ${VCPKG_ROOT} | ` - Out-File -FilePath ${Env:GITHUB_PATH} -Encoding utf8 -Append - Write-Output "VCPKG_ROOT=${VCPKG_ROOT}" | ` - Out-File -FilePath ${Env:GITHUB_ENV} -Encoding utf8 -Append - - name: Setup NuGet credentials for vcpkg caching - shell: bash - run: | - $(vcpkg fetch nuget | tail -n 1) \ - sources add \ - -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" \ - -storepasswordincleartext \ - -name "GitHub" \ - -username "$GITHUB_REPOSITORY_OWNER" \ - -password "${{ secrets.GITHUB_TOKEN }}" - $(vcpkg fetch nuget | tail -n 1) \ - setapikey "${{ secrets.GITHUB_TOKEN }}" \ - -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" - - name: Build - shell: cmd - run: | - set VCPKG_ROOT_KEEP=%VCPKG_ROOT% - call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - set VCPKG_ROOT=%VCPKG_ROOT_KEEP% - bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build" - - name: Register Flight SQL ODBC Driver - shell: cmd - run: | - call "cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd" ${{ github.workspace }}\build\cpp\%ARROW_BUILD_TYPE%\arrow_flight_sql_odbc.dll - # GH-48270 TODO: Resolve segementation fault during Arrow library unload - # GH-48269 TODO: Enable Flight & Flight SQL testing in MSVC CI - # GH-48547 TODO: enable ODBC tests after GH-48270 and GH-48269 are resolved. - - # GH-47787 TODO: Build ODBC installer + run: archery docker push ubuntu-cuda-cpp report-extra-cpp: if: github.event_name == 'schedule' && always() needs: - - docker - - jni-linux - - jni-macos - - msvc-arm64 - - odbc + - cuda-cpp uses: ./.github/workflows/report_ci.yml secrets: inherit