From 1e0f23de30984b26e8a911a560d25b9426300d47 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 20 Nov 2019 19:38:48 -0800 Subject: [PATCH 1/4] [SPARK-29981][BUILD] Add hive-1.2/2.3 profiles --- dev/run-tests.py | 4 ++-- dev/test-dependencies.sh | 11 +++++++--- pom.xml | 43 +++++++++++++++++++++++++--------------- sql/hive/pom.xml | 2 +- 4 files changed, 38 insertions(+), 22 deletions(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index fc8b7251a85f..2d52ead06a04 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -283,8 +283,8 @@ def get_hadoop_profiles(hadoop_version): """ sbt_maven_hadoop_profiles = { - "hadoop2.7": ["-Phadoop-2.7"], - "hadoop3.2": ["-Phadoop-3.2"], + "hadoop2.7": ["-Phadoop-2.7", "-Phive-1.2"], + "hadoop3.2": ["-Phadoop-3.2", "-Phive-2.3"], } if hadoop_version in sbt_maven_hadoop_profiles: diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index cc0292e9c2ea..7d5725aaf137 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -67,15 +67,20 @@ $MVN -q versions:set -DnewVersion=$TEMP_VERSION -DgenerateBackupPoms=false > /de # Generate manifests for each Hadoop profile: for HADOOP_PROFILE in "${HADOOP_PROFILES[@]}"; do + if [[ $HADOOP_PROFILE == **hadoop-3** ]]; then + HIVE_PROFILE=hive-2.3 + else + HIVE_PROFILE=hive-1.2 + fi echo "Performing Maven install for $HADOOP_PROFILE" - $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE jar:jar jar:test-jar install:install clean -q + $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE jar:jar jar:test-jar install:install clean -q echo "Performing Maven validate for $HADOOP_PROFILE" - $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE validate -q + $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE validate -q echo "Generating dependency manifest for $HADOOP_PROFILE" mkdir -p dev/pr-deps - $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE dependency:build-classpath -pl assembly -am \ + $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE dependency:build-classpath -pl assembly -am \ | grep "Dependencies classpath:" -A 1 \ | tail -n 1 | tr ":" "\n" | rev | cut -d "/" -f 1 | rev | sort \ | grep -v spark > dev/pr-deps/spark-deps-$HADOOP_PROFILE diff --git a/pom.xml b/pom.xml index 44593b78c9a0..a9b544bba158 100644 --- a/pom.xml +++ b/pom.xml @@ -128,19 +128,19 @@ 3.4.14 2.7.1 0.4.2 - org.spark-project.hive - + org.apache.hive + core - 1.2.1.spark2 + 2.3.6 2.3.6 - 1.2.1 + 2.3.5 2.3.1 10.12.1.1 1.10.1 1.5.7 - nohive + com.twitter 1.6.0 9.4.18.v20190429 @@ -181,7 +181,7 @@ 3.8.1 2.6.2 - 3.2.10 + 4.1.17 3.0.15 2.29 2.10.5 @@ -228,7 +228,7 @@ --> compile compile - ${hive.deps.scope} + provided compile compile test @@ -2921,16 +2921,27 @@ 3.2.0 2.13.0 - org.apache.hive - core - ${hive23.version} - 2.3.5 - - provided - - 4.1.17 + + + + hive-1.2 + + org.spark-project.hive + + + 1.2.1.spark2 + + 1.2.1 + ${hive.deps.scope} + nohive + 3.2.10 + + + + + hive-2.3 + diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 882cabd34707..69961acd510d 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -209,7 +209,7 @@ - hadoop-3.2 + hive-2.3 ${hive.group} From cd2ccbf88fd5a69bfa964942c16bf4524f1c4464 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 20 Nov 2019 20:54:30 -0800 Subject: [PATCH 2/4] Update GitHub Action --- .github/workflows/master.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 5eb00c4aba0f..a317914fb2ac 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -16,10 +16,13 @@ jobs: matrix: java: [ '1.8', '11' ] hadoop: [ 'hadoop-2.7', 'hadoop-3.2' ] + hive: [ 'hive-1.2', 'hive-2.3' ] exclude: - java: '11' hadoop: 'hadoop-2.7' - name: Build Spark with JDK ${{ matrix.java }} and ${{ matrix.hadoop }} + - java: '11' + hive: 'hive-1.2' + name: Build Spark - JDK${{ matrix.java }}/${{ matrix.hadoop }}/${{ matrix.hive }} steps: - uses: actions/checkout@master @@ -44,7 +47,7 @@ jobs: run: | export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" export MAVEN_CLI_OPTS="--no-transfer-progress" - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install rm -rf ~/.m2/repository/org/apache/spark From c9a05a5f251a6e6bcc36da89e2b51bc608715ffe Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 20 Nov 2019 20:57:52 -0800 Subject: [PATCH 3/4] Update AppVeyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index c3baa1f82a39..1c8ecede4c1d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -54,7 +54,7 @@ install: build_script: # '-Djna.nosys=true' is required to avoid kernel32.dll load failure. # See SPARK-28759. - - cmd: mvn -DskipTests -Psparkr -Phive -Djna.nosys=true package + - cmd: mvn -DskipTests -Psparkr -Phive -Phive-1.2 -Djna.nosys=true package environment: NOT_CRAN: true From f93742a311c594cecbf404c495aed23afb22c381 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 20 Nov 2019 21:20:32 -0800 Subject: [PATCH 4/4] Exclude invalid combination --- .github/workflows/master.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index a317914fb2ac..5ddd34893b50 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -22,6 +22,8 @@ jobs: hadoop: 'hadoop-2.7' - java: '11' hive: 'hive-1.2' + - hadoop: 'hadoop-3.2' + hive: 'hive-1.2' name: Build Spark - JDK${{ matrix.java }}/${{ matrix.hadoop }}/${{ matrix.hive }} steps: