diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 5eb00c4aba0f..5ddd34893b50 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -16,10 +16,15 @@ jobs:
matrix:
java: [ '1.8', '11' ]
hadoop: [ 'hadoop-2.7', 'hadoop-3.2' ]
+ hive: [ 'hive-1.2', 'hive-2.3' ]
exclude:
- java: '11'
hadoop: 'hadoop-2.7'
- name: Build Spark with JDK ${{ matrix.java }} and ${{ matrix.hadoop }}
+ - java: '11'
+ hive: 'hive-1.2'
+ - hadoop: 'hadoop-3.2'
+ hive: 'hive-1.2'
+ name: Build Spark - JDK${{ matrix.java }}/${{ matrix.hadoop }}/${{ matrix.hive }}
steps:
- uses: actions/checkout@master
@@ -44,7 +49,7 @@ jobs:
run: |
export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
export MAVEN_CLI_OPTS="--no-transfer-progress"
- ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install
+ ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install
rm -rf ~/.m2/repository/org/apache/spark
diff --git a/appveyor.yml b/appveyor.yml
index c3baa1f82a39..1c8ecede4c1d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -54,7 +54,7 @@ install:
build_script:
# '-Djna.nosys=true' is required to avoid kernel32.dll load failure.
# See SPARK-28759.
- - cmd: mvn -DskipTests -Psparkr -Phive -Djna.nosys=true package
+ - cmd: mvn -DskipTests -Psparkr -Phive -Phive-1.2 -Djna.nosys=true package
environment:
NOT_CRAN: true
diff --git a/dev/run-tests.py b/dev/run-tests.py
index fc8b7251a85f..2d52ead06a04 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -283,8 +283,8 @@ def get_hadoop_profiles(hadoop_version):
"""
sbt_maven_hadoop_profiles = {
- "hadoop2.7": ["-Phadoop-2.7"],
- "hadoop3.2": ["-Phadoop-3.2"],
+ "hadoop2.7": ["-Phadoop-2.7", "-Phive-1.2"],
+ "hadoop3.2": ["-Phadoop-3.2", "-Phive-2.3"],
}
if hadoop_version in sbt_maven_hadoop_profiles:
diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh
index cc0292e9c2ea..7d5725aaf137 100755
--- a/dev/test-dependencies.sh
+++ b/dev/test-dependencies.sh
@@ -67,15 +67,20 @@ $MVN -q versions:set -DnewVersion=$TEMP_VERSION -DgenerateBackupPoms=false > /de
# Generate manifests for each Hadoop profile:
for HADOOP_PROFILE in "${HADOOP_PROFILES[@]}"; do
+ if [[ $HADOOP_PROFILE == **hadoop-3** ]]; then
+ HIVE_PROFILE=hive-2.3
+ else
+ HIVE_PROFILE=hive-1.2
+ fi
echo "Performing Maven install for $HADOOP_PROFILE"
- $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE jar:jar jar:test-jar install:install clean -q
+ $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE jar:jar jar:test-jar install:install clean -q
echo "Performing Maven validate for $HADOOP_PROFILE"
- $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE validate -q
+ $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE validate -q
echo "Generating dependency manifest for $HADOOP_PROFILE"
mkdir -p dev/pr-deps
- $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE dependency:build-classpath -pl assembly -am \
+ $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE dependency:build-classpath -pl assembly -am \
| grep "Dependencies classpath:" -A 1 \
| tail -n 1 | tr ":" "\n" | rev | cut -d "/" -f 1 | rev | sort \
| grep -v spark > dev/pr-deps/spark-deps-$HADOOP_PROFILE
diff --git a/pom.xml b/pom.xml
index 44593b78c9a0..a9b544bba158 100644
--- a/pom.xml
+++ b/pom.xml
@@ -128,19 +128,19 @@
3.4.14
2.7.1
0.4.2
- org.spark-project.hive
-
+ org.apache.hive
+ core
- 1.2.1.spark2
+ 2.3.6
2.3.6
- 1.2.1
+ 2.3.5
2.3.1
10.12.1.1
1.10.1
1.5.7
- nohive
+
com.twitter
1.6.0
9.4.18.v20190429
@@ -181,7 +181,7 @@
3.8.1
2.6.2
- 3.2.10
+ 4.1.17
3.0.15
2.29
2.10.5
@@ -228,7 +228,7 @@
-->
compile
compile
- ${hive.deps.scope}
+ provided
compile
compile
test
@@ -2921,16 +2921,27 @@
3.2.0
2.13.0
- org.apache.hive
- core
- ${hive23.version}
- 2.3.5
-
- provided
-
- 4.1.17
+
+
+
+ hive-1.2
+
+ org.spark-project.hive
+
+
+ 1.2.1.spark2
+
+ 1.2.1
+ ${hive.deps.scope}
+ nohive
+ 3.2.10
+
+
+
+
+ hive-2.3
+
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 882cabd34707..69961acd510d 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -209,7 +209,7 @@
- hadoop-3.2
+ hive-2.3
${hive.group}