[SPARK-30005][INFRA] Update test-dependencies.sh to check hive-1.2/2.3 profile

### What changes were proposed in this pull request?

This PR aims to update `test-dependencies.sh` to validate all available `Hadoop/Hive` combination.

### Why are the changes needed?

Previously, we have been checking only `Hadoop2.7/Hive1.2` and `Hadoop3.2/Hive2.3`.
We need to validate `Hadoop2.7/Hive2.3` additionally for Apache Spark 3.0.

### Does this PR introduce any user-facing change?

No. (This is a dev-only change).

### How was this patch tested?

Pass the GitHub Action (Linter) with the newly updated manifest because this is only dependency check.

Closes #26646 from dongjoon-hyun/SPARK-30005.

Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: dongjoon-hyun

dev/deps/spark-deps-hadoop-2.7 dev/deps/spark-deps-hadoop-2.7-hive-1.2dev/deps/spark-deps-hadoop-2.7 renamed to dev/deps/spark-deps-hadoop-2.7-hive-1.2

@@ -0,0 +1,224 @@
+HikariCP-2.5.1.jar
+JLargeArrays-1.5.jar
+JTransforms-3.1.jar
+RoaringBitmap-0.7.45.jar
+ST4-4.0.4.jar
+activation-1.1.1.jar
+aircompressor-0.10.jar
+algebra_2.12-2.0.0-M2.jar
+antlr-runtime-3.5.2.jar
+antlr4-runtime-4.7.1.jar
+aopalliance-1.0.jar
+aopalliance-repackaged-2.5.0.jar
+apacheds-i18n-2.0.0-M15.jar
+apacheds-kerberos-codec-2.0.0-M15.jar
+api-asn1-api-1.0.0-M20.jar
+api-util-1.0.0-M20.jar
+arpack_combined_all-0.1.jar
+arrow-format-0.15.1.jar
+arrow-memory-0.15.1.jar
+arrow-vector-0.15.1.jar
+audience-annotations-0.5.0.jar
+automaton-1.11-8.jar
+avro-1.8.2.jar
+avro-ipc-1.8.2.jar
+avro-mapred-1.8.2-hadoop2.jar
+bonecp-0.8.0.RELEASE.jar
+breeze-macros_2.12-1.0.jar
+breeze_2.12-1.0.jar
+cats-kernel_2.12-2.0.0-M4.jar
+chill-java-0.9.3.jar
+chill_2.12-0.9.3.jar
+commons-beanutils-1.9.4.jar
+commons-cli-1.2.jar
+commons-codec-1.10.jar
+commons-collections-3.2.2.jar
+commons-compiler-3.0.15.jar
+commons-compress-1.8.1.jar
+commons-configuration-1.6.jar
+commons-crypto-1.0.0.jar
+commons-dbcp-1.4.jar
+commons-digester-1.8.jar
+commons-httpclient-3.1.jar
+commons-io-2.4.jar
+commons-lang-2.6.jar
+commons-lang3-3.8.1.jar
+commons-logging-1.1.3.jar
+commons-math3-3.4.1.jar
+commons-net-3.1.jar
+commons-pool-1.5.4.jar
+commons-text-1.6.jar
+compress-lzf-1.0.3.jar
+core-1.1.2.jar
+curator-client-2.7.1.jar
+curator-framework-2.7.1.jar
+curator-recipes-2.7.1.jar
+datanucleus-api-jdo-4.2.4.jar
+datanucleus-core-4.1.17.jar
+datanucleus-rdbms-4.1.19.jar
+derby-10.12.1.1.jar
+dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
+flatbuffers-java-1.9.0.jar
+generex-1.0.2.jar
+gson-2.2.4.jar
+guava-14.0.1.jar
+guice-3.0.jar
+guice-servlet-3.0.jar
+hadoop-annotations-2.7.4.jar
+hadoop-auth-2.7.4.jar
+hadoop-client-2.7.4.jar
+hadoop-common-2.7.4.jar
+hadoop-hdfs-2.7.4.jar
+hadoop-mapreduce-client-app-2.7.4.jar
+hadoop-mapreduce-client-common-2.7.4.jar
+hadoop-mapreduce-client-core-2.7.4.jar
+hadoop-mapreduce-client-jobclient-2.7.4.jar
+hadoop-mapreduce-client-shuffle-2.7.4.jar
+hadoop-yarn-api-2.7.4.jar
+hadoop-yarn-client-2.7.4.jar
+hadoop-yarn-common-2.7.4.jar
+hadoop-yarn-server-common-2.7.4.jar
+hadoop-yarn-server-web-proxy-2.7.4.jar
+hive-beeline-2.3.6.jar
+hive-cli-2.3.6.jar
+hive-common-2.3.6.jar
+hive-exec-2.3.6-core.jar
+hive-jdbc-2.3.6.jar
+hive-llap-common-2.3.6.jar
+hive-metastore-2.3.6.jar
+hive-serde-2.3.6.jar
+hive-service-rpc-2.3.6.jar
+hive-shims-0.23-2.3.6.jar
+hive-shims-2.3.6.jar
+hive-shims-common-2.3.6.jar
+hive-shims-scheduler-2.3.6.jar
+hive-storage-api-2.6.0.jar
+hive-vector-code-gen-2.3.6.jar
+hk2-api-2.5.0.jar
+hk2-locator-2.5.0.jar
+hk2-utils-2.5.0.jar
+htrace-core-3.1.0-incubating.jar
+httpclient-4.5.6.jar
+httpcore-4.4.10.jar
+istack-commons-runtime-3.0.8.jar
+ivy-2.4.0.jar
+jackson-annotations-2.10.0.jar
+jackson-core-2.10.0.jar
+jackson-core-asl-1.9.13.jar
+jackson-databind-2.10.0.jar
+jackson-dataformat-yaml-2.10.0.jar
+jackson-jaxrs-1.9.13.jar
+jackson-mapper-asl-1.9.13.jar
+jackson-module-jaxb-annotations-2.10.0.jar
+jackson-module-paranamer-2.10.0.jar
+jackson-module-scala_2.12-2.10.0.jar
+jackson-xc-1.9.13.jar
+jakarta.activation-api-1.2.1.jar
+jakarta.annotation-api-1.3.4.jar
+jakarta.inject-2.5.0.jar
+jakarta.ws.rs-api-2.1.5.jar
+jakarta.xml.bind-api-2.3.2.jar
+janino-3.0.15.jar
+javassist-3.22.0-CR2.jar
+javax.inject-1.jar
+javax.jdo-3.2.0-m3.jar
+javax.servlet-api-3.1.0.jar
+javolution-5.5.1.jar
+jaxb-api-2.2.2.jar
+jaxb-runtime-2.3.2.jar
+jcl-over-slf4j-1.7.16.jar
+jdo-api-3.0.1.jar
+jersey-client-2.29.jar
+jersey-common-2.29.jar
+jersey-container-servlet-2.29.jar
+jersey-container-servlet-core-2.29.jar
+jersey-hk2-2.29.jar
+jersey-media-jaxb-2.29.jar
+jersey-server-2.29.jar
+jetty-6.1.26.jar
+jetty-sslengine-6.1.26.jar
+jetty-util-6.1.26.jar
+jline-2.14.6.jar
+joda-time-2.10.5.jar
+jodd-core-3.5.2.jar
+jpam-1.1.jar
+json-1.8.jar
+json4s-ast_2.12-3.6.6.jar
+json4s-core_2.12-3.6.6.jar
+json4s-jackson_2.12-3.6.6.jar
+json4s-scalap_2.12-3.6.6.jar
+jsp-api-2.1.jar
+jsr305-3.0.0.jar
+jta-1.1.jar
+jul-to-slf4j-1.7.16.jar
+kryo-shaded-4.0.2.jar
+kubernetes-client-4.6.1.jar
+kubernetes-model-4.6.1.jar
+kubernetes-model-common-4.6.1.jar
+leveldbjni-all-1.8.jar
+libfb303-0.9.3.jar
+libthrift-0.12.0.jar
+log4j-1.2.17.jar
+logging-interceptor-3.12.0.jar
+lz4-java-1.6.0.jar
+machinist_2.12-0.6.8.jar
+macro-compat_2.12-1.1.1.jar
+mesos-1.4.0-shaded-protobuf.jar
+metrics-core-4.1.1.jar
+metrics-graphite-4.1.1.jar
+metrics-jmx-4.1.1.jar
+metrics-json-4.1.1.jar
+metrics-jvm-4.1.1.jar
+minlog-1.3.0.jar
+netty-all-4.1.42.Final.jar
+objenesis-2.5.1.jar
+okapi-shade-0.4.2.jar
+okhttp-3.12.0.jar
+okio-1.15.0.jar
+opencsv-2.3.jar
+orc-core-1.5.7.jar
+orc-mapreduce-1.5.7.jar
+orc-shims-1.5.7.jar
+oro-2.0.8.jar
+osgi-resource-locator-1.0.3.jar
+paranamer-2.8.jar
+parquet-column-1.10.1.jar
+parquet-common-1.10.1.jar
+parquet-encoding-1.10.1.jar
+parquet-format-2.4.0.jar
+parquet-hadoop-1.10.1.jar
+parquet-jackson-1.10.1.jar
+protobuf-java-2.5.0.jar
+py4j-0.10.8.1.jar
+pyrolite-4.30.jar
+scala-collection-compat_2.12-2.1.1.jar
+scala-compiler-2.12.10.jar
+scala-library-2.12.10.jar
+scala-parser-combinators_2.12-1.1.2.jar
+scala-reflect-2.12.10.jar
+scala-xml_2.12-1.2.0.jar
+shapeless_2.12-2.3.3.jar
+shims-0.7.45.jar
+slf4j-api-1.7.16.jar
+slf4j-log4j12-1.7.16.jar
+snakeyaml-1.24.jar
+snappy-java-1.1.7.3.jar
+spire-macros_2.12-0.17.0-M1.jar
+spire-platform_2.12-0.17.0-M1.jar
+spire-util_2.12-0.17.0-M1.jar
+spire_2.12-0.17.0-M1.jar
+stax-api-1.0-2.jar
+stax-api-1.0.1.jar
+stream-2.9.6.jar
+super-csv-2.2.0.jar
+transaction-api-1.1.jar
+univocity-parsers-2.8.3.jar
+validation-api-2.0.1.Final.jar
+velocity-1.5.jar
+xbean-asm7-shaded-4.15.jar
+xercesImpl-2.9.1.jar
+xmlenc-0.52.jar
+xz-1.5.jar
+zjsonpatch-0.3.0.jar
+zookeeper-3.4.14.jar
+zstd-jni-1.4.3-1.jar

dev/deps/spark-deps-hadoop-2.7-hive-2.3

@@ -31,9 +31,10 @@ export LC_ALL=C
 # NOTE: These should match those in the release publishing script
 HADOOP2_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pkubernetes -Pyarn -Phive"
 MVN="build/mvn"
-HADOOP_PROFILES=(
-    hadoop-2.7
-    hadoop-3.2
+HADOOP_HIVE_PROFILES=(
+    hadoop-2.7-hive-1.2
+    hadoop-2.7-hive-2.3
+    hadoop-3.2-hive-2.3
 )
 
 # We'll switch the version to a temp. one, publish POMs using that new version, then switch back to
@@ -66,24 +67,29 @@ trap reset_version EXIT
 $MVN -q versions:set -DnewVersion=$TEMP_VERSION -DgenerateBackupPoms=false > /dev/null
 
 # Generate manifests for each Hadoop profile:
-for HADOOP_PROFILE in "${HADOOP_PROFILES[@]}"; do
-  if [[ $HADOOP_PROFILE == **hadoop-3** ]]; then
+for HADOOP_HIVE_PROFILE in "${HADOOP_HIVE_PROFILES[@]}"; do
+  if [[ $HADOOP_HIVE_PROFILE == **hadoop-3.2-hive-2.3** ]]; then
+    HADOOP_PROFILE=hadoop-3.2
+    HIVE_PROFILE=hive-2.3
+  elif [[ $HADOOP_HIVE_PROFILE == **hadoop-2.7-hive-2.3** ]]; then
+    HADOOP_PROFILE=hadoop-2.7
     HIVE_PROFILE=hive-2.3
   else
+    HADOOP_PROFILE=hadoop-2.7
     HIVE_PROFILE=hive-1.2
   fi
-  echo "Performing Maven install for $HADOOP_PROFILE"
+  echo "Performing Maven install for $HADOOP_HIVE_PROFILE"
   $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE jar:jar jar:test-jar install:install clean -q
 
-  echo "Performing Maven validate for $HADOOP_PROFILE"
+  echo "Performing Maven validate for $HADOOP_HIVE_PROFILE"
   $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE validate -q
 
-  echo "Generating dependency manifest for $HADOOP_PROFILE"
+  echo "Generating dependency manifest for $HADOOP_HIVE_PROFILE"
   mkdir -p dev/pr-deps
   $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE dependency:build-classpath -pl assembly -am \
     | grep "Dependencies classpath:" -A 1 \
     | tail -n 1 | tr ":" "\n" | rev | cut -d "/" -f 1 | rev | sort \
-    | grep -v spark > dev/pr-deps/spark-deps-$HADOOP_PROFILE
+    | grep -v spark > dev/pr-deps/spark-deps-$HADOOP_HIVE_PROFILE
 done
 
 if [[ $@ == **replace-manifest** ]]; then
@@ -93,13 +99,13 @@ if [[ $@ == **replace-manifest** ]]; then
   exit 0
 fi
 
-for HADOOP_PROFILE in "${HADOOP_PROFILES[@]}"; do
+for HADOOP_HIVE_PROFILE in "${HADOOP_HIVE_PROFILES[@]}"; do
   set +e
   dep_diff="$(
     git diff \
     --no-index \
-    dev/deps/spark-deps-$HADOOP_PROFILE \
-    dev/pr-deps/spark-deps-$HADOOP_PROFILE \
+    dev/deps/spark-deps-$HADOOP_HIVE_PROFILE \
+    dev/pr-deps/spark-deps-$HADOOP_HIVE_PROFILE \
   )"
   set -e
   if [ "$dep_diff" != "" ]; then