diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 index b2c1777ee2ad..a2b603b33c0e 100644 --- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3 @@ -3,7 +3,6 @@ JLargeArrays-1.5.jar JTransforms-3.1.jar RoaringBitmap-0.7.45.jar ST4-4.0.4.jar -accessors-smart-1.2.jar activation-1.1.1.jar aircompressor-0.10.jar algebra_2.12-2.0.0-M2.jar @@ -11,6 +10,10 @@ antlr-runtime-3.5.2.jar antlr4-runtime-4.7.1.jar aopalliance-1.0.jar aopalliance-repackaged-2.5.0.jar +apacheds-i18n-2.0.0-M15.jar +apacheds-kerberos-codec-2.0.0-M15.jar +api-asn1-api-1.0.0-M20.jar +api-util-1.0.0-M20.jar arpack_combined_all-0.1.jar arrow-format-0.15.1.jar arrow-memory-0.15.1.jar @@ -32,10 +35,10 @@ commons-codec-1.10.jar commons-collections-3.2.2.jar commons-compiler-3.0.15.jar commons-compress-1.8.1.jar -commons-configuration2-2.1.1.jar +commons-configuration-1.6.jar commons-crypto-1.0.0.jar -commons-daemon-1.0.13.jar commons-dbcp-1.4.jar +commons-digester-1.8.jar commons-httpclient-3.1.jar commons-io-2.4.jar commons-lang-2.6.jar @@ -47,37 +50,35 @@ commons-pool-1.5.4.jar commons-text-1.6.jar compress-lzf-1.0.3.jar core-1.1.2.jar -curator-client-2.13.0.jar -curator-framework-2.13.0.jar -curator-recipes-2.13.0.jar +curator-client-2.7.1.jar +curator-framework-2.7.1.jar +curator-recipes-2.7.1.jar datanucleus-api-jdo-4.2.4.jar datanucleus-core-4.1.17.jar datanucleus-rdbms-4.1.19.jar derby-10.12.1.1.jar -dnsjava-2.1.7.jar dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar -ehcache-3.3.1.jar flatbuffers-java-1.9.0.jar generex-1.0.2.jar -geronimo-jcache_1.0_spec-1.0-alpha-1.jar gson-2.2.4.jar guava-14.0.1.jar -guice-4.0.jar -guice-servlet-4.0.jar -hadoop-annotations-3.2.0.jar -hadoop-auth-3.2.0.jar -hadoop-client-3.2.0.jar -hadoop-common-3.2.0.jar -hadoop-hdfs-client-3.2.0.jar -hadoop-mapreduce-client-common-3.2.0.jar -hadoop-mapreduce-client-core-3.2.0.jar -hadoop-mapreduce-client-jobclient-3.2.0.jar -hadoop-yarn-api-3.2.0.jar -hadoop-yarn-client-3.2.0.jar -hadoop-yarn-common-3.2.0.jar -hadoop-yarn-registry-3.2.0.jar -hadoop-yarn-server-common-3.2.0.jar -hadoop-yarn-server-web-proxy-3.2.0.jar +guice-3.0.jar +guice-servlet-3.0.jar +hadoop-annotations-2.7.4.jar +hadoop-auth-2.7.4.jar +hadoop-client-2.7.4.jar +hadoop-common-2.7.4.jar +hadoop-hdfs-2.7.4.jar +hadoop-mapreduce-client-app-2.7.4.jar +hadoop-mapreduce-client-common-2.7.4.jar +hadoop-mapreduce-client-core-2.7.4.jar +hadoop-mapreduce-client-jobclient-2.7.4.jar +hadoop-mapreduce-client-shuffle-2.7.4.jar +hadoop-yarn-api-2.7.4.jar +hadoop-yarn-client-2.7.4.jar +hadoop-yarn-common-2.7.4.jar +hadoop-yarn-server-common-2.7.4.jar +hadoop-yarn-server-web-proxy-2.7.4.jar hive-beeline-2.3.6.jar hive-cli-2.3.6.jar hive-common-2.3.6.jar @@ -96,7 +97,7 @@ hive-vector-code-gen-2.3.6.jar hk2-api-2.5.0.jar hk2-locator-2.5.0.jar hk2-utils-2.5.0.jar -htrace-core4-4.1.0-incubating.jar +htrace-core-3.1.0-incubating.jar httpclient-4.5.6.jar httpcore-4.4.10.jar istack-commons-runtime-3.0.8.jar @@ -106,12 +107,12 @@ jackson-core-2.10.0.jar jackson-core-asl-1.9.13.jar jackson-databind-2.10.0.jar jackson-dataformat-yaml-2.10.0.jar -jackson-jaxrs-base-2.9.5.jar -jackson-jaxrs-json-provider-2.9.5.jar +jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar jackson-module-jaxb-annotations-2.10.0.jar jackson-module-paranamer-2.10.0.jar jackson-module-scala_2.12-2.10.0.jar +jackson-xc-1.9.13.jar jakarta.activation-api-1.2.1.jar jakarta.annotation-api-1.3.4.jar jakarta.inject-2.5.0.jar @@ -123,9 +124,8 @@ javax.inject-1.jar javax.jdo-3.2.0-m3.jar javax.servlet-api-3.1.0.jar javolution-5.5.1.jar -jaxb-api-2.2.11.jar +jaxb-api-2.2.2.jar jaxb-runtime-2.3.2.jar -jcip-annotations-1.0-1.jar jcl-over-slf4j-1.7.16.jar jdo-api-3.0.1.jar jersey-client-2.29.jar @@ -135,14 +135,14 @@ jersey-container-servlet-core-2.29.jar jersey-hk2-2.29.jar jersey-media-jaxb-2.29.jar jersey-server-2.29.jar -jetty-webapp-9.4.18.v20190429.jar -jetty-xml-9.4.18.v20190429.jar +jetty-6.1.26.jar +jetty-sslengine-6.1.26.jar +jetty-util-6.1.26.jar jline-2.14.6.jar joda-time-2.10.5.jar jodd-core-3.5.2.jar jpam-1.1.jar json-1.8.jar -json-smart-2.3.jar json4s-ast_2.12-3.6.6.jar json4s-core_2.12-3.6.6.jar json4s-jackson_2.12-3.6.6.jar @@ -151,20 +151,6 @@ jsp-api-2.1.jar jsr305-3.0.0.jar jta-1.1.jar jul-to-slf4j-1.7.16.jar -kerb-admin-1.0.1.jar -kerb-client-1.0.1.jar -kerb-common-1.0.1.jar -kerb-core-1.0.1.jar -kerb-crypto-1.0.1.jar -kerb-identity-1.0.1.jar -kerb-server-1.0.1.jar -kerb-simplekdc-1.0.1.jar -kerb-util-1.0.1.jar -kerby-asn1-1.0.1.jar -kerby-config-1.0.1.jar -kerby-pkix-1.0.1.jar -kerby-util-1.0.1.jar -kerby-xdr-1.0.1.jar kryo-shaded-4.0.2.jar kubernetes-client-4.6.1.jar kubernetes-model-4.6.1.jar @@ -184,12 +170,9 @@ metrics-jmx-4.1.1.jar metrics-json-4.1.1.jar metrics-jvm-4.1.1.jar minlog-1.3.0.jar -mssql-jdbc-6.2.1.jre7.jar netty-all-4.1.42.Final.jar -nimbus-jose-jwt-4.41.1.jar objenesis-2.5.1.jar okapi-shade-0.4.2.jar -okhttp-2.7.5.jar okhttp-3.12.0.jar okio-1.15.0.jar opencsv-2.3.jar @@ -208,7 +191,6 @@ parquet-jackson-1.10.1.jar protobuf-java-2.5.0.jar py4j-0.10.8.1.jar pyrolite-4.30.jar -re2j-1.1.jar scala-collection-compat_2.12-2.1.1.jar scala-compiler-2.12.10.jar scala-library-2.12.10.jar @@ -225,17 +207,17 @@ spire-macros_2.12-0.17.0-M1.jar spire-platform_2.12-0.17.0-M1.jar spire-util_2.12-0.17.0-M1.jar spire_2.12-0.17.0-M1.jar +stax-api-1.0-2.jar stax-api-1.0.1.jar -stax2-api-3.1.4.jar stream-2.9.6.jar super-csv-2.2.0.jar -token-provider-1.0.1.jar transaction-api-1.1.jar univocity-parsers-2.8.3.jar validation-api-2.0.1.Final.jar velocity-1.5.jar -woodstox-core-5.0.3.jar xbean-asm7-shaded-4.15.jar +xercesImpl-2.9.1.jar +xmlenc-0.52.jar xz-1.5.jar zjsonpatch-0.3.0.jar zookeeper-3.4.14.jar diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala index 6e97c37af7df..e64223295305 100644 --- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala +++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.kafka010 -import java.io.{File, IOException} +import java.io.{File, IOException, PrintWriter} import java.net.{InetAddress, InetSocketAddress} import java.nio.charset.StandardCharsets import java.util.{Collections, Properties, UUID} @@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit import javax.security.auth.login.Configuration import scala.collection.JavaConverters._ +import scala.io.Source import scala.util.Random import com.google.common.io.Files @@ -136,9 +137,33 @@ class KafkaTestUtils( kdcConf.setProperty(MiniKdc.DEBUG, "true") kdc = new MiniKdc(kdcConf, kdcDir) kdc.start() + rewriteKrb5Conf() kdcReady = true } + private def rewriteKrb5Conf(): Unit = { + val krb5Conf = Source.fromFile(kdc.getKrb5conf, "UTF-8").getLines() + val rewriteKrb5Conf = krb5Conf.map(s => + if (s.contains("libdefaults")) { + val addedConfig = + addedKrb5Config("default_tkt_enctypes", "aes128-cts-hmac-sha1-96") + + addedKrb5Config("default_tgs_enctypes", "aes128-cts-hmac-sha1-96") + s + addedConfig + } else { + s + }) + kdc.getKrb5conf.delete() + val writer = new PrintWriter(kdc.getKrb5conf) + // scalastyle:off + rewriteKrb5Conf.foreach(writer.println) + // scalastyle:on + writer.close() + } + + private def addedKrb5Config(key: String, value: String): String = { + System.lineSeparator() + s" $key=$value" + } + private def createKeytabsAndJaasConfigFile(): String = { assert(kdcReady, "KDC should be set up beforehand") val baseDir = Utils.createTempDir() @@ -171,6 +196,7 @@ class KafkaTestUtils( | useKeyTab=true | storeKey=true | useTicketCache=false + | refreshKrb5Config=true | keyTab="${zkServerKeytabFile.getAbsolutePath()}" | principal="$zkServerUser@$realm"; |}; @@ -180,6 +206,7 @@ class KafkaTestUtils( | useKeyTab=true | storeKey=true | useTicketCache=false + | refreshKrb5Config=true | keyTab="${zkClientKeytabFile.getAbsolutePath()}" | principal="$zkClientUser@$realm"; |}; diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index 02734e82ed26..160d446cf422 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -202,7 +202,7 @@ enables store-specific committers. --> - hadoop-3.2 + skip-hadoop-3.2-profile src/hadoop-3/main/scala src/hadoop-3/test/scala diff --git a/pom.xml b/pom.xml index 94214a53c3ac..fe5bae1ae394 100644 --- a/pom.xml +++ b/pom.xml @@ -2918,8 +2918,8 @@ hadoop-3.2 - 3.2.0 - 2.13.0 + 2.7.4 + 2.7.1