Skip to content

[SPARK-24590][BUILD] Make Jenkins tests passed with hadoop 3 profile #21588

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
HyukjinKwon wants to merge 7 commits into from
Closed

[SPARK-24590][BUILD] Make Jenkins tests passed with hadoop 3 profile #21588

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4585d2d
Make Jenkins tests passed with hadoop 3 profile
HyukjinKwon May 28, 2018
fe8bc03
Add Hadoop version assume for Hive_2_1_DDLSuite.scala
HyukjinKwon Jun 19, 2018
dd04f18
Skip conditionally
HyukjinKwon Jun 19, 2018
be6e288
Lazy HiveClient
HyukjinKwon Jun 19, 2018
c4c82ba
Fix all other failures
HyukjinKwon Jun 20, 2018
b9eeef3
Revert the changes needed for testing
HyukjinKwon Jun 20, 2018
a011e50
typo
HyukjinKwon Sep 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix all other failures
  • Loading branch information
@HyukjinKwon
HyukjinKwon committed Sep 27, 2018
commit c4c82ba2e859ea78b51dcb84c67e0d75cac39327
6 changes: 6 additions & 0 deletions sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import java.net.URI
import java.util.Locale

import org.apache.hadoop.fs.Path
import org.apache.hadoop.util.VersionInfo
import org.scalatest.BeforeAndAfterEach

import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
Expand Down Expand Up @@ -2697,6 +2698,11 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
}

test(s"basic DDL using locale tr - caseSensitive $caseSensitive") {
// There seems a bug about dealing with non-ascii compatible characters in file names
// for the usage of File.toPath() in few specific JDKs, which looks updated in HADOOP-12045
// for local file systems specifically. This affects Hadoop 2.8.0+ per the JIRA.
// See https://stackoverflow.com/questions/37409379/invalidpathexception-for-chinese-filename
assume(VersionInfo.getVersion < "2.8.0")
withSQLConf(SQLConf.CASE_SENSITIVE.key -> s"$caseSensitive") {
withLocale("tr") {
val dbName = "DaTaBaSe_I"
Expand Down
2 changes: 2 additions & 0 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ class HiveSparkSubmitSuite
}

test("SPARK-9757 Persist Parquet relation with decimal column") {
assume(VersionInfo.getVersion < "3.0.0", "Only Hive 2.3+ supports Hadoop 3+. See HIVE-16081.")
val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
val args = Seq(
"--class", SPARK_9757.getClass.getName.stripSuffix("$"),
Expand Down Expand Up @@ -252,6 +253,7 @@ class HiveSparkSubmitSuite
}

test("SPARK-16901: set javax.jdo.option.ConnectionURL") {
assume(VersionInfo.getVersion < "3.0.0", "Only Hive 2.3+ supports Hadoop 3+. See HIVE-16081.")
// In this test, we set javax.jdo.option.ConnectionURL and set metastore version to
// 0.13. This test will make sure that javax.jdo.option.ConnectionURL will not be
// overridden by hive's default settings when we create a HiveConf object inside
Expand Down
6 changes: 6 additions & 0 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import java.util.Date
import scala.language.existentials

import org.apache.hadoop.fs.Path
import org.apache.hadoop.util.VersionInfo
import org.apache.parquet.format.converter.ParquetMetadataConverter.NO_FILTER
import org.apache.parquet.hadoop.ParquetFileReader
import org.scalatest.BeforeAndAfterEach
Expand Down Expand Up @@ -397,6 +398,11 @@ class HiveDDLSuite
}

test("create Hive-serde table and view with unicode columns and comment") {
// There seems a bug about dealing with non-ascii compatible characters in file names
// for the usage of File.toPath() in few specific JDKs, which looks updated in HADOOP-12045
// for local file systems specifically. This affects Hadoop 2.8.0+ per the JIRA.
// See https://stackoverflow.com/questions/37409379/invalidpathexception-for-chinese-filename
assume(VersionInfo.getVersion < "2.8.0")
val catalog = spark.sessionState.catalog
val tabName = "tab1"
val viewName = "view1"
Expand Down
16 changes: 8 additions & 8 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,6 @@ class Hive_2_1_DDLSuite extends SparkFunSuite with TestHiveSingleton with Before
new HiveExternalCatalog(sparkConf, hadoopConf)
}

override def afterEach: Unit = {
catalog.listTables("default").foreach { t =>
catalog.dropTable("default", t, true, false)
}
spark.sessionState.catalog.reset()
}

override def afterAll(): Unit = {
try {
catalog = null
Expand All @@ -84,7 +77,14 @@ class Hive_2_1_DDLSuite extends SparkFunSuite with TestHiveSingleton with Before

private def test_2_1(title: String)(func: => Unit): Unit = test(title) {
assume(VersionInfo.getVersion < "3.0.0", "Only Hive 2.3+ supports Hadoop 3+. See HIVE-16081.")
func
try {
func
} finally {
catalog.listTables("default").foreach { t =>
catalog.dropTable("default", t, true, false)
}
spark.sessionState.catalog.reset()
}
}

test_2_1("SPARK-21617: ALTER TABLE for non-compatible DataSource tables") {
Expand Down