Skip to content

[SPARK-22676][FOLLOW-UP] fix code style for test. #21091

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jinxing64 wants to merge 3 commits into from
Closed

[SPARK-22676][FOLLOW-UP] fix code style for test. #21091

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fd099bf
[SPARK-22676][FOLLOW-UP] fix code style for test.
Apr 18, 2018
e9ff4d0
replace withTempPath with withTempDir
Apr 19, 2018
3bf4536
extract a common method for two tests
Apr 19, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
replace withTempPath with withTempDir
  • Loading branch information
jinxing committed Apr 19, 2018
commit e9ff4d0e629dbc02970ba9216476bbccd26ae528
18 changes: 8 additions & 10 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,10 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl
import spark.implicits._

test("SPARK-5068: query data when path doesn't exist") {
withSQLConf((SQLConf.HIVE_VERIFY_PARTITION_PATH.key -> "true")) {
withSQLConf(SQLConf.HIVE_VERIFY_PARTITION_PATH.key -> "true") {
withTempView("testData") {
withTable("table_with_partition", "createAndInsertTest") {
withTempPath { tmpDir =>
tmpDir.mkdir()
withTempDir { tmpDir =>
val testData = sparkContext.parallelize(
(1 to 10).map(i => TestData(i, i.toString))).toDF()
testData.createOrReplaceTempView("testData")
Expand All @@ -57,7 +56,7 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl

// test for the exist path
checkAnswer(sql("select key,value from table_with_partition"),
testData.union(testData).union(testData).union(testData).collect)
testData.union(testData).union(testData).union(testData))

// delete the path of one partition
tmpDir.listFiles
Expand All @@ -66,19 +65,18 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl

// test for after delete the path
checkAnswer(sql("select key,value from table_with_partition"),
testData.union(testData).union(testData).collect)
testData.union(testData).union(testData))
}
}
}
}
}

test("Replace spark.sql.hive.verifyPartitionPath by spark.files.ignoreMissingFiles") {
Copy link
Contributor

@cloud-fan cloud-fan Apr 19, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These 2 tests are exactly same except the config, can we create a common method for it?

Copy link
Author

@jinxing64 jinxing64 Apr 19, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I extracted a common method, check again.

withSQLConf((SQLConf.HIVE_VERIFY_PARTITION_PATH.key -> "false")) {
withSQLConf(SQLConf.HIVE_VERIFY_PARTITION_PATH.key -> "false") {
withTempView("testData") {
withTable("table_with_partition", "createAndInsertTest") {
withTempPath { tmpDir =>
tmpDir.mkdir()
withTempDir { tmpDir =>
sparkContext.conf.set(IGNORE_MISSING_FILES.key, "true")
val testData = sparkContext.parallelize(
(1 to 10).map(i => TestData(i, i.toString))).toDF()
Expand All @@ -98,7 +96,7 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl

// test for the exist path
checkAnswer(sql("select key,value from table_with_partition"),
testData.union(testData).union(testData).union(testData).collect)
testData.union(testData).union(testData).union(testData))

// delete the path of one partition
tmpDir.listFiles
Expand All @@ -107,7 +105,7 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl

// test for after delete the path
checkAnswer(sql("select key,value from table_with_partition"),
testData.union(testData).union(testData).collect)
testData.union(testData).union(testData))
}
}
}
Expand Down