Skip to content

Commit fd099bf

Browse files
author
jinxing
committed
[SPARK-22676][FOLLOW-UP] fix code style for test.
1 parent 1e3b876 commit fd099bf

File tree

1 file changed

+73
-69
lines changed

1 file changed

+73
-69
lines changed

‎sql/hive/src/test/scala/org/apache/spark/sql/hive/QueryPartitionSuite.scala‎

Lines changed: 73 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -34,79 +34,83 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl
3434
import spark.implicits._
3535

3636
test("SPARK-5068: query data when path doesn't exist") {
37-
withSQLConf((SQLConf.HIVE_VERIFY_PARTITION_PATH.key, "true")) {
38-
val testData = sparkContext.parallelize(
39-
(1 to 10).map(i => TestData(i, i.toString))).toDF()
40-
testData.createOrReplaceTempView("testData")
41-
42-
val tmpDir = Files.createTempDir()
43-
// create the table for test
44-
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
45-
s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ")
46-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
47-
"SELECT key,value FROM testData")
48-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
49-
"SELECT key,value FROM testData")
50-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='3') " +
51-
"SELECT key,value FROM testData")
52-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='4') " +
53-
"SELECT key,value FROM testData")
54-
55-
// test for the exist path
56-
checkAnswer(sql("select key,value from table_with_partition"),
57-
testData.toDF.collect ++ testData.toDF.collect
58-
++ testData.toDF.collect ++ testData.toDF.collect)
59-
60-
// delete the path of one partition
61-
tmpDir.listFiles
62-
.find { f => f.isDirectory && f.getName().startsWith("ds=") }
63-
.foreach { f => Utils.deleteRecursively(f) }
64-
65-
// test for after delete the path
66-
checkAnswer(sql("select key,value from table_with_partition"),
67-
testData.toDF.collect ++ testData.toDF.collect ++ testData.toDF.collect)
68-
69-
sql("DROP TABLE IF EXISTS table_with_partition")
70-
sql("DROP TABLE IF EXISTS createAndInsertTest")
37+
withSQLConf((SQLConf.HIVE_VERIFY_PARTITION_PATH.key -> "true")) {
38+
withTempView("testData") {
39+
withTable("table_with_partition", "createAndInsertTest") {
40+
withTempPath { tmpDir =>
41+
tmpDir.mkdir()
42+
val testData = sparkContext.parallelize(
43+
(1 to 10).map(i => TestData(i, i.toString))).toDF()
44+
testData.createOrReplaceTempView("testData")
45+
46+
// create the table for test
47+
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
48+
s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ")
49+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
50+
"SELECT key,value FROM testData")
51+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
52+
"SELECT key,value FROM testData")
53+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='3') " +
54+
"SELECT key,value FROM testData")
55+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='4') " +
56+
"SELECT key,value FROM testData")
57+
58+
// test for the exist path
59+
checkAnswer(sql("select key,value from table_with_partition"),
60+
testData.union(testData).union(testData).union(testData).collect)
61+
62+
// delete the path of one partition
63+
tmpDir.listFiles
64+
.find { f => f.isDirectory && f.getName().startsWith("ds=") }
65+
.foreach { f => Utils.deleteRecursively(f) }
66+
67+
// test for after delete the path
68+
checkAnswer(sql("select key,value from table_with_partition"),
69+
testData.union(testData).union(testData).collect)
70+
}
71+
}
72+
}
7173
}
7274
}
7375

7476
test("Replace spark.sql.hive.verifyPartitionPath by spark.files.ignoreMissingFiles") {
75-
withSQLConf((SQLConf.HIVE_VERIFY_PARTITION_PATH.key, "false")) {
76-
sparkContext.conf.set(IGNORE_MISSING_FILES.key, "true")
77-
val testData = sparkContext.parallelize(
78-
(1 to 10).map(i => TestData(i, i.toString))).toDF()
79-
testData.createOrReplaceTempView("testData")
80-
81-
val tmpDir = Files.createTempDir()
82-
// create the table for test
83-
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
84-
s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ")
85-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
86-
"SELECT key,value FROM testData")
87-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
88-
"SELECT key,value FROM testData")
89-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='3') " +
90-
"SELECT key,value FROM testData")
91-
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='4') " +
92-
"SELECT key,value FROM testData")
93-
94-
// test for the exist path
95-
checkAnswer(sql("select key,value from table_with_partition"),
96-
testData.toDF.collect ++ testData.toDF.collect
97-
++ testData.toDF.collect ++ testData.toDF.collect)
98-
99-
// delete the path of one partition
100-
tmpDir.listFiles
101-
.find { f => f.isDirectory && f.getName().startsWith("ds=") }
102-
.foreach { f => Utils.deleteRecursively(f) }
103-
104-
// test for after delete the path
105-
checkAnswer(sql("select key,value from table_with_partition"),
106-
testData.toDF.collect ++ testData.toDF.collect ++ testData.toDF.collect)
107-
108-
sql("DROP TABLE IF EXISTS table_with_partition")
109-
sql("DROP TABLE IF EXISTS createAndInsertTest")
77+
withSQLConf((SQLConf.HIVE_VERIFY_PARTITION_PATH.key -> "false")) {
78+
withTempView("testData") {
79+
withTable("table_with_partition", "createAndInsertTest") {
80+
withTempPath { tmpDir =>
81+
tmpDir.mkdir()
82+
sparkContext.conf.set(IGNORE_MISSING_FILES.key, "true")
83+
val testData = sparkContext.parallelize(
84+
(1 to 10).map(i => TestData(i, i.toString))).toDF()
85+
testData.createOrReplaceTempView("testData")
86+
87+
// create the table for test
88+
sql(s"CREATE TABLE table_with_partition(key int,value string) " +
89+
s"PARTITIONED by (ds string) location '${tmpDir.toURI}' ")
90+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') " +
91+
"SELECT key,value FROM testData")
92+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='2') " +
93+
"SELECT key,value FROM testData")
94+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='3') " +
95+
"SELECT key,value FROM testData")
96+
sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='4') " +
97+
"SELECT key,value FROM testData")
98+
99+
// test for the exist path
100+
checkAnswer(sql("select key,value from table_with_partition"),
101+
testData.union(testData).union(testData).union(testData).collect)
102+
103+
// delete the path of one partition
104+
tmpDir.listFiles
105+
.find { f => f.isDirectory && f.getName().startsWith("ds=") }
106+
.foreach { f => Utils.deleteRecursively(f) }
107+
108+
// test for after delete the path
109+
checkAnswer(sql("select key,value from table_with_partition"),
110+
testData.union(testData).union(testData).collect)
111+
}
112+
}
113+
}
110114
}
111115
}
112116

0 commit comments

Comments
 (0)