Skip to content

Commit 51d1d78

Browse files
kevinyu98kevinyu98
committed
move the test cases
1 parent fe472c8 commit 51d1d78

File tree

4 files changed

+159
-126
lines changed

4 files changed

+159
-126
lines changed

‎sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala‎

Lines changed: 0 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -186,82 +186,6 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
186186
}
187187
}
188188

189-
protected def testORCTableLocation(isConvertMetastore: Boolean): Unit = {
190-
withTempDir { dir =>
191-
val someDF1 = Seq((1, 1, "orc1"), (2, 2, "orc2")).toDF("c1", "c2", "c3").repartition(1)
192-
withTable("tbl1", "tbl2", "tbl3", "tbl4") {
193-
val dataDir = s"${dir.getCanonicalPath}/l3/l2/l1/"
194-
val parentDir = s"${dir.getCanonicalPath}/l3/l2/"
195-
val l3Dir = s"${dir.getCanonicalPath}/l3/"
196-
val wildcardParentDir = new File(s"${dir}/l3/l2/*").toURI
197-
val wildcardL3Dir = new File(s"${dir}/l3/*").toURI
198-
someDF1.write.orc(dataDir)
199-
val parentDirStatement =
200-
s"""
201-
|CREATE EXTERNAL TABLE tbl1(
202-
| c1 int,
203-
| c2 int,
204-
| c3 string)
205-
|STORED AS orc
206-
|LOCATION '${parentDir}'""".stripMargin
207-
sql(parentDirStatement)
208-
val parentDirSqlStatement = s"select * from tbl1"
209-
if (isConvertMetastore) {
210-
checkAnswer(sql(parentDirSqlStatement), Nil)
211-
} else {
212-
checkAnswer(sql(parentDirSqlStatement),
213-
(1 to 2).map(i => Row(i, i, s"orc$i")))
214-
}
215-
216-
val l3DirStatement =
217-
s"""
218-
|CREATE EXTERNAL TABLE tbl2(
219-
| c1 int,
220-
| c2 int,
221-
| c3 string)
222-
|STORED AS orc
223-
|LOCATION '${l3Dir}'""".stripMargin
224-
sql(l3DirStatement)
225-
val l3DirSqlStatement = s"select * from tbl2"
226-
if (isConvertMetastore) {
227-
checkAnswer(sql(l3DirSqlStatement), Nil)
228-
} else {
229-
checkAnswer(sql(l3DirSqlStatement),
230-
(1 to 2).map(i => Row(i, i, s"orc$i")))
231-
}
232-
233-
val wildcardStatement =
234-
s"""
235-
|CREATE EXTERNAL TABLE tbl3(
236-
| c1 int,
237-
| c2 int,
238-
| c3 string)
239-
|STORED AS orc
240-
|LOCATION '$wildcardParentDir'""".stripMargin
241-
sql(wildcardStatement)
242-
val wildcardSqlStatement = s"select * from tbl3"
243-
if (isConvertMetastore) {
244-
checkAnswer(sql(wildcardSqlStatement),
245-
(1 to 2).map(i => Row(i, i, s"orc$i")))
246-
} else {
247-
checkAnswer(sql(wildcardSqlStatement), Nil)
248-
}
249-
250-
val wildcardL3Statement =
251-
s"""
252-
|CREATE EXTERNAL TABLE tbl4(
253-
| c1 int,
254-
| c2 int,
255-
| c3 string)
256-
|STORED AS orc
257-
|LOCATION '$wildcardL3Dir'""".stripMargin
258-
sql(wildcardL3Statement)
259-
val wildcardL3SqlStatement = s"select * from tbl4"
260-
checkAnswer(sql(wildcardL3SqlStatement), Nil)
261-
}
262-
}
263-
}
264-
265189
test("create temporary orc table") {
266190
checkAnswer(sql("SELECT COUNT(*) FROM normal_orc_source"), Row(10))
267191

‎sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import org.apache.spark.util.Utils
3232
class HiveParquetSourceSuite extends ParquetPartitioningTest {
3333
import testImplicits._
3434
import spark._
35+
import java.io.IOException
3536

3637
override def beforeAll(): Unit = {
3738
super.beforeAll()
@@ -222,4 +223,66 @@ class HiveParquetSourceSuite extends ParquetPartitioningTest {
222223
assert(df4.columns === Array("str", "max_int"))
223224
}
224225
}
226+
227+
test("SPARK-25993 CREATE EXTERNAL TABLE with subdirectories") {
228+
withTempPath { path =>
229+
withTable("tbl1", "tbl2", "tbl3") {
230+
val someDF1 = Seq((1, 1, "parq1"), (2, 2, "parq2")).
231+
toDF("c1", "c2", "c3").repartition(1)
232+
val dataDir = s"${path.getCanonicalPath}/l3/l2/l1/"
233+
val parentDir = s"${path.getCanonicalPath}/l3/l2/"
234+
val wildcardParentDir = new File(s"${path}/l3/l2/*").toURI
235+
val wildcardL3Dir = new File(s"${path}/l3/*").toURI
236+
someDF1.write.parquet(dataDir)
237+
val parentDirStatement =
238+
s"""
239+
|CREATE EXTERNAL TABLE tbl1(
240+
| c1 int,
241+
| c2 int,
242+
| c3 string)
243+
|STORED AS parquet
244+
|LOCATION '${parentDir}'""".stripMargin
245+
sql(parentDirStatement)
246+
val wildcardStatement =
247+
s"""
248+
|CREATE EXTERNAL TABLE tbl2(
249+
| c1 int,
250+
| c2 int,
251+
| c3 string)
252+
|STORED AS parquet
253+
|LOCATION '${wildcardParentDir}'""".stripMargin
254+
sql(wildcardStatement)
255+
val wildcardL3Statement =
256+
s"""
257+
|CREATE EXTERNAL TABLE tbl3(
258+
| c1 int,
259+
| c2 int,
260+
| c3 string)
261+
|STORED AS parquet
262+
|LOCATION '${wildcardL3Dir}'""".stripMargin
263+
sql(wildcardL3Statement)
264+
265+
Seq("true", "false").foreach { parquetConversion =>
266+
withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> parquetConversion) {
267+
if (parquetConversion == "true") {
268+
checkAnswer(sql("select * from tbl1"), Nil)
269+
checkAnswer(sql("select * from tbl2"),
270+
(1 to 2).map(i => Row(i, i, s"parq$i")))
271+
checkAnswer(sql("select * from tbl3"), Nil)
272+
} else {
273+
Seq("select * from tbl1", "select * from tbl2", "select * from tbl3").foreach {
274+
sqlStmt =>
275+
try {
276+
sql(sqlStmt)
277+
} catch {
278+
case e: IOException =>
279+
assert(e.getMessage().contains("java.io.IOException: Not a file"))
280+
}
281+
}
282+
}
283+
}
284+
}
285+
}
286+
}
287+
}
225288
}

‎sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala‎

Lines changed: 0 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -2370,51 +2370,4 @@ class HiveDDLSuite
23702370
))
23712371
}
23722372
}
2373-
2374-
test("SPARK-25993 Add test cases for resolution of Parquet table location") {
2375-
withTempPath { path =>
2376-
val someDF1 = Seq((1, 1, "parq1"), (2, 2, "parq2")).toDF("c1", "c2", "c3").repartition(1)
2377-
withTable("tbl1", "tbl2", "tbl3") {
2378-
val dataDir = s"${path.getCanonicalPath}/l3/l2/l1/"
2379-
val parentDir = s"${path.getCanonicalPath}/l3/l2/"
2380-
val l3Dir = s"${path.getCanonicalPath}/l3/"
2381-
val wildcardParentDir = new File(s"${path}/l3/l2/*").toURI
2382-
val wildcardL3Dir = new File(s"${path}/l3/*").toURI
2383-
someDF1.write.parquet(dataDir)
2384-
val parentDirStatement =
2385-
s"""
2386-
|CREATE EXTERNAL TABLE tbl1(
2387-
| c1 int,
2388-
| c2 int,
2389-
| c3 string)
2390-
|STORED AS parquet
2391-
|LOCATION '${parentDir}'""".stripMargin
2392-
sql(parentDirStatement)
2393-
checkAnswer(sql("select * from tbl1"), Nil)
2394-
2395-
val wildcardStatement =
2396-
s"""
2397-
|CREATE EXTERNAL TABLE tbl2(
2398-
| c1 int,
2399-
| c2 int,
2400-
| c3 string)
2401-
|STORED AS parquet
2402-
|LOCATION '${wildcardParentDir}'""".stripMargin
2403-
sql(wildcardStatement)
2404-
checkAnswer(sql("select * from tbl2"),
2405-
(1 to 2).map(i => Row(i, i, s"parq$i")))
2406-
2407-
val wildcardL3Statement =
2408-
s"""
2409-
|CREATE EXTERNAL TABLE tbl3(
2410-
| c1 int,
2411-
| c2 int,
2412-
| c3 string)
2413-
|STORED AS parquet
2414-
|LOCATION '${wildcardL3Dir}'""".stripMargin
2415-
sql(wildcardL3Statement)
2416-
checkAnswer(sql("select * from tbl3"), Nil)
2417-
}
2418-
}
2419-
}
24202373
}

‎sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala‎

Lines changed: 96 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,11 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
7373
sql(
7474
s"""
7575
|CREATE TABLE $tableName
76+
7677
|USING org.apache.spark.sql.hive.orc
7778
|OPTIONS (
78-
| PATH '${new File(orcTableAsDir.getAbsolutePath).toURI}'
79+
| PATH '${new File(orcTableAsDir.getAbsolutePath
80+
).toURI}'
7981
|)
8082
""".stripMargin)
8183

@@ -191,10 +193,101 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
191193
}
192194
}
193195

194-
test("SPARK-25993 Add test cases for resolution of ORC table location") {
196+
test("SPARK-25993 CREATE EXTERNAL TABLE with subdirectories") {
195197
Seq(true, false).foreach { convertMetastore =>
196198
withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> s"$convertMetastore") {
197-
testORCTableLocation(convertMetastore)
199+
withTempDir { dir =>
200+
val dataDir = new File(s"${dir.getCanonicalPath}/l3/l2/l1/").toURI
201+
val parentDir = s"${dir.getCanonicalPath}/l3/l2/"
202+
val l3Dir = s"${dir.getCanonicalPath}/l3/"
203+
val wildcardParentDir = new File(s"${dir}/l3/l2/*").toURI
204+
val wildcardL3Dir = new File(s"${dir}/l3/*").toURI
205+
206+
try {
207+
hiveClient.runSqlHive("USE default")
208+
hiveClient.runSqlHive(
209+
"""
210+
|CREATE EXTERNAL TABLE hive_orc(
211+
| C1 INT,
212+
| C2 INT,
213+
| C3 STRING)
214+
|STORED AS orc""".stripMargin)
215+
// Hive throws an exception if I assign the location in the create table statement.
216+
hiveClient.runSqlHive(
217+
s"ALTER TABLE hive_orc SET LOCATION '$dataDir'")
218+
hiveClient.runSqlHive(
219+
"""
220+
|INSERT INTO TABLE hive_orc
221+
|VALUES (1, 1, 'orc1'), (2, 2, 'orc2')""".stripMargin)
222+
223+
withTable("tbl1", "tbl2", "tbl3", "tbl4") {
224+
val parentDirStatement =
225+
s"""
226+
|CREATE EXTERNAL TABLE tbl1(
227+
| c1 int,
228+
| c2 int,
229+
| c3 string)
230+
|STORED AS orc
231+
|LOCATION '${parentDir}'""".stripMargin
232+
sql(parentDirStatement)
233+
val parentDirSqlStatement = s"select * from tbl1"
234+
if (convertMetastore) {
235+
checkAnswer(sql(parentDirSqlStatement), Nil)
236+
} else {
237+
checkAnswer(sql(parentDirSqlStatement),
238+
(1 to 2).map(i => Row(i, i, s"orc$i")))
239+
}
240+
241+
val l3DirStatement =
242+
s"""
243+
|CREATE EXTERNAL TABLE tbl2(
244+
| c1 int,
245+
| c2 int,
246+
| c3 string)
247+
|STORED AS orc
248+
|LOCATION '${l3Dir}'""".stripMargin
249+
sql(l3DirStatement)
250+
val l3DirSqlStatement = s"select * from tbl2"
251+
if (convertMetastore) {
252+
checkAnswer(sql(l3DirSqlStatement), Nil)
253+
} else {
254+
checkAnswer(sql(l3DirSqlStatement),
255+
(1 to 2).map(i => Row(i, i, s"orc$i")))
256+
}
257+
258+
val wildcardStatement =
259+
s"""
260+
|CREATE EXTERNAL TABLE tbl3(
261+
| c1 int,
262+
| c2 int,
263+
| c3 string)
264+
|STORED AS orc
265+
|LOCATION '$wildcardParentDir'""".stripMargin
266+
sql(wildcardStatement)
267+
val wildcardSqlStatement = s"select * from tbl3"
268+
if (convertMetastore) {
269+
checkAnswer(sql(wildcardSqlStatement),
270+
(1 to 2).map(i => Row(i, i, s"orc$i")))
271+
} else {
272+
checkAnswer(sql(wildcardSqlStatement), Nil)
273+
}
274+
275+
val wildcardL3Statement =
276+
s"""
277+
|CREATE EXTERNAL TABLE tbl4(
278+
| c1 int,
279+
| c2 int,
280+
| c3 string)
281+
|STORED AS orc
282+
|LOCATION '$wildcardL3Dir'""".stripMargin
283+
sql(wildcardL3Statement)
284+
val wildcardL3SqlStatement = s"select * from tbl4"
285+
checkAnswer(sql(wildcardL3SqlStatement), Nil)
286+
}
287+
} finally {
288+
hiveClient.runSqlHive("DROP TABLE IF EXISTS hive_orc")
289+
}
290+
}
198291
}
199292
}
200293
}

0 commit comments

Comments
 (0)