Skip to content

[SPARK-25993][SQL][TESTS] Add test cases for CREATE EXTERNAL TABLE with subdirectories #27130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kevinyu98 wants to merge 5 commits into from
Closed

[SPARK-25993][SQL][TESTS] Add test cases for CREATE EXTERNAL TABLE with subdirectories #27130

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9722230
rebase on spark 3.0
kevinyu98 Jan 8, 2020
9bc32ab
address comments
kevinyu98 Jan 9, 2020
2cf3e26
address comments
kevinyu98 Jan 12, 2020
0bb628f
address comments
kevinyu98 Jan 17, 2020
39f271f
address comments
kevinyu98 Jan 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 122 additions & 0 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSourceSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.spark.sql.hive

import java.io.File
import java.io.IOException

import org.apache.spark.sql.{Row, SaveMode}
import org.apache.spark.sql.catalyst.catalog.HiveTableRelation
Expand Down Expand Up @@ -222,4 +223,125 @@ class HiveParquetSourceSuite extends ParquetPartitioningTest {
assert(df4.columns === Array("str", "max_int"))
}
}

test("SPARK-25993 CREATE EXTERNAL TABLE with subdirectories") {
Seq("true", "false").foreach { parquetConversion =>
withSQLConf(HiveUtils.CONVERT_METASTORE_PARQUET.key -> parquetConversion) {
withTempPath { path =>
withTable("tbl1", "tbl2", "tbl3", "tbl4", "tbl5", "tbl6") {
val someDF1 = Seq((1, 1, "parq1"), (2, 2, "parq2")).
toDF("c1", "c2", "c3").repartition(1)
val someDF2 = Seq((3, 3, "parq3"), (4, 4, "parq4")).
toDF("c1", "c2", "c3").repartition(1)
val someDF3 = Seq((5, 5, "parq5"), (6, 6, "parq6")).
toDF("c1", "c2", "c3").repartition(1)
someDF1.write.parquet(s"${path.getCanonicalPath}/l1/")
someDF2.write.parquet(s"${path.getCanonicalPath}/l1/l2/")
someDF3.write.parquet(s"${path.getCanonicalPath}/l1/l2/l3/")

val topDirStatement =
s"""
|CREATE EXTERNAL TABLE tbl1(
| c1 int,
| c2 int,
| c3 string)
|STORED AS parquet
|LOCATION '${s"${path.getCanonicalPath}"}'""".stripMargin
sql(topDirStatement)
if (parquetConversion == "true") {
checkAnswer(sql("SELECT * FROM tbl1"), Nil)
} else {
val msg = intercept[IOException] {
sql("SELECT * FROM tbl1").show()
}.getMessage
assert(msg.contains("Not a file:"))
}

val l1DirStatement =
s"""
|CREATE EXTERNAL TABLE tbl2(
| c1 int,
| c2 int,
| c3 string)
|STORED AS parquet
|LOCATION '${s"${path.getCanonicalPath}/l1/"}'""".stripMargin
sql(l1DirStatement)
if (parquetConversion == "true") {
checkAnswer(sql("SELECT * FROM tbl2"), (1 to 2).map(i => Row(i, i, s"parq$i")))
} else {
val msg = intercept[IOException] {
sql("SELECT * FROM tbl2").show()
}.getMessage
assert(msg.contains("Not a file:"))
}

val l2DirStatement =
s"""
|CREATE EXTERNAL TABLE tbl3(
| c1 int,
| c2 int,
| c3 string)
|STORED AS parquet
|LOCATION '${s"${path.getCanonicalPath}/l1/l2/"}'""".stripMargin
sql(l2DirStatement)
if (parquetConversion == "true") {
checkAnswer(sql("SELECT * FROM tbl3"), (3 to 4).map(i => Row(i, i, s"parq$i")))
} else {
val msg = intercept[IOException] {
sql("SELECT * FROM tbl3").show()
}.getMessage
assert(msg.contains("Not a file:"))
}

val wildcardTopDirStatement =
s"""
|CREATE EXTERNAL TABLE tbl4(
| c1 int,
| c2 int,
| c3 string)
|STORED AS parquet
|LOCATION '${new File(s"${path}/*").toURI}'""".stripMargin
sql(wildcardTopDirStatement)
if (parquetConversion == "true") {
checkAnswer(sql("SELECT * FROM tbl4"), (1 to 2).map(i => Row(i, i, s"parq$i")))
} else {
val msg = intercept[IOException] {
sql("SELECT * FROM tbl4").show()
}.getMessage
assert(msg.contains("Not a file:"))
}

val wildcardL1DirStatement =
s"""
|CREATE EXTERNAL TABLE tbl5(
| c1 int,
| c2 int,
| c3 string)
|STORED AS parquet
|LOCATION '${new File(s"${path}/l1/*").toURI}'""".stripMargin
sql(wildcardL1DirStatement)
if (parquetConversion == "true") {
checkAnswer(sql("SELECT * FROM tbl5"), (1 to 4).map(i => Row(i, i, s"parq$i")))
} else {
val msg = intercept[IOException] {
sql("SELECT * FROM tbl5").show()
}.getMessage
assert(msg.contains("Not a file:"))
}

val wildcardL2DirStatement =
s"""
|CREATE EXTERNAL TABLE tbl6(
| c1 int,
| c2 int,
| c3 string)
|STORED AS parquet
|LOCATION '${new File(s"${path}/l1/l2/*").toURI}'""".stripMargin
sql(wildcardL2DirStatement)
checkAnswer(sql("SELECT * FROM tbl6"), (3 to 6).map(i => Row(i, i, s"parq$i")))
}
}
}
}
}
}
162 changes: 162 additions & 0 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,4 +170,166 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
test("SPARK-11412 read and merge orc schemas in parallel") {
testMergeSchemasInParallel(OrcFileOperator.readOrcSchemasInParallel)
}

test("SPARK-25993 CREATE EXTERNAL TABLE with subdirectories") {
Seq(true, false).foreach { convertMetastore =>
withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> s"$convertMetastore") {
withTempDir { dir =>
try {
val orcTblStatement1 =
s"""
|CREATE EXTERNAL TABLE orc_tbl1(
| c1 int,
| c2 int,
| c3 string)
|STORED AS orc
|LOCATION '${s"${dir.getCanonicalPath}/l1/"}'""".stripMargin
sql(orcTblStatement1)

val orcTblInsertL1 =
s"INSERT INTO TABLE orc_tbl1 VALUES (1, 1, 'orc1'), (2, 2, 'orc2')".stripMargin
sql(orcTblInsertL1)

val orcTblStatement2 =
s"""
|CREATE EXTERNAL TABLE orc_tbl2(
| c1 int,
| c2 int,
| c3 string)
|STORED AS orc
|LOCATION '${s"${dir.getCanonicalPath}/l1/l2/"}'""".stripMargin
sql(orcTblStatement2)

val orcTblInsertL2 =
s"INSERT INTO TABLE orc_tbl2 VALUES (3, 3, 'orc3'), (4, 4, 'orc4')".stripMargin
sql(orcTblInsertL2)

val orcTblStatement3 =
s"""
|CREATE EXTERNAL TABLE orc_tbl3(
| c1 int,
| c2 int,
| c3 string)
|STORED AS orc
|LOCATION '${s"${dir.getCanonicalPath}/l1/l2/l3/"}'""".stripMargin
sql(orcTblStatement3)

val orcTblInsertL3 =
s"INSERT INTO TABLE orc_tbl3 VALUES (5, 5, 'orc5'), (6, 6, 'orc6')".stripMargin
sql(orcTblInsertL3)

withTable("tbl1", "tbl2", "tbl3", "tbl4", "tbl5", "tbl6") {
val topDirStatement =
s"""
|CREATE EXTERNAL TABLE tbl1(
| c1 int,
| c2 int,
| c3 string)
|STORED AS orc
|LOCATION '${s"${dir.getCanonicalPath}"}'""".stripMargin
sql(topDirStatement)
val topDirSqlStatement = s"SELECT * FROM tbl1"
if (convertMetastore) {
checkAnswer(sql(topDirSqlStatement), Nil)
} else {
checkAnswer(sql(topDirSqlStatement),
(1 to 6).map(i => Row(i, i, s"orc$i")))
}

val l1DirStatement =
s"""
|CREATE EXTERNAL TABLE tbl2(
| c1 int,
| c2 int,
| c3 string)
|STORED AS orc
|LOCATION '${s"${dir.getCanonicalPath}/l1/"}'""".stripMargin
sql(l1DirStatement)
val l1DirSqlStatement = s"SELECT * FROM tbl2"
if (convertMetastore) {
checkAnswer(sql(l1DirSqlStatement),
(1 to 2).map(i => Row(i, i, s"orc$i")))
Copy link
Member

@dongjoon-hyun dongjoon-hyun Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's make line 250 and 251 as one line like HiveParquetSourceSuite.

Copy link
Contributor Author

@kevinyu98 kevinyu98 Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure

} else {
checkAnswer(sql(l1DirSqlStatement),
(1 to 6).map(i => Row(i, i, s"orc$i")))
Copy link
Member

@dongjoon-hyun dongjoon-hyun Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto.

Copy link
Contributor Author

@kevinyu98 kevinyu98 Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

}

val l2DirStatement =
s"""
|CREATE EXTERNAL TABLE tbl3(
| c1 int,
| c2 int,
| c3 string)
|STORED AS orc
|LOCATION '${s"${dir.getCanonicalPath}/l1/l2/"}'""".stripMargin
sql(l2DirStatement)
val l2DirSqlStatement = s"SELECT * FROM tbl3"
if (convertMetastore) {
checkAnswer(sql(l2DirSqlStatement),
(3 to 4).map(i => Row(i, i, s"orc$i")))
Copy link
Member

@dongjoon-hyun dongjoon-hyun Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto.

} else {
checkAnswer(sql(l2DirSqlStatement),
(3 to 6).map(i => Row(i, i, s"orc$i")))
Copy link
Member

@dongjoon-hyun dongjoon-hyun Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto.

Copy link
Contributor Author

@kevinyu98 kevinyu98 Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

}

val wildcardTopDirStatement =
s"""
|CREATE EXTERNAL TABLE tbl4(
| c1 int,
| c2 int,
| c3 string)
|STORED AS orc
|LOCATION '${new File(s"${dir}/*").toURI}'""".stripMargin
sql(wildcardTopDirStatement)
val wildcardTopDirSqlStatement = s"SELECT * FROM tbl4"
if (convertMetastore) {
checkAnswer(sql(wildcardTopDirSqlStatement),
(1 to 2).map(i => Row(i, i, s"orc$i")))
Copy link
Member

@dongjoon-hyun dongjoon-hyun Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto.

Copy link
Contributor Author

@kevinyu98 kevinyu98 Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

} else {
checkAnswer(sql(wildcardTopDirSqlStatement), Nil)
}

val wildcardL1DirStatement =
s"""
|CREATE EXTERNAL TABLE tbl5(
| c1 int,
| c2 int,
| c3 string)
|STORED AS orc
|LOCATION '${new File(s"${dir}/l1/*").toURI}'""".stripMargin
sql(wildcardL1DirStatement)
val wildcardL1DirSqlStatement = s"SELECT * FROM tbl5"
if (convertMetastore) {
checkAnswer(sql(wildcardL1DirSqlStatement),
(1 to 4).map(i => Row(i, i, s"orc$i")))
Copy link
Member

@dongjoon-hyun dongjoon-hyun Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto.

Copy link
Contributor Author

@kevinyu98 kevinyu98 Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

} else {
checkAnswer(sql(wildcardL1DirSqlStatement), Nil)
}

val wildcardL2Statement =
s"""
|CREATE EXTERNAL TABLE tbl6(
| c1 int,
| c2 int,
| c3 string)
|STORED AS orc
|LOCATION '${new File(s"${dir}/l1/l2/*").toURI}'""".stripMargin
sql(wildcardL2Statement)
val wildcardL2SqlStatement = s"SELECT * FROM tbl6"
if (convertMetastore) {
checkAnswer(sql(wildcardL2SqlStatement),
(3 to 6).map(i => Row(i, i, s"orc$i")))
Copy link
Member

@dongjoon-hyun dongjoon-hyun Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto.

} else {
checkAnswer(sql(wildcardL2SqlStatement), Nil)
}
}
} finally {
Copy link
Member

@dongjoon-hyun dongjoon-hyun Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shall we convert this try..finally with withTable like HiveParquetSourceSuite?

Copy link
Contributor Author

@kevinyu98 kevinyu98 Jan 17, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

sql("DROP TABLE IF EXISTS orc_tbl1")
sql("DROP TABLE IF EXISTS orc_tbl2")
sql("DROP TABLE IF EXISTS orc_tbl3")
}
}
}
}
}
}