[SPARK-25993][SQL][TESTS] Add test cases for CREATE EXTERNAL TABLE with subdirectories #27130
Next
Next commit
rebase on spark 3.0
- Loading branch information
commit 9722230e29609102e48e763984c7dfd9c972f1f4
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -170,4 +170,155 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton { | |
| test("SPARK-11412 read and merge orc schemas in parallel") { | ||
| testMergeSchemasInParallel(OrcFileOperator.readOrcSchemasInParallel) | ||
| } | ||
|
|
||
| test("SPARK-25993 CREATE EXTERNAL TABLE with subdirectories") { | ||
| Seq(true, false).foreach { convertMetastore => | ||
| withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> s"$convertMetastore") { | ||
| withTempDir { dir => | ||
| try { | ||
| hiveClient.runSqlHive("USE default") | ||
| hiveClient.runSqlHive( | ||
|
||
| """ | ||
| |CREATE EXTERNAL TABLE hive_orc( | ||
|
||
| | C1 INT, | ||
| | C2 INT, | ||
| | C3 STRING) | ||
| |STORED AS orc""".stripMargin) | ||
| // Hive throws an exception if I assign the location in the create table statement. | ||
| hiveClient.runSqlHive( | ||
| s"ALTER TABLE hive_orc SET LOCATION " + | ||
| s"'${new File(s"${dir.getCanonicalPath}/l1/").toURI}'") | ||
| hiveClient.runSqlHive( | ||
| """ | ||
| |INSERT INTO TABLE hive_orc | ||
| |VALUES (1, 1, 'orc1'), (2, 2, 'orc2')""".stripMargin) | ||
|
|
||
| hiveClient.runSqlHive( | ||
| s"ALTER TABLE hive_orc SET LOCATION " + | ||
| s"'${new File(s"${dir.getCanonicalPath}/l1/l2/").toURI}'") | ||
| hiveClient.runSqlHive( | ||
| """ | ||
| |INSERT INTO TABLE hive_orc | ||
| |VALUES (3, 3, 'orc3'), (4, 4, 'orc4')""".stripMargin) | ||
|
|
||
| hiveClient.runSqlHive( | ||
| s"ALTER TABLE hive_orc SET LOCATION " + | ||
| s"'${new File(s"${dir.getCanonicalPath}/l1/l2/l3/").toURI}'") | ||
| hiveClient.runSqlHive( | ||
| """ | ||
| |INSERT INTO TABLE hive_orc | ||
| |VALUES (5, 5, 'orc5'), (6, 6, 'orc6')""".stripMargin) | ||
|
|
||
| withTable("tbl1", "tbl2", "tbl3", "tbl4", "tbl5", "tbl6") { | ||
| val topDirStatement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl1( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${s"${dir.getCanonicalPath}"}'""".stripMargin | ||
| sql(topDirStatement) | ||
| val topDirSqlStatement = s"select * from tbl1" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(topDirSqlStatement), Nil) | ||
| } else { | ||
| checkAnswer(sql(topDirSqlStatement), | ||
| (1 to 6).map(i => Row(i, i, s"orc$i"))) | ||
| } | ||
|
|
||
| val l1DirStatement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl2( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${s"${dir.getCanonicalPath}/l1/"}'""".stripMargin | ||
| sql(l1DirStatement) | ||
| val l1DirSqlStatement = s"select * from tbl2" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(l1DirSqlStatement), | ||
| (1 to 2).map(i => Row(i, i, s"orc$i"))) | ||
|
||
| } else { | ||
| checkAnswer(sql(l1DirSqlStatement), | ||
| (1 to 6).map(i => Row(i, i, s"orc$i"))) | ||
|
||
| } | ||
|
|
||
| val l2DirStatement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl3( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${s"${dir.getCanonicalPath}/l1/l2/"}'""".stripMargin | ||
| sql(l2DirStatement) | ||
| val l2DirSqlStatement = s"select * from tbl3" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(l2DirSqlStatement), | ||
| (3 to 4).map(i => Row(i, i, s"orc$i"))) | ||
|
||
| } else { | ||
| checkAnswer(sql(l2DirSqlStatement), | ||
| (3 to 6).map(i => Row(i, i, s"orc$i"))) | ||
|
||
| } | ||
|
|
||
| val wildcardTopDirStatement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl4( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${new File(s"${dir}/*").toURI}'""".stripMargin | ||
| sql(wildcardTopDirStatement) | ||
| val wildcardTopDirSqlStatement = s"select * from tbl4" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(wildcardTopDirSqlStatement), | ||
| (1 to 2).map(i => Row(i, i, s"orc$i"))) | ||
|
||
| } else { | ||
| checkAnswer(sql(wildcardTopDirSqlStatement), Nil) | ||
| } | ||
|
|
||
| val wildcardL1DirStatement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl5( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${new File(s"${dir}/l1/*").toURI}'""".stripMargin | ||
| sql(wildcardL1DirStatement) | ||
| val wildcardL1DirSqlStatement = s"select * from tbl5" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(wildcardL1DirSqlStatement), | ||
| (1 to 4).map(i => Row(i, i, s"orc$i"))) | ||
|
||
| } else { | ||
| checkAnswer(sql(wildcardL1DirSqlStatement), Nil) | ||
| } | ||
|
|
||
| val wildcardL2Statement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl6( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${new File(s"${dir}/l1/l2/*").toURI}'""".stripMargin | ||
| sql(wildcardL2Statement) | ||
| val wildcardL2SqlStatement = s"select * from tbl6" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(wildcardL2SqlStatement), | ||
| (3 to 6).map(i => Row(i, i, s"orc$i"))) | ||
|
||
| } else { | ||
| checkAnswer(sql(wildcardL2SqlStatement), Nil) | ||
| } | ||
| } | ||
| } finally { | ||
|
||
| hiveClient.runSqlHive("DROP TABLE IF EXISTS hive_orc") | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Shall we capitalize the SQL statement like
SELECT * FROM tbl1?