Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Datasource partition table should load empty partitions.
  • Loading branch information
wangyum committed Jul 26, 2018
commit 90f99fc6e645a65232cc52c81717402df1fd97df
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,12 @@ case class InsertIntoHadoopFsRelationCommand(


// update metastore partition metadata
refreshUpdatedPartitions(updatedPartitionPaths)
if (staticPartitions.nonEmpty) {
// Avoid empty partition can't loaded.
refreshUpdatedPartitions(Set(PartitioningUtils.getPathFragment(staticPartitions)))
} else {
refreshUpdatedPartitions(updatedPartitionPaths)
}

// refresh cached files in FileIndex
fileIndex.foreach(_.refresh())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,11 @@ object PartitioningUtils {
}.mkString("/")
}

def getPathFragment(partitions: TablePartitionSpec): String = partitions.map {
case (k, v) =>
escapePathName(k) + "=" + escapePathName(v)
}.mkString("/")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TablePartitionSpec is a map. The output of this func might be different from the actual schema, right?

For example, the partition schema is [partCol1, partCol2]. Your output could place partCol2 before partCol1

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right. Fixed it.


/**
* Normalize the column names in partition specification, w.r.t. the real partition column names
* and case sensitivity. e.g., if the partition spec has a column named `monTh`, and there is a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2249,6 +2249,27 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
}
}

test("Datasource partition table should load empty partitions") {
withTable("t", "t1") {
withTempPath { dir =>
spark.sql("CREATE TABLE t(a int) USING parquet")
spark.sql("CREATE TABLE t1(a int, b string, c string) " +
s"USING parquet PARTITIONED BY(b, c) LOCATION '${dir.toURI}'")

val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t1"))
assert(table.location == makeQualifiedPath(dir.getAbsolutePath))

spark.sql("INSERT INTO TABLE t1 PARTITION(b='b', c='c') SELECT * FROM t WHERE 1 = 0")

assert(spark.sql("SHOW PARTITIONS t1").count() == 1)

assert(new File(dir, "b=b/c=c").exists())

checkAnswer(spark.table("t1"), Nil)
}
}
}

Seq(true, false).foreach { shouldDelete =>
val tcName = if (shouldDelete) "non-existing" else "existed"
test(s"CTAS for external data source table with a $tcName location") {
Expand Down