Skip to content
Closed
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,8 @@ class HadoopTableReader(
def fillPartitionKeys(rawPartValues: Array[String], row: InternalRow): Unit = {
partitionKeyAttrs.foreach { case (attr, ordinal) =>
val partOrdinal = partitionKeys.indexOf(attr)
row(ordinal) = Cast(Literal(rawPartValues(partOrdinal)), attr.dataType).eval(null)
row(ordinal) = Cast(Literal(rawPartValues(partOrdinal)), attr.dataType,
Option(sparkSession.sessionState.conf.sessionLocalTimeZone)).eval(null)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@ case class HiveTableScanExec(
hadoopConf)

private def castFromString(value: String, dataType: DataType) = {
Cast(Literal(value), dataType).eval(null)
Cast(Literal(value), dataType,
Option(sparkSession.sessionState.conf.sessionLocalTimeZone)).eval(null)
}

private def addColumnMetadataToConf(hiveConf: Configuration): Unit = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,4 +68,25 @@ class QueryPartitionSuite extends QueryTest with SQLTestUtils with TestHiveSingl
sql("DROP TABLE IF EXISTS createAndInsertTest")
}
}

test("SPARK-21739: Cast expression should initialize timezoneId " +
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test can pass without the change in TableReader.scala. We need another test case.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, it should select the TimestampType column. Thanks for reminder, I will fix it.

"when it is called statically to convert something into TimestampType") {
// create table for test
sql("CREATE TABLE table_with_timestamp_partition(value int) PARTITIONED by (ts timestamp)")
sql("INSERT OVERWRITE TABLE table_with_timestamp_partition " +
"partition (ts = '2010-01-01 00:00:00.000') VALUES (1)")
sql("INSERT OVERWRITE TABLE table_with_timestamp_partition " +
"partition (ts = '2010-01-02 00:00:00.000') VALUES (2)")

// test for Cast expression in TableReader
checkAnswer(sql("select value from table_with_timestamp_partition"),
Seq(Row(1), Row(2)))

// test for Cast expression in HiveTableScanExec
checkAnswer(sql("select value from table_with_timestamp_partition " +
"where ts = '2010-01-02 00:00:00.000'"), Row(2))

sql("DROP TABLE IF EXISTS table_with_timestamp_partition")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

use WithTable. You can check how we do it in the other test cases

}

}