From 6ae9c879e271dddf7142d6f0a10c76d08e2b6334 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Tue, 2 Apr 2024 10:53:15 -0700 Subject: [PATCH 1/2] fix --- .../spark/sql/catalyst/expressions/datetimeExpressions.scala | 4 ++-- .../test/scala/org/apache/spark/sql/DateFunctionsSuite.scala | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index a9155e8daf10..b81e1ee13380 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1235,11 +1235,11 @@ object TryToTimestampExpressionBuilder extends ExpressionBuilder { override def build(funcName: String, expressions: Seq[Expression]): Expression = { val numArgs = expressions.length if (numArgs == 1 || numArgs == 2) { - ParseToTimestamp( + TryEval(ParseToTimestamp( expressions.head, expressions.drop(1).lastOption, SQLConf.get.timestampType, - failOnError = false) + failOnError = false)) } else { throw QueryCompilationErrors.wrongNumArgsError(funcName, Seq(1, 2), numArgs) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala index b261ecfb0cee..07445c8910d0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala @@ -1392,4 +1392,9 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession { checkAnswer(df.selectExpr("try_to_timestamp(a)"), Seq(Row(ts))) checkAnswer(df.select(try_to_timestamp(col("a"))), Seq(Row(ts))) } + + test("try_to_timestamp: return null on SparkUpgradeException") { + val df = spark.sql("SELECT try_to_timestamp('2016-12-1', 'yyyy-MM-dd')") + checkAnswer(df, Seq(Row(null))) + } } From 98fe991697eea681b12d26ab41eb055436ee159f Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Tue, 2 Apr 2024 11:00:09 -0700 Subject: [PATCH 2/2] add comment --- .../spark/sql/catalyst/expressions/datetimeExpressions.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index b81e1ee13380..6fb3712dd6de 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1235,6 +1235,8 @@ object TryToTimestampExpressionBuilder extends ExpressionBuilder { override def build(funcName: String, expressions: Seq[Expression]): Expression = { val numArgs = expressions.length if (numArgs == 1 || numArgs == 2) { + // The expression ParseToTimestamp will throw an SparkUpgradeException if the input is invalid + // even when failOnError is false. We need to catch the exception and return null. TryEval(ParseToTimestamp( expressions.head, expressions.drop(1).lastOption,