From 456de1700549172b2b8d731874cd0f5a5579289b Mon Sep 17 00:00:00 2001 From: eyal farago Date: Fri, 29 Jul 2016 10:55:28 +0300 Subject: [PATCH 1/6] SPARK-16791_cast_struct_with_timestamp_field_fails: added a failing test case. --- .../spark/sql/catalyst/expressions/CastSuite.scala | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index 5ae0527a9c7a..5ad4692a92b5 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -727,6 +727,19 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { } } + test("cast struct with a timestamp field") { + val origSchema = StructType( + Seq(StructField( "tsField", TimestampType, false )) + ) + val trgSchema = StructType( + // nine out of ten times I'm casting a struct, it's to normalize its fields nullability + Seq(StructField( "tsField", TimestampType, true )) + ) + val inp = Literal.create(InternalRow(0L), origSchema) + val expected = InternalRow(0L) + checkEvaluation( Cast(inp, trgSchema), expected) + } + test("complex casting") { val complex = Literal.create( Row( From f13113d817fb8508959367501afe2a5b26be7a53 Mon Sep 17 00:00:00 2001 From: eyal farago Date: Fri, 29 Jul 2016 10:56:43 +0300 Subject: [PATCH 2/6] SPARK-16791_cast_struct_with_timestamp_field_fails: trivial fix. --- .../scala/org/apache/spark/sql/catalyst/expressions/Cast.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index c452765af2dd..09526ff729ba 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -416,7 +416,7 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w } private[this] def cast(from: DataType, to: DataType): Any => Any = to match { - case dt if dt == child.dataType => identity[Any] + case dt if dt == from => identity[Any] case StringType => castToString(from) case BinaryType => castToBinary(from) case DateType => castToDate(from) From a3b456c96eb559db553b746c17580e9218e109f8 Mon Sep 17 00:00:00 2001 From: eyal farago Date: Fri, 29 Jul 2016 20:55:12 +0300 Subject: [PATCH 3/6] SPARK-16791_cast_struct_with_timestamp_field_fails: fix style comments from reviewers. --- .../apache/spark/sql/catalyst/expressions/Cast.scala | 2 +- .../spark/sql/catalyst/expressions/CastSuite.scala | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 09526ff729ba..70fff5195625 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -416,7 +416,7 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w } private[this] def cast(from: DataType, to: DataType): Any => Any = to match { - case dt if dt == from => identity[Any] + case dt if dt == from => identity[Any] case StringType => castToString(from) case BinaryType => castToBinary(from) case DateType => castToDate(from) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index 5ad4692a92b5..9c679284f148 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -728,16 +728,16 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { } test("cast struct with a timestamp field") { - val origSchema = StructType( - Seq(StructField( "tsField", TimestampType, false )) + val originalSchema = StructType( + Seq(StructField( "tsField", TimestampType, nullable = false )) ) - val trgSchema = StructType( + val targetSchema = StructType( // nine out of ten times I'm casting a struct, it's to normalize its fields nullability - Seq(StructField( "tsField", TimestampType, true )) + Seq(StructField( "tsField", TimestampType, nullable = true )) ) - val inp = Literal.create(InternalRow(0L), origSchema) + val inp = Literal.create(InternalRow(0L), originalSchema) val expected = InternalRow(0L) - checkEvaluation( Cast(inp, trgSchema), expected) + checkEvaluation(cast(inp, targetSchema), expected ) } test("complex casting") { From 34fdb78786b7202792296b54e598e4a5b1791a7a Mon Sep 17 00:00:00 2001 From: eyal farago Date: Sat, 30 Jul 2016 11:07:04 +0300 Subject: [PATCH 4/6] SPARK-16791_cast_struct_with_timestamp_field_fails: some more formatting. --- .../spark/sql/catalyst/expressions/CastSuite.scala | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index 9c679284f148..63f9da36941a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -728,13 +728,10 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { } test("cast struct with a timestamp field") { - val originalSchema = StructType( - Seq(StructField( "tsField", TimestampType, nullable = false )) - ) - val targetSchema = StructType( - // nine out of ten times I'm casting a struct, it's to normalize its fields nullability - Seq(StructField( "tsField", TimestampType, nullable = true )) - ) + val originalSchema = new StructType().add( "tsField", TimestampType, nullable = false ) + // nine out of ten times I'm casting a struct, it's to normalize its fields nullability + val targetSchema = new StructType().add( "tsField", TimestampType, nullable = true ) + val inp = Literal.create(InternalRow(0L), originalSchema) val expected = InternalRow(0L) checkEvaluation(cast(inp, targetSchema), expected ) From 6e7f69f11e02632837e97c428e39ae3de0926e7b Mon Sep 17 00:00:00 2001 From: eyal farago Date: Sat, 30 Jul 2016 18:34:38 +0300 Subject: [PATCH 5/6] SPARK-16791_cast_struct_with_timestamp_field_fails: yet another formatting --- .../org/apache/spark/sql/catalyst/expressions/CastSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index 63f9da36941a..ff83aa43e7d8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -728,9 +728,9 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { } test("cast struct with a timestamp field") { - val originalSchema = new StructType().add( "tsField", TimestampType, nullable = false ) + val originalSchema = new StructType().add("tsField", TimestampType, nullable = false) // nine out of ten times I'm casting a struct, it's to normalize its fields nullability - val targetSchema = new StructType().add( "tsField", TimestampType, nullable = true ) + val targetSchema = new StructType().add("tsField", TimestampType, nullable = true) val inp = Literal.create(InternalRow(0L), originalSchema) val expected = InternalRow(0L) From e0549a998f3d75eeedf4334e9d61ad57dbe52efe Mon Sep 17 00:00:00 2001 From: Eyal Farago Date: Mon, 1 Aug 2016 14:13:01 +0300 Subject: [PATCH 6/6] SPARK-16791_cast_struct_with_timestamp_field_fails: remove one more extra space --- .../org/apache/spark/sql/catalyst/expressions/CastSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index ff83aa43e7d8..5c35baacef2f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -734,7 +734,7 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { val inp = Literal.create(InternalRow(0L), originalSchema) val expected = InternalRow(0L) - checkEvaluation(cast(inp, targetSchema), expected ) + checkEvaluation(cast(inp, targetSchema), expected) } test("complex casting") {