Skip to content

[SPARK-24734][SQL] Fix type coercions and nullabilities of nested data types of some functions. #21704

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ueshin wants to merge 21 commits into from
Closed

[SPARK-24734][SQL] Fix type coercions and nullabilities of nested data types of some functions. #21704

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
d87a8c6
Fix containsNull of Concat for array type.
ueshin Jul 3, 2018
30d5aed
Add a test.
ueshin Jul 4, 2018
2e624df
Revert "Fix containsNull of Concat for array type."
ueshin Jul 9, 2018
fa73b32
Merge branch 'master' into issues/SPARK-24734/concat_containsnull
ueshin Jul 9, 2018
da0702b
Suppress unneeded casts.
ueshin Jul 9, 2018
b31e401
Fix `Concat`.
ueshin Jul 9, 2018
7a838b0
Fix `CreateArray`.
ueshin Jul 9, 2018
01c9ff3
Fix `CreateMap`.
ueshin Jul 9, 2018
b2ca587
Fix `Coalesce`.
ueshin Jul 9, 2018
3d8891e
Fix `Least` and `Greatest`.
ueshin Jul 9, 2018
1fa692a
Merge branch 'master' into issues/SPARK-24734/concat_containsnull
ueshin Jul 9, 2018
444383d
Fix `MapConcat`.
ueshin Jul 9, 2018
3e1f7e4
Fix udts.
ueshin Jul 10, 2018
2c54e38
Rename to `findCommonTypeDifferentOnlyInNullFlags`.
ueshin Jul 10, 2018
5f1b865
Merge branch 'master' into issues/SPARK-24734/concat_containsnull
ueshin Jul 11, 2018
2ab025f
Use `ComplexTypeMergingExpression` trait.
ueshin Jul 11, 2018
db254e5
Fix udts.
ueshin Jul 11, 2018
b412f7b
Merge branch 'master' into issues/SPARK-24734/concat_containsnull
ueshin Jul 12, 2018
f701242
Make `ArraySetLike` `ComplexTypeMergingExpression`.
ueshin Jul 12, 2018
5115961
Use `TypeCoercion.haveSameType`.
ueshin Jul 13, 2018
e489e8b
Revert "Make `ArraySetLike` `ComplexTypeMergingExpression`."
ueshin Jul 16, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix Concat.
  • Loading branch information
@ueshin
ueshin committed Jul 9, 2018
commit b31e401777bbdb02c21f0dca26b743383a237f08
3 changes: 2 additions & 1 deletion ...alyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2007,7 +2007,8 @@ case class Concat(children: Seq[Expression]) extends Expression {
}
}

override def dataType: DataType = children.map(_.dataType).headOption.getOrElse(StringType)
override def dataType: DataType =
TypeCoercion.findWiderNullablilityType(children.map(_.dataType)).getOrElse(StringType)

lazy val javaType: String = CodeGenerator.javaType(dataType)

Expand Down
39 changes: 26 additions & 13 deletions ...src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -919,11 +919,11 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper

test("Concat") {
// Primitive-type elements
val ai0 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType))
val ai1 = Literal.create(Seq.empty[Integer], ArrayType(IntegerType))
val ai2 = Literal.create(Seq(4, null, 5), ArrayType(IntegerType))
val ai3 = Literal.create(Seq(null, null), ArrayType(IntegerType))
val ai4 = Literal.create(null, ArrayType(IntegerType))
val ai0 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType, containsNull = false))
val ai1 = Literal.create(Seq.empty[Integer], ArrayType(IntegerType, containsNull = false))
val ai2 = Literal.create(Seq(4, null, 5), ArrayType(IntegerType, containsNull = true))
val ai3 = Literal.create(Seq(null, null), ArrayType(IntegerType, containsNull = true))
val ai4 = Literal.create(null, ArrayType(IntegerType, containsNull = false))

checkEvaluation(Concat(Seq(ai0)), Seq(1, 2, 3))
checkEvaluation(Concat(Seq(ai0, ai1)), Seq(1, 2, 3))
Expand All @@ -936,14 +936,18 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(Concat(Seq(ai4, ai0)), null)

// Non-primitive-type elements
val as0 = Literal.create(Seq("a", "b", "c"), ArrayType(StringType))
val as1 = Literal.create(Seq.empty[String], ArrayType(StringType))
val as2 = Literal.create(Seq("d", null, "e"), ArrayType(StringType))
val as3 = Literal.create(Seq(null, null), ArrayType(StringType))
val as4 = Literal.create(null, ArrayType(StringType))

val aa0 = Literal.create(Seq(Seq("a", "b"), Seq("c")), ArrayType(ArrayType(StringType)))
val aa1 = Literal.create(Seq(Seq("d"), Seq("e", "f")), ArrayType(ArrayType(StringType)))
val as0 = Literal.create(Seq("a", "b", "c"), ArrayType(StringType, containsNull = false))
val as1 = Literal.create(Seq.empty[String], ArrayType(StringType, containsNull = false))
val as2 = Literal.create(Seq("d", null, "e"), ArrayType(StringType, containsNull = true))
val as3 = Literal.create(Seq(null, null), ArrayType(StringType, containsNull = true))
val as4 = Literal.create(null, ArrayType(StringType, containsNull = false))

val aa0 = Literal.create(Seq(Seq("a", "b"), Seq("c")),
ArrayType(ArrayType(StringType, containsNull = false), containsNull = false))
val aa1 = Literal.create(Seq(Seq("d"), Seq("e", "f")),
ArrayType(ArrayType(StringType, containsNull = false), containsNull = false))
val aa2 = Literal.create(Seq(Seq("g", null), null),
ArrayType(ArrayType(StringType, containsNull = true), containsNull = true))

checkEvaluation(Concat(Seq(as0)), Seq("a", "b", "c"))
checkEvaluation(Concat(Seq(as0, as1)), Seq("a", "b", "c"))
Expand All @@ -956,6 +960,15 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(Concat(Seq(as4, as0)), null)

checkEvaluation(Concat(Seq(aa0, aa1)), Seq(Seq("a", "b"), Seq("c"), Seq("d"), Seq("e", "f")))

assert(Concat(Seq(ai0, ai1)).dataType.asInstanceOf[ArrayType].containsNull === false)
assert(Concat(Seq(ai0, ai2)).dataType.asInstanceOf[ArrayType].containsNull === true)
assert(Concat(Seq(as0, as1)).dataType.asInstanceOf[ArrayType].containsNull === false)
assert(Concat(Seq(as0, as2)).dataType.asInstanceOf[ArrayType].containsNull === true)
assert(Concat(Seq(aa0, aa1)).dataType ===
ArrayType(ArrayType(StringType, containsNull = false), containsNull = false))
assert(Concat(Seq(aa0, aa2)).dataType ===
ArrayType(ArrayType(StringType, containsNull = true), containsNull = true))
}

test("Flatten") {
Expand Down