Skip to content

[SPARK-21811][SQL]Inconsistency when finding the widest common type of a combination of DateType, StringType, and NumericType #19033

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ryanbald wants to merge 1 commit into from
Closed

[SPARK-21811][SQL]Inconsistency when finding the widest common type of a combination of DateType, StringType, and NumericType #19033

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,27 @@ object TypeCoercion {
}

private def findWiderCommonType(types: Seq[DataType]): Option[DataType] = {
var awaitingString = false
Copy link
Contributor

@jiangxb1987 jiangxb1987 Feb 26, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we should make a pass of all dataTypes here, and see whether there exists StringType and all other dataTypes can be promoted to StringType, under that case we can default return StringType instead of None, WDYT?

types.foldLeft[Option[DataType]](Some(NullType))((r, c) => r match {
case Some(d) => findWiderTypeForTwo(d, c)
case None => None
case Some(d) => (d, c) match {
case (DateType, _: NumericType) | (_: NumericType, DateType) |
(TimestampType, _: NumericType) | (_: NumericType, TimestampType) =>
awaitingString = true
None
case _ => findWiderTypeForTwo(d, c)
}
case None if awaitingString => c match {
case DateType => None
case TimestampType => None
case (_: NumericType) => None
case StringType =>
awaitingString = false
Some(StringType)
case _ =>
awaitingString = false
None
}
case _ => None
})
}

Expand Down
24 changes: 17 additions & 7 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,11 @@ class TypeCoercionSuite extends AnalysisTest {
Coalesce(Seq(nullLit, floatNullLit, doubleLit, stringLit)),
Coalesce(Seq(Cast(nullLit, StringType), Cast(floatNullLit, StringType),
Cast(doubleLit, StringType), Cast(stringLit, StringType))))

ruleTest(rule,
Coalesce(Seq(timestampLit, intLit, stringLit)),
Coalesce(Seq(Cast(timestampLit, StringType), Cast(intLit, StringType),
Cast(stringLit, StringType))))
}

test("CreateArray casts") {
Expand Down Expand Up @@ -938,27 +943,32 @@ class TypeCoercionSuite extends AnalysisTest {
AttributeReference("i", IntegerType)(),
AttributeReference("u", DecimalType.SYSTEM_DEFAULT)(),
AttributeReference("b", ByteType)(),
AttributeReference("d", DoubleType)())
AttributeReference("d", DoubleType)(),
AttributeReference("a", DateType)())
val secondTable = LocalRelation(
AttributeReference("s", StringType)(),
AttributeReference("d", DecimalType(2, 1))(),
AttributeReference("f", FloatType)(),
AttributeReference("l", LongType)())
AttributeReference("l", LongType)(),
AttributeReference("t", TimestampType)())
val thirdTable = LocalRelation(
AttributeReference("m", StringType)(),
AttributeReference("n", DecimalType.SYSTEM_DEFAULT)(),
AttributeReference("p", FloatType)(),
AttributeReference("q", DoubleType)())
val forthTable = LocalRelation(
AttributeReference("q", DoubleType)(),
AttributeReference("l", LongType)())
val fourthTable = LocalRelation(
AttributeReference("m", StringType)(),
AttributeReference("n", DecimalType.SYSTEM_DEFAULT)(),
AttributeReference("p", ByteType)(),
AttributeReference("q", DoubleType)())
AttributeReference("q", DoubleType)(),
AttributeReference("s", StringType)())

val expectedTypes = Seq(StringType, DecimalType.SYSTEM_DEFAULT, FloatType, DoubleType)
val expectedTypes = Seq(StringType, DecimalType.SYSTEM_DEFAULT, FloatType, DoubleType,
StringType)

val unionRelation = widenSetOperationTypes(
Union(firstTable :: secondTable :: thirdTable :: forthTable :: Nil)).asInstanceOf[Union]
Union(firstTable :: secondTable :: thirdTable :: fourthTable :: Nil)).asInstanceOf[Union]
assert(unionRelation.children.length == 4)
checkOutput(unionRelation.children.head, expectedTypes)
checkOutput(unionRelation.children(1), expectedTypes)
Expand Down