Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Revert "[SPARK-28395][SQL] Division operator support integral division"
This reverts commit 6926849.
  • Loading branch information
xuanyuanking committed Dec 9, 2019
commit 9638331d303b9b3292fc0a2d5a82ecdf3c9322c5
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ object TypeCoercion {
CaseWhenCoercion ::
IfCoercion ::
StackCoercion ::
Division(conf) ::
Division ::
ImplicitTypeCasts ::
DateTimeOperations ::
WindowFrameCoercion ::
Expand Down Expand Up @@ -662,7 +662,7 @@ object TypeCoercion {
* Hive only performs integral division with the DIV operator. The arguments to / are always
* converted to fractional types.
*/
case class Division(conf: SQLConf) extends TypeCoercionRule {
object Division extends TypeCoercionRule {
override protected def coerceTypes(
plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
// Skip nodes who has not been resolved yet,
Expand All @@ -673,12 +673,7 @@ object TypeCoercion {
case d: Divide if d.dataType == DoubleType => d
case d: Divide if d.dataType.isInstanceOf[DecimalType] => d
case Divide(left, right) if isNumericOrNull(left) && isNumericOrNull(right) =>
(left.dataType, right.dataType) match {
case (_: IntegralType, _: IntegralType) if conf.preferIntegralDivision =>
IntegralDivide(left, right)
case _ =>
Divide(Cast(left, DoubleType), Cast(right, DoubleType))
}
Divide(Cast(left, DoubleType), Cast(right, DoubleType))
}

private def isNumericOrNull(ex: Expression): Boolean = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1669,21 +1669,6 @@ object SQLConf {
.booleanConf
.createWithDefault(false)

val PREFER_INTEGRAL_DIVISION = buildConf("spark.sql.function.preferIntegralDivision")
.internal()
.doc("When true, will perform integral division with the / operator " +
"if both sides are integral types. This is for PostgreSQL test cases only.")
.booleanConf
.createWithDefault(false)

val ALLOW_CREATING_MANAGED_TABLE_USING_NONEMPTY_LOCATION =
buildConf("spark.sql.legacy.allowCreatingManagedTableUsingNonemptyLocation")
.internal()
.doc("When this option is set to true, creating managed tables with nonempty location " +
"is allowed. Otherwise, an analysis exception is thrown. ")
.booleanConf
.createWithDefault(false)

val VALIDATE_PARTITION_COLUMNS =
buildConf("spark.sql.sources.validatePartitionColumns")
.internal()
Expand Down Expand Up @@ -2537,8 +2522,6 @@ class SQLConf extends Serializable with Logging {

def eltOutputAsString: Boolean = getConf(ELT_OUTPUT_AS_STRING)

def preferIntegralDivision: Boolean = getConf(PREFER_INTEGRAL_DIVISION)

def validatePartitionColumns: Boolean = getConf(VALIDATE_PARTITION_COLUMNS)

def partitionOverwriteMode: PartitionOverwriteMode.Value =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1425,7 +1425,7 @@ class TypeCoercionSuite extends AnalysisTest {

test("SPARK-15776 Divide expression's dataType should be casted to Double or Decimal " +
"in aggregation function like sum") {
val rules = Seq(FunctionArgumentConversion, Division(conf))
val rules = Seq(FunctionArgumentConversion, Division)
// Casts Integer to Double
ruleTest(rules, sum(Divide(4, 3)), sum(Divide(Cast(4, DoubleType), Cast(3, DoubleType))))
// Left expression is Double, right expression is Int. Another rule ImplicitTypeCasts will
Expand All @@ -1444,35 +1444,12 @@ class TypeCoercionSuite extends AnalysisTest {
}

test("SPARK-17117 null type coercion in divide") {
val rules = Seq(FunctionArgumentConversion, Division(conf), ImplicitTypeCasts)
val rules = Seq(FunctionArgumentConversion, Division, ImplicitTypeCasts)
val nullLit = Literal.create(null, NullType)
ruleTest(rules, Divide(1L, nullLit), Divide(Cast(1L, DoubleType), Cast(nullLit, DoubleType)))
ruleTest(rules, Divide(nullLit, 1L), Divide(Cast(nullLit, DoubleType), Cast(1L, DoubleType)))
}

test("SPARK-28395 Division operator support integral division") {
val rules = Seq(FunctionArgumentConversion, Division(conf))
Seq(true, false).foreach { preferIntegralDivision =>
withSQLConf(SQLConf.PREFER_INTEGRAL_DIVISION.key -> s"$preferIntegralDivision") {
val result1 = if (preferIntegralDivision) {
IntegralDivide(1L, 1L)
} else {
Divide(Cast(1L, DoubleType), Cast(1L, DoubleType))
}
ruleTest(rules, Divide(1L, 1L), result1)
val result2 = if (preferIntegralDivision) {
IntegralDivide(1, Cast(1, ShortType))
} else {
Divide(Cast(1, DoubleType), Cast(Cast(1, ShortType), DoubleType))
}
ruleTest(rules, Divide(1, Cast(1, ShortType)), result2)

ruleTest(rules, Divide(1L, 1D), Divide(Cast(1L, DoubleType), Cast(1D, DoubleType)))
ruleTest(rules, Divide(Decimal(1.1), 1L), Divide(Decimal(1.1), 1L))
}
}
}

test("binary comparison with string promotion") {
val rule = TypeCoercion.PromoteStrings(conf)
ruleTest(rule,
Expand Down