-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-28067][SQL] Fix incorrect results for decimal aggregate sum by returning null on decimal overflow #27627
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
edb32b9
8fe9aa4
1adc512
d90c790
136e6dc
6979e8d
4119e02
cc2fec0
23739c9
fa45378
fbd80a6
de2d68f
8339e28
59a00c4
7795888
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.expressions | |
|
|
||
| import org.apache.spark.sql.catalyst.InternalRow | ||
| import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, EmptyBlock, ExprCode} | ||
| import org.apache.spark.sql.catalyst.expressions.codegen.Block._ | ||
| import org.apache.spark.sql.internal.SQLConf | ||
| import org.apache.spark.sql.types._ | ||
|
|
||
|
|
@@ -144,3 +145,54 @@ case class CheckOverflow( | |
|
|
||
| override def sql: String = child.sql | ||
| } | ||
|
|
||
| // A variant `CheckOverflow`, which treats null as overflow. This is necessary in `Sum`. | ||
| case class CheckOverflowInSum( | ||
| child: Expression, | ||
| dataType: DecimalType, | ||
| nullOnOverflow: Boolean) extends UnaryExpression { | ||
|
|
||
| override def nullable: Boolean = true | ||
|
||
|
|
||
| override def eval(input: InternalRow): Any = { | ||
| val value = child.eval(input) | ||
| if (value == null) { | ||
| if (nullOnOverflow) null else throw new ArithmeticException("Overflow in sum of decimals.") | ||
| } else { | ||
| value.asInstanceOf[Decimal].toPrecision( | ||
| dataType.precision, | ||
| dataType.scale, | ||
| Decimal.ROUND_HALF_UP, | ||
| nullOnOverflow) | ||
| } | ||
| } | ||
|
|
||
| override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { | ||
| val childGen = child.genCode(ctx) | ||
| val nullHandling = if (nullOnOverflow) { | ||
| "" | ||
| } else { | ||
| s""" | ||
| |throw new ArithmeticException("Overflow in sum of decimals."); | ||
| |""".stripMargin | ||
| } | ||
| val code = code""" | ||
| |${childGen.code} | ||
| |boolean ${ev.isNull} = ${childGen.isNull}; | ||
| |Decimal ${ev.value} = null; | ||
| |if (${childGen.isNull}) { | ||
| | $nullHandling | ||
| |} else { | ||
| | ${ev.value} = ${childGen.value}.toPrecision( | ||
| | ${dataType.precision}, ${dataType.scale}, Decimal.ROUND_HALF_UP(), $nullOnOverflow); | ||
| | ${ev.isNull} = ${ev.value} == null; | ||
| |} | ||
| |""".stripMargin | ||
|
|
||
| ev.copy(code = code) | ||
| } | ||
|
|
||
| override def toString: String = s"CheckOverflowInSum($child, $dataType, $nullOnOverflow)" | ||
|
|
||
| override def sql: String = child.sql | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.