Skip to content

Commit 2364cc2

Browse files
committed
address comments.
1 parent 49daace commit 2364cc2

File tree

2 files changed

+5
-2
lines changed

2 files changed

+5
-2
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,7 @@ case class FilterExec(condition: Expression, child: SparkPlan)
9595

9696
// If one expression and its children are null intolerant, it is null intolerant.
9797
private def isNullIntolerant(expr: Expression): Boolean = expr match {
98-
case e: NullIntolerant =>
99-
if (e.isInstanceOf[LeafExpression]) true else e.children.forall(isNullIntolerant)
98+
case e: NullIntolerant => e.children.forall(isNullIntolerant)
10099
case _ => false
101100
}
102101

sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1667,6 +1667,10 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
16671667
expr = "_1 + _2", expectedNonNullableColumns = Seq("_1", "_2"))
16681668
verifyNullabilityInFilterExec(df,
16691669
expr = "_1", expectedNonNullableColumns = Seq("_1"))
1670+
// `constructIsNotNullConstraints` infers the IsNotNull(_2) from IsNotNull(_2 + Rand())
1671+
// Thus, we are able to set nullability of _2 to false.
1672+
// If IsNotNull(_2) is not given from `constructIsNotNullConstraints`, the impl of
1673+
// isNullIntolerant in `FilterExec` needs an update for more advanced inference.
16701674
verifyNullabilityInFilterExec(df,
16711675
expr = "_2 + Rand()", expectedNonNullableColumns = Seq("_2"))
16721676
verifyNullabilityInFilterExec(df,

0 commit comments

Comments
 (0)