is null

Signed-off-by: jayzhan211 <jayzhan211@gmail.com>

datafusion/physical-expr-common/Cargo.toml

@@ -45,4 +45,4 @@ datafusion-common = { workspace = true, default-features = true }
 datafusion-expr = { workspace = true }
 half = { workspace = true }
 hashbrown = { version = "0.14", features = ["raw"] }
-paste = "^1.0"
+paste = "^1.0"

datafusion/physical-expr-common/src/expressions/cast.rs

@@ -16,8 +16,8 @@
 // under the License.
 
 use crate::physical_expr::down_cast_any_ref;
-use crate::sort_properties::SortProperties;
 use crate::physical_expr::PhysicalExpr;
+use crate::sort_properties::SortProperties;
 use std::any::Any;
 use std::fmt;
 use std::hash::{Hash, Hasher};

datafusion/physical-expr-common/src/expressions/in_list.rs

@@ -22,18 +22,21 @@ use std::fmt::Debug;
 use std::hash::{Hash, Hasher};
 use std::sync::Arc;
 
-use crate::physical_expr::{down_cast_any_ref, physical_exprs_bag_equal};
 use crate::physical_expr::PhysicalExpr;
+use crate::physical_expr::{down_cast_any_ref, physical_exprs_bag_equal};
 
+use arrow::array::downcast_primitive_array;
+use arrow::array::{
+    as_largestring_array, downcast_array, downcast_dictionary_array, Array,
+    ArrayAccessor, ArrayData, ArrayIter, ArrayRef, BooleanArray,
+};
 use arrow::buffer::BooleanBuffer;
 use arrow::compute::kernels::boolean::{not, or_kleene};
 use arrow::compute::kernels::cmp::eq;
 use arrow::compute::take;
 use arrow::datatypes::{i256, DataType, Schema};
 use arrow::record_batch::RecordBatch;
 use arrow::util::bit_iterator::BitIndexIterator;
-use arrow::array::{as_largestring_array, downcast_array, downcast_dictionary_array, Array, ArrayAccessor, ArrayData, ArrayIter, ArrayRef, BooleanArray};
-use arrow::array::downcast_primitive_array;
 use datafusion_common::cast::{
     as_boolean_array, as_generic_binary_array, as_string_array,
 };
@@ -474,7 +477,7 @@ mod tests {
     use datafusion_common::plan_err;
     use datafusion_common::Result;
     use datafusion_expr::type_coercion::binary::comparison_coercion;
-    
+
     use arrow::datatypes::Field;
     use arrow::datatypes::TimeUnit;
 

datafusion/physical-expr/src/expressions/is_not_null.rs → datafusion/physical-expr-common/src/expressions/is_not_null.rs

@@ -21,7 +21,7 @@ use std::hash::{Hash, Hasher};
 use std::{any::Any, sync::Arc};
 
 use crate::physical_expr::down_cast_any_ref;
-use crate::PhysicalExpr;
+use crate::physical_expr::PhysicalExpr;
 use arrow::compute;
 use arrow::{
     datatypes::{DataType, Schema},
@@ -115,7 +115,7 @@ pub fn is_not_null(arg: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>>
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::expressions::col;
+    use crate::expressions::column::col;
     use arrow::{
         array::{BooleanArray, StringArray},
         datatypes::*,

datafusion/physical-expr/src/expressions/is_null.rs → datafusion/physical-expr-common/src/expressions/is_null.rs

@@ -27,7 +27,7 @@ use arrow::{
 };
 
 use crate::physical_expr::down_cast_any_ref;
-use crate::PhysicalExpr;
+use crate::physical_expr::PhysicalExpr;
 use datafusion_common::Result;
 use datafusion_common::ScalarValue;
 use datafusion_expr::ColumnarValue;
@@ -116,7 +116,7 @@ pub fn is_null(arg: Arc<dyn PhysicalExpr>) -> Result<Arc<dyn PhysicalExpr>> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::expressions::col;
+    use crate::expressions::column::col;
     use arrow::{
         array::{BooleanArray, StringArray},
         datatypes::*,

datafusion/physical-expr-common/src/expressions/like.rs

@@ -20,11 +20,11 @@ use std::{any::Any, sync::Arc};
 
 use crate::physical_expr::{down_cast_any_ref, PhysicalExpr};
 
+use crate::expressions::datum::apply_cmp;
 use arrow::record_batch::RecordBatch;
 use arrow_schema::{DataType, Schema};
 use datafusion_common::{internal_err, Result};
 use datafusion_expr::ColumnarValue;
-use crate::expressions::datum::apply_cmp;
 
 // Like expression
 #[derive(Debug, Hash)]

datafusion/physical-expr-common/src/expressions/mod.rs

@@ -23,6 +23,8 @@ pub mod cast;
 pub mod column;
 pub mod datum;
 pub mod in_list;
+pub mod is_not_null;
+pub mod is_null;
 pub mod like;
 pub mod literal;
 pub mod negative;

datafusion/physical-expr-common/src/expressions/negative.rs

@@ -22,8 +22,8 @@ use std::hash::{Hash, Hasher};
 use std::sync::Arc;
 
 use crate::physical_expr::down_cast_any_ref;
-use crate::sort_properties::SortProperties;
 use crate::physical_expr::PhysicalExpr;
+use crate::sort_properties::SortProperties;
 
 use arrow::{
     compute::kernels::numeric::neg_wrapping,

datafusion/physical-expr-common/src/physical_expr.rs

@@ -26,16 +26,24 @@ use arrow::datatypes::{DataType, Schema};
 use arrow::record_batch::RecordBatch;
 
 use datafusion_common::utils::DataPtr;
-use datafusion_common::{exec_err, internal_err, not_impl_err, DFSchema, Result, ScalarValue};
+use datafusion_common::{
+    exec_err, internal_err, not_impl_err, plan_err, DFSchema, Result, ScalarValue,
+};
 use datafusion_expr::execution_props::ExecutionProps;
 use datafusion_expr::expr::{Alias, InList};
 use datafusion_expr::interval_arithmetic::Interval;
-use datafusion_expr::{Between, BinaryExpr, Cast, ColumnarValue, Expr, Like, Operator, TryCast};
+use datafusion_expr::var_provider::{is_system_variables, VarType};
+use datafusion_expr::{
+    Between, BinaryExpr, Cast, ColumnarValue, Expr, GetFieldAccess, GetIndexedField,
+    Like, Operator, TryCast,
+};
 
 use crate::expressions::binary::binary;
 use crate::expressions::cast::cast;
 use crate::expressions::column::Column;
 use crate::expressions::in_list::in_list;
+use crate::expressions::is_not_null::is_not_null;
+use crate::expressions::is_null::is_null;
 use crate::expressions::like::like;
 use crate::expressions::literal::{lit, Literal};
 use crate::expressions::negative::negative;
@@ -330,25 +338,25 @@ pub fn create_physical_expr(
             Ok(Arc::new(Column::new(&c.name, idx)))
         }
         Expr::Literal(value) => Ok(Arc::new(Literal::new(value.clone()))),
-        // Expr::ScalarVariable(_, variable_names) => {
-        //     if is_system_variables(variable_names) {
-        //         match execution_props.get_var_provider(VarType::System) {
-        //             Some(provider) => {
-        //                 let scalar_value = provider.get_value(variable_names.clone())?;
-        //                 Ok(Arc::new(Literal::new(scalar_value)))
-        //             }
-        //             _ => plan_err!("No system variable provider found"),
-        //         }
-        //     } else {
-        //         match execution_props.get_var_provider(VarType::UserDefined) {
-        //             Some(provider) => {
-        //                 let scalar_value = provider.get_value(variable_names.clone())?;
-        //                 Ok(Arc::new(Literal::new(scalar_value)))
-        //             }
-        //             _ => plan_err!("No user defined variable provider found"),
-        //         }
-        //     }
-        // }
+        Expr::ScalarVariable(_, variable_names) => {
+            if is_system_variables(variable_names) {
+                match execution_props.get_var_provider(VarType::System) {
+                    Some(provider) => {
+                        let scalar_value = provider.get_value(variable_names.clone())?;
+                        Ok(Arc::new(Literal::new(scalar_value)))
+                    }
+                    _ => plan_err!("No system variable provider found"),
+                }
+            } else {
+                match execution_props.get_var_provider(VarType::UserDefined) {
+                    Some(provider) => {
+                        let scalar_value = provider.get_value(variable_names.clone())?;
+                        Ok(Arc::new(Literal::new(scalar_value)))
+                    }
+                    _ => plan_err!("No user defined variable provider found"),
+                }
+            }
+        }
         Expr::IsTrue(expr) => {
             let binary_op = datafusion_expr::binary_expr(
                 expr.as_ref().clone(),
@@ -486,33 +494,29 @@ pub fn create_physical_expr(
             create_physical_expr(expr, input_dfschema, execution_props)?,
             input_schema,
         ),
-        // Expr::IsNull(expr) => expressions::is_null(create_physical_expr(
-        //     expr,
-        //     input_dfschema,
-        //     execution_props,
-        // )?),
-        // Expr::IsNotNull(expr) => expressions::is_not_null(create_physical_expr(
-        //     expr,
-        //     input_dfschema,
-        //     execution_props,
-        // )?),
-        // Expr::GetIndexedField(GetIndexedField { expr: _, field }) => match field {
-        //     GetFieldAccess::NamedStructField { name: _ } => {
-        //         internal_err!(
-        //             "NamedStructField should be rewritten in OperatorToFunction"
-        //         )
-        //     }
-        //     GetFieldAccess::ListIndex { key: _ } => {
-        //         internal_err!("ListIndex should be rewritten in OperatorToFunction")
-        //     }
-        //     GetFieldAccess::ListRange {
-        //         start: _,
-        //         stop: _,
-        //         stride: _,
-        //     } => {
-        //         internal_err!("ListRange should be rewritten in OperatorToFunction")
-        //     }
-        // },
+        Expr::IsNull(expr) => {
+            is_null(create_physical_expr(expr, input_dfschema, execution_props)?)
+        }
+        Expr::IsNotNull(expr) => {
+            is_not_null(create_physical_expr(expr, input_dfschema, execution_props)?)
+        }
+        Expr::GetIndexedField(GetIndexedField { expr: _, field }) => match field {
+            GetFieldAccess::NamedStructField { name: _ } => {
+                internal_err!(
+                    "NamedStructField should be rewritten in OperatorToFunction"
+                )
+            }
+            GetFieldAccess::ListIndex { key: _ } => {
+                internal_err!("ListIndex should be rewritten in OperatorToFunction")
+            }
+            GetFieldAccess::ListRange {
+                start: _,
+                stop: _,
+                stride: _,
+            } => {
+                internal_err!("ListRange should be rewritten in OperatorToFunction")
+            }
+        },
 
         // Expr::ScalarFunction(ScalarFunction { func_def, args }) => {
         //     let physical_args =
@@ -568,9 +572,7 @@ pub fn create_physical_expr(
             list,
             negated,
         }) => match expr.as_ref() {
-            Expr::Literal(ScalarValue::Utf8(None)) => {
-                Ok(lit(ScalarValue::Boolean(None)))
-            }
+            Expr::Literal(ScalarValue::Utf8(None)) => Ok(lit(ScalarValue::Boolean(None))),
             _ => {
                 let value_expr =
                     create_physical_expr(expr, input_dfschema, execution_props)?;

datafusion/physical-expr/src/expressions/mod.rs

@@ -19,8 +19,6 @@
 
 mod case;
 mod column;
-mod is_not_null;
-mod is_null;
 mod no_op;
 
 /// Module with some convenient methods used in expression building
@@ -70,20 +68,26 @@ pub use datafusion_functions_aggregate::first_last::{
 };
 
 pub use case::{case, CaseExpr};
-pub use datafusion_physical_expr_common::expressions::cast::{cast, cast_with_options, CastExpr};
 pub use column::UnKnownColumn;
 pub use datafusion_expr::utils::format_state_name;
 pub use datafusion_physical_expr_common::expressions::binary::{binary, BinaryExpr};
+pub use datafusion_physical_expr_common::expressions::cast::{
+    cast, cast_with_options, CastExpr,
+};
 pub use datafusion_physical_expr_common::expressions::column::{col, Column};
-pub use datafusion_physical_expr_common::expressions::literal::{lit, Literal};
-pub use datafusion_physical_expr_common::expressions::try_cast::{try_cast, TryCastExpr};
 pub use datafusion_physical_expr_common::expressions::in_list::{in_list, InListExpr};
-pub use is_not_null::{is_not_null, IsNotNullExpr};
-pub use is_null::{is_null, IsNullExpr};
+pub use datafusion_physical_expr_common::expressions::is_not_null::{
+    is_not_null, IsNotNullExpr,
+};
+pub use datafusion_physical_expr_common::expressions::is_null::{is_null, IsNullExpr};
 pub use datafusion_physical_expr_common::expressions::like::{like, LikeExpr};
-pub use datafusion_physical_expr_common::expressions::negative::{negative, NegativeExpr};
-pub use no_op::NoOp;
+pub use datafusion_physical_expr_common::expressions::literal::{lit, Literal};
+pub use datafusion_physical_expr_common::expressions::negative::{
+    negative, NegativeExpr,
+};
 pub use datafusion_physical_expr_common::expressions::not::{not, NotExpr};
+pub use datafusion_physical_expr_common::expressions::try_cast::{try_cast, TryCastExpr};
+pub use no_op::NoOp;
 
 #[cfg(test)]
 pub(crate) mod tests {

datafusion/physical-expr/src/physical_expr.rs

@@ -45,7 +45,6 @@ pub fn physical_exprs_equal(
     lhs.len() == rhs.len() && izip!(lhs, rhs).all(|(lhs, rhs)| lhs.eq(rhs))
 }
 
-
 /// This utility function removes duplicates from the given `exprs` vector.
 /// Note that this function does not necessarily preserve its input ordering.
 pub fn deduplicate_physical_exprs(exprs: &mut Vec<Arc<dyn PhysicalExpr>>) {