From b6c246e3ce43e2da5da60dee8787a141ec42a501 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Fri, 18 Oct 2019 12:07:07 +0800 Subject: [PATCH 1/3] [SPARK-27879][SQL] Add support for bit_and and bit_or aggregates --- .../catalyst/analysis/FunctionRegistry.scala | 3 + .../aggregate/bitwiseAggregates.scala | 100 +++++++++++ .../inputs/postgreSQL/aggregates_part2.sql | 54 +++--- .../udf/postgreSQL/udf-aggregates_part2.sql | 54 +++--- .../postgreSQL/aggregates_part2.sql.out | 164 +++++++++++------- 5 files changed, 247 insertions(+), 128 deletions(-) create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index addeca507c0f..c6fd1f394ba0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -525,6 +525,9 @@ object FunctionRegistry { expression[BitwiseOr]("|"), expression[BitwiseXor]("^"), expression[BitwiseCount]("bit_count"), + expression[BitAndAgg]("bit_and"), + expression[BitOrAgg]("bit_or"), + // json expression[StructsToJson]("to_json"), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala new file mode 100644 index 000000000000..b2c4bf92673b --- /dev/null +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions.aggregate + +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BitwiseAnd, BitwiseOr, ExpectsInputTypes, Expression, ExpressionDescription, If, IsNotNull, IsNull, Literal, Nvl2} +import org.apache.spark.sql.types.{AbstractDataType, DataType, IntegralType} + +@ExpressionDescription( + usage = "_FUNC_(expr) - Returns the bitwise AND of all non-null input values, or null if none.", + examples = """ + Examples: + > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col); + 1 + """, + since = "3.0.0") +case class BitAndAgg(child: Expression) extends DeclarativeAggregate with ExpectsInputTypes { + + override def nodeName: String = "bit_and" + + override def children: Seq[Expression] = child :: Nil + + override def nullable: Boolean = true + + override def dataType: DataType = child.dataType + + override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) + + private lazy val bitAnd = AttributeReference("bit_and", child.dataType)() + + override lazy val aggBufferAttributes: Seq[AttributeReference] = bitAnd :: Nil + + override lazy val initialValues: Seq[Literal] = Literal.create(null, dataType) :: Nil + + override lazy val updateExpressions: Seq[Expression] = + If(IsNull(bitAnd), + child, + If(IsNull(child), bitAnd, BitwiseAnd(bitAnd, child))) :: Nil + + override lazy val mergeExpressions: Seq[Expression] = + If(IsNull(bitAnd.left), + bitAnd.right, + If(IsNull(bitAnd.right), bitAnd.left, BitwiseAnd(bitAnd.left, bitAnd.right))) :: Nil + + override lazy val evaluateExpression: AttributeReference = bitAnd + +} + +@ExpressionDescription( + usage = "_FUNC_(expr) - Returns the bitwise OR of all non-null input values, or null if none.", + examples = """ + Examples: + > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col); + 7 + """, + since = "3.0.0") +case class BitOrAgg(child: Expression) extends DeclarativeAggregate with ExpectsInputTypes { + + override def nodeName: String = "bit_or" + + override def children: Seq[Expression] = child :: Nil + + override def nullable: Boolean = true + + override def dataType: DataType = child.dataType + + override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) + + private lazy val bitOr = AttributeReference("bit_or", child.dataType)() + + override lazy val aggBufferAttributes: Seq[AttributeReference] = bitOr :: Nil + + override lazy val initialValues: Seq[Literal] = Literal.create(null, dataType) :: Nil + + override lazy val updateExpressions: Seq[Expression] = + If(IsNull(bitOr), + child, + If(IsNull(child), bitOr, BitwiseOr(bitOr, child))) :: Nil + + override lazy val mergeExpressions: Seq[Expression] = + If(IsNull(bitOr.left), + bitOr.right, + If(IsNull(bitOr.right), bitOr.left, BitwiseOr(bitOr.left, bitOr.right))) :: Nil + + override lazy val evaluateExpression: AttributeReference = bitOr +} diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part2.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part2.sql index 144a3bc2a38f..8ed1aeb03f36 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part2.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part2.sql @@ -41,42 +41,28 @@ create temporary view int4_tbl as select * from values -- -- test for bitwise integer aggregates -- --- CREATE TEMPORARY TABLE bitwise_test( --- i2 INT2, --- i4 INT4, --- i8 INT8, --- i INTEGER, --- x INT2, --- y BIT(4) --- ); +CREATE OR REPLACE TEMPORARY VIEW bitwise_test AS SELECT * FROM VALUES + (1, 1, 1, 1L), + (3, 3, 3, null), + (7, 7, 7, 3L) AS bitwise_test(b1, b2, b3, b4); -- empty case --- SELECT --- BIT_AND(i2) AS "?", --- BIT_OR(i4) AS "?" --- FROM bitwise_test; - --- COPY bitwise_test FROM STDIN NULL 'null'; --- 1 1 1 1 1 B0101 --- 3 3 3 null 2 B0100 --- 7 7 7 3 4 B1100 --- \. - --- SELECT --- BIT_AND(i2) AS "1", --- BIT_AND(i4) AS "1", --- BIT_AND(i8) AS "1", --- BIT_AND(i) AS "?", --- BIT_AND(x) AS "0", --- BIT_AND(y) AS "0100", --- --- BIT_OR(i2) AS "7", --- BIT_OR(i4) AS "7", --- BIT_OR(i8) AS "7", --- BIT_OR(i) AS "?", --- BIT_OR(x) AS "7", --- BIT_OR(y) AS "1101" --- FROM bitwise_test; +SELECT BIT_AND(b1) AS n1, BIT_OR(b2) AS n2 FROM bitwise_test where 1 = 0; + +-- null case +SELECT BIT_AND(b4) AS n1, BIT_OR(b4) AS n2 FROM bitwise_test where b4 is null; + + +SELECT + BIT_AND(cast(b1 as tinyint)) AS a1, + BIT_AND(cast(b2 as smallint)) AS b1, + BIT_AND(b3) AS c1, + BIT_AND(b4) AS d1, + BIT_OR(cast(b1 as tinyint)) AS e7, + BIT_OR(cast(b2 as smallint)) AS f7, + BIT_OR(b3) AS g7, + BIT_OR(b4) AS h3 +FROM bitwise_test; -- -- test boolean aggregates diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-aggregates_part2.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-aggregates_part2.sql index d82fcd90a22a..b4054850062b 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-aggregates_part2.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-aggregates_part2.sql @@ -43,42 +43,28 @@ create temporary view int4_tbl as select * from values -- -- test for bitwise integer aggregates -- --- CREATE TEMPORARY TABLE bitwise_test( --- i2 INT2, --- i4 INT4, --- i8 INT8, --- i INTEGER, --- x INT2, --- y BIT(4) --- ); +CREATE OR REPLACE TEMPORARY VIEW bitwise_test AS SELECT * FROM VALUES + (1, 1, 1, 1L), + (3, 3, 3, null), + (7, 7, 7, 3L) AS bitwise_test(b1, b2, b3, b4); -- empty case --- SELECT --- BIT_AND(i2) AS "?", --- BIT_OR(i4) AS "?" --- FROM bitwise_test; - --- COPY bitwise_test FROM STDIN NULL 'null'; --- 1 1 1 1 1 B0101 --- 3 3 3 null 2 B0100 --- 7 7 7 3 4 B1100 --- \. - --- SELECT --- BIT_AND(i2) AS "1", --- BIT_AND(i4) AS "1", --- BIT_AND(i8) AS "1", --- BIT_AND(i) AS "?", --- BIT_AND(x) AS "0", --- BIT_AND(y) AS "0100", --- --- BIT_OR(i2) AS "7", --- BIT_OR(i4) AS "7", --- BIT_OR(i8) AS "7", --- BIT_OR(i) AS "?", --- BIT_OR(x) AS "7", --- BIT_OR(y) AS "1101" --- FROM bitwise_test; +SELECT BIT_AND(b1) AS n1, BIT_OR(b2) AS n2 FROM bitwise_test where 1 = 0; + +-- null case +SELECT BIT_AND(b4) AS n1, BIT_OR(b4) AS n2 FROM bitwise_test where b4 is null; + + +SELECT + BIT_AND(cast(b1 as tinyint)) AS a1, + BIT_AND(cast(b2 as smallint)) AS b1, + BIT_AND(b3) AS c1, + BIT_AND(b4) AS d1, + BIT_OR(cast(b1 as tinyint)) AS e7, + BIT_OR(cast(b2 as smallint)) AS f7, + BIT_OR(b3) AS g7, + BIT_OR(b4) AS h3 +FROM bitwise_test; -- -- test boolean aggregates diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part2.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part2.sql.out index d5362809c804..72723aa8ceb7 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part2.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part2.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 21 +-- Number of queries: 25 -- !query 0 @@ -17,6 +17,50 @@ struct<> -- !query 1 +CREATE OR REPLACE TEMPORARY VIEW bitwise_test AS SELECT * FROM VALUES + (1, 1, 1, 1L), + (3, 3, 3, null), + (7, 7, 7, 3L) AS bitwise_test(b1, b2, b3, b4) +-- !query 1 schema +struct<> +-- !query 1 output + + + +-- !query 2 +SELECT BIT_AND(b1) AS n1, BIT_OR(b2) AS n2 FROM bitwise_test where 1 = 0 +-- !query 2 schema +struct +-- !query 2 output +NULL NULL + + +-- !query 3 +SELECT BIT_AND(b4) AS n1, BIT_OR(b4) AS n2 FROM bitwise_test where b4 is null +-- !query 3 schema +struct +-- !query 3 output +NULL NULL + + +-- !query 4 +SELECT + BIT_AND(cast(b1 as tinyint)) AS a1, + BIT_AND(cast(b2 as smallint)) AS b1, + BIT_AND(b3) AS c1, + BIT_AND(b4) AS d1, + BIT_OR(cast(b1 as tinyint)) AS e7, + BIT_OR(cast(b2 as smallint)) AS f7, + BIT_OR(b3) AS g7, + BIT_OR(b4) AS h3 +FROM bitwise_test +-- !query 4 schema +struct +-- !query 4 output +1 1 1 1 7 7 7 3 + + +-- !query 5 SELECT (NULL AND NULL) IS NULL AS `t`, (TRUE AND NULL) IS NULL AS `t`, @@ -27,13 +71,13 @@ SELECT NOT (TRUE AND FALSE) AS `t`, NOT (FALSE AND TRUE) AS `t`, NOT (FALSE AND FALSE) AS `t` --- !query 1 schema +-- !query 5 schema struct --- !query 1 output +-- !query 5 output true true false true false true true true true --- !query 2 +-- !query 6 SELECT (NULL OR NULL) IS NULL AS `t`, (TRUE OR NULL) IS NULL AS `t`, @@ -44,32 +88,32 @@ SELECT (TRUE OR FALSE) AS `t`, (FALSE OR TRUE) AS `t`, NOT (FALSE OR FALSE) AS `t` --- !query 2 schema +-- !query 6 schema struct --- !query 2 output +-- !query 6 output true false true false true true true true true --- !query 3 +-- !query 7 CREATE OR REPLACE TEMPORARY VIEW bool_test AS SELECT * FROM VALUES (TRUE, null, FALSE, null), (FALSE, TRUE, null, null), (null, TRUE, FALSE, null) AS bool_test(b1, b2, b3, b4) --- !query 3 schema +-- !query 7 schema struct<> --- !query 3 output +-- !query 7 output --- !query 4 +-- !query 8 SELECT BOOL_AND(b1) AS n1, BOOL_OR(b3) AS n2 FROM bool_test WHERE 1 = 0 --- !query 4 schema +-- !query 8 schema struct --- !query 4 output +-- !query 8 output NULL NULL --- !query 5 +-- !query 9 SELECT BOOL_AND(b1) AS f1, BOOL_AND(b2) AS t2, @@ -78,13 +122,13 @@ SELECT BOOL_AND(NOT b2) AS f5, BOOL_AND(NOT b3) AS t6 FROM bool_test --- !query 5 schema +-- !query 9 schema struct --- !query 5 output +-- !query 9 output false true false NULL false true --- !query 6 +-- !query 10 SELECT EVERY(b1) AS f1, EVERY(b2) AS t2, @@ -93,13 +137,13 @@ SELECT EVERY(NOT b2) AS f5, EVERY(NOT b3) AS t6 FROM bool_test --- !query 6 schema +-- !query 10 schema struct --- !query 6 output +-- !query 10 output false true false NULL false true --- !query 7 +-- !query 11 SELECT BOOL_OR(b1) AS t1, BOOL_OR(b2) AS t2, @@ -108,113 +152,113 @@ SELECT BOOL_OR(NOT b2) AS f5, BOOL_OR(NOT b3) AS t6 FROM bool_test --- !query 7 schema +-- !query 11 schema struct --- !query 7 output +-- !query 11 output true true false NULL false true --- !query 8 +-- !query 12 select min(unique1) from tenk1 --- !query 8 schema +-- !query 12 schema struct --- !query 8 output +-- !query 12 output 0 --- !query 9 +-- !query 13 select max(unique1) from tenk1 --- !query 9 schema +-- !query 13 schema struct --- !query 9 output +-- !query 13 output 9999 --- !query 10 +-- !query 14 select max(unique1) from tenk1 where unique1 < 42 --- !query 10 schema +-- !query 14 schema struct --- !query 10 output +-- !query 14 output 41 --- !query 11 +-- !query 15 select max(unique1) from tenk1 where unique1 > 42 --- !query 11 schema +-- !query 15 schema struct --- !query 11 output +-- !query 15 output 9999 --- !query 12 +-- !query 16 select max(unique1) from tenk1 where unique1 > 42000 --- !query 12 schema +-- !query 16 schema struct --- !query 12 output +-- !query 16 output NULL --- !query 13 +-- !query 17 select max(tenthous) from tenk1 where thousand = 33 --- !query 13 schema +-- !query 17 schema struct --- !query 13 output +-- !query 17 output 9033 --- !query 14 +-- !query 18 select min(tenthous) from tenk1 where thousand = 33 --- !query 14 schema +-- !query 18 schema struct --- !query 14 output +-- !query 18 output 33 --- !query 15 +-- !query 19 select distinct max(unique2) from tenk1 --- !query 15 schema +-- !query 19 schema struct --- !query 15 output +-- !query 19 output 9999 --- !query 16 +-- !query 20 select max(unique2) from tenk1 order by 1 --- !query 16 schema +-- !query 20 schema struct --- !query 16 output +-- !query 20 output 9999 --- !query 17 +-- !query 21 select max(unique2) from tenk1 order by max(unique2) --- !query 17 schema +-- !query 21 schema struct --- !query 17 output +-- !query 21 output 9999 --- !query 18 +-- !query 22 select max(unique2) from tenk1 order by max(unique2)+1 --- !query 18 schema +-- !query 22 schema struct --- !query 18 output +-- !query 22 output 9999 --- !query 19 +-- !query 23 select t1.max_unique2, g from (select max(unique2) as max_unique2 FROM tenk1) t1 LATERAL VIEW explode(array(1,2,3)) t2 AS g order by g desc --- !query 19 schema +-- !query 23 schema struct --- !query 19 output +-- !query 23 output 9999 3 9999 2 9999 1 --- !query 20 +-- !query 24 select max(100) from tenk1 --- !query 20 schema +-- !query 24 schema struct --- !query 20 output +-- !query 24 output 100 From 58b99e92e13a7bf99450bf3233945388cbc496d4 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Fri, 18 Oct 2019 16:31:00 +0800 Subject: [PATCH 2/3] re gren golden file and add window /group by tests --- .../catalyst/analysis/FunctionRegistry.scala | 1 - .../inputs/postgreSQL/aggregates_part2.sql | 11 +- .../postgreSQL/aggregates_part2.sql.out | 159 ++++++++++------- .../postgreSQL/udf-aggregates_part2.sql.out | 164 +++++++++++------- 4 files changed, 213 insertions(+), 122 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index c6fd1f394ba0..04e8963944fd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -528,7 +528,6 @@ object FunctionRegistry { expression[BitAndAgg]("bit_and"), expression[BitOrAgg]("bit_or"), - // json expression[StructsToJson]("to_json"), expression[JsonToStructs]("from_json"), diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part2.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part2.sql index 8ed1aeb03f36..ba91366014e1 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part2.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part2.sql @@ -52,7 +52,6 @@ SELECT BIT_AND(b1) AS n1, BIT_OR(b2) AS n2 FROM bitwise_test where 1 = 0; -- null case SELECT BIT_AND(b4) AS n1, BIT_OR(b4) AS n2 FROM bitwise_test where b4 is null; - SELECT BIT_AND(cast(b1 as tinyint)) AS a1, BIT_AND(cast(b2 as smallint)) AS b1, @@ -64,6 +63,16 @@ SELECT BIT_OR(b4) AS h3 FROM bitwise_test; +-- group by +SELECT b1 , bit_and(b2), bit_or(b4) FROM bitwise_test GROUP BY b1; + +--having +SELECT b1, bit_and(b2) FROM bitwise_test GROUP BY b1 HAVING bit_and(b2) < 7; + +-- window +SELECT b1, b2, bit_and(b2) OVER (PARTITION BY b1 ORDER BY b2) FROM bitwise_test; +SELECT b1, b2, bit_or(b2) OVER (PARTITION BY b1 ORDER BY b2) FROM bitwise_test; + -- -- test boolean aggregates -- diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part2.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part2.sql.out index 72723aa8ceb7..fdca51ff1325 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part2.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part2.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 25 +-- Number of queries: 29 -- !query 0 @@ -61,6 +61,45 @@ struct +-- !query 5 output +1 1 1 +3 3 NULL +7 7 3 + + +-- !query 6 +SELECT b1, bit_and(b2) FROM bitwise_test GROUP BY b1 HAVING bit_and(b2) < 7 +-- !query 6 schema +struct +-- !query 6 output +1 1 +3 3 + + +-- !query 7 +SELECT b1, b2, bit_and(b2) OVER (PARTITION BY b1 ORDER BY b2) FROM bitwise_test +-- !query 7 schema +struct +-- !query 7 output +1 1 1 +3 3 3 +7 7 7 + + +-- !query 8 +SELECT b1, b2, bit_or(b2) OVER (PARTITION BY b1 ORDER BY b2) FROM bitwise_test +-- !query 8 schema +struct +-- !query 8 output +1 1 1 +3 3 3 +7 7 7 + + +-- !query 9 SELECT (NULL AND NULL) IS NULL AS `t`, (TRUE AND NULL) IS NULL AS `t`, @@ -71,13 +110,13 @@ SELECT NOT (TRUE AND FALSE) AS `t`, NOT (FALSE AND TRUE) AS `t`, NOT (FALSE AND FALSE) AS `t` --- !query 5 schema +-- !query 9 schema struct --- !query 5 output +-- !query 9 output true true false true false true true true true --- !query 6 +-- !query 10 SELECT (NULL OR NULL) IS NULL AS `t`, (TRUE OR NULL) IS NULL AS `t`, @@ -88,32 +127,32 @@ SELECT (TRUE OR FALSE) AS `t`, (FALSE OR TRUE) AS `t`, NOT (FALSE OR FALSE) AS `t` --- !query 6 schema +-- !query 10 schema struct --- !query 6 output +-- !query 10 output true false true false true true true true true --- !query 7 +-- !query 11 CREATE OR REPLACE TEMPORARY VIEW bool_test AS SELECT * FROM VALUES (TRUE, null, FALSE, null), (FALSE, TRUE, null, null), (null, TRUE, FALSE, null) AS bool_test(b1, b2, b3, b4) --- !query 7 schema +-- !query 11 schema struct<> --- !query 7 output +-- !query 11 output --- !query 8 +-- !query 12 SELECT BOOL_AND(b1) AS n1, BOOL_OR(b3) AS n2 FROM bool_test WHERE 1 = 0 --- !query 8 schema +-- !query 12 schema struct --- !query 8 output +-- !query 12 output NULL NULL --- !query 9 +-- !query 13 SELECT BOOL_AND(b1) AS f1, BOOL_AND(b2) AS t2, @@ -122,13 +161,13 @@ SELECT BOOL_AND(NOT b2) AS f5, BOOL_AND(NOT b3) AS t6 FROM bool_test --- !query 9 schema +-- !query 13 schema struct --- !query 9 output +-- !query 13 output false true false NULL false true --- !query 10 +-- !query 14 SELECT EVERY(b1) AS f1, EVERY(b2) AS t2, @@ -137,13 +176,13 @@ SELECT EVERY(NOT b2) AS f5, EVERY(NOT b3) AS t6 FROM bool_test --- !query 10 schema +-- !query 14 schema struct --- !query 10 output +-- !query 14 output false true false NULL false true --- !query 11 +-- !query 15 SELECT BOOL_OR(b1) AS t1, BOOL_OR(b2) AS t2, @@ -152,113 +191,113 @@ SELECT BOOL_OR(NOT b2) AS f5, BOOL_OR(NOT b3) AS t6 FROM bool_test --- !query 11 schema +-- !query 15 schema struct --- !query 11 output +-- !query 15 output true true false NULL false true --- !query 12 +-- !query 16 select min(unique1) from tenk1 --- !query 12 schema +-- !query 16 schema struct --- !query 12 output +-- !query 16 output 0 --- !query 13 +-- !query 17 select max(unique1) from tenk1 --- !query 13 schema +-- !query 17 schema struct --- !query 13 output +-- !query 17 output 9999 --- !query 14 +-- !query 18 select max(unique1) from tenk1 where unique1 < 42 --- !query 14 schema +-- !query 18 schema struct --- !query 14 output +-- !query 18 output 41 --- !query 15 +-- !query 19 select max(unique1) from tenk1 where unique1 > 42 --- !query 15 schema +-- !query 19 schema struct --- !query 15 output +-- !query 19 output 9999 --- !query 16 +-- !query 20 select max(unique1) from tenk1 where unique1 > 42000 --- !query 16 schema +-- !query 20 schema struct --- !query 16 output +-- !query 20 output NULL --- !query 17 +-- !query 21 select max(tenthous) from tenk1 where thousand = 33 --- !query 17 schema +-- !query 21 schema struct --- !query 17 output +-- !query 21 output 9033 --- !query 18 +-- !query 22 select min(tenthous) from tenk1 where thousand = 33 --- !query 18 schema +-- !query 22 schema struct --- !query 18 output +-- !query 22 output 33 --- !query 19 +-- !query 23 select distinct max(unique2) from tenk1 --- !query 19 schema +-- !query 23 schema struct --- !query 19 output +-- !query 23 output 9999 --- !query 20 +-- !query 24 select max(unique2) from tenk1 order by 1 --- !query 20 schema +-- !query 24 schema struct --- !query 20 output +-- !query 24 output 9999 --- !query 21 +-- !query 25 select max(unique2) from tenk1 order by max(unique2) --- !query 21 schema +-- !query 25 schema struct --- !query 21 output +-- !query 25 output 9999 --- !query 22 +-- !query 26 select max(unique2) from tenk1 order by max(unique2)+1 --- !query 22 schema +-- !query 26 schema struct --- !query 22 output +-- !query 26 output 9999 --- !query 23 +-- !query 27 select t1.max_unique2, g from (select max(unique2) as max_unique2 FROM tenk1) t1 LATERAL VIEW explode(array(1,2,3)) t2 AS g order by g desc --- !query 23 schema +-- !query 27 schema struct --- !query 23 output +-- !query 27 output 9999 3 9999 2 9999 1 --- !query 24 +-- !query 28 select max(100) from tenk1 --- !query 24 schema +-- !query 28 schema struct --- !query 24 output +-- !query 28 output 100 diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out index 9efc7f65fdfc..3b54637c449d 100644 --- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 21 +-- Number of queries: 25 -- !query 0 @@ -17,6 +17,50 @@ struct<> -- !query 1 +CREATE OR REPLACE TEMPORARY VIEW bitwise_test AS SELECT * FROM VALUES + (1, 1, 1, 1L), + (3, 3, 3, null), + (7, 7, 7, 3L) AS bitwise_test(b1, b2, b3, b4) +-- !query 1 schema +struct<> +-- !query 1 output + + + +-- !query 2 +SELECT BIT_AND(b1) AS n1, BIT_OR(b2) AS n2 FROM bitwise_test where 1 = 0 +-- !query 2 schema +struct +-- !query 2 output +NULL NULL + + +-- !query 3 +SELECT BIT_AND(b4) AS n1, BIT_OR(b4) AS n2 FROM bitwise_test where b4 is null +-- !query 3 schema +struct +-- !query 3 output +NULL NULL + + +-- !query 4 +SELECT + BIT_AND(cast(b1 as tinyint)) AS a1, + BIT_AND(cast(b2 as smallint)) AS b1, + BIT_AND(b3) AS c1, + BIT_AND(b4) AS d1, + BIT_OR(cast(b1 as tinyint)) AS e7, + BIT_OR(cast(b2 as smallint)) AS f7, + BIT_OR(b3) AS g7, + BIT_OR(b4) AS h3 +FROM bitwise_test +-- !query 4 schema +struct +-- !query 4 output +1 1 1 1 7 7 7 3 + + +-- !query 5 SELECT (NULL AND NULL) IS NULL AS `t`, (TRUE AND NULL) IS NULL AS `t`, @@ -27,13 +71,13 @@ SELECT NOT (TRUE AND FALSE) AS `t`, NOT (FALSE AND TRUE) AS `t`, NOT (FALSE AND FALSE) AS `t` --- !query 1 schema +-- !query 5 schema struct --- !query 1 output +-- !query 5 output true true false true false true true true true --- !query 2 +-- !query 6 SELECT (NULL OR NULL) IS NULL AS `t`, (TRUE OR NULL) IS NULL AS `t`, @@ -44,32 +88,32 @@ SELECT (TRUE OR FALSE) AS `t`, (FALSE OR TRUE) AS `t`, NOT (FALSE OR FALSE) AS `t` --- !query 2 schema +-- !query 6 schema struct --- !query 2 output +-- !query 6 output true false true false true true true true true --- !query 3 +-- !query 7 CREATE OR REPLACE TEMPORARY VIEW bool_test AS SELECT * FROM VALUES (TRUE, null, FALSE, null), (FALSE, TRUE, null, null), (null, TRUE, FALSE, null) AS bool_test(b1, b2, b3, b4) --- !query 3 schema +-- !query 7 schema struct<> --- !query 3 output +-- !query 7 output --- !query 4 +-- !query 8 SELECT BOOL_AND(b1) AS n1, BOOL_OR(b3) AS n2 FROM bool_test WHERE 1 = 0 --- !query 4 schema +-- !query 8 schema struct --- !query 4 output +-- !query 8 output NULL NULL --- !query 5 +-- !query 9 SELECT BOOL_AND(b1) AS f1, BOOL_AND(b2) AS t2, @@ -78,13 +122,13 @@ SELECT BOOL_AND(NOT b2) AS f5, BOOL_AND(NOT b3) AS t6 FROM bool_test --- !query 5 schema +-- !query 9 schema struct --- !query 5 output +-- !query 9 output false true false NULL false true --- !query 6 +-- !query 10 SELECT EVERY(b1) AS f1, EVERY(b2) AS t2, @@ -93,13 +137,13 @@ SELECT EVERY(NOT b2) AS f5, EVERY(NOT b3) AS t6 FROM bool_test --- !query 6 schema +-- !query 10 schema struct --- !query 6 output +-- !query 10 output false true false NULL false true --- !query 7 +-- !query 11 SELECT BOOL_OR(b1) AS t1, BOOL_OR(b2) AS t2, @@ -108,113 +152,113 @@ SELECT BOOL_OR(NOT b2) AS f5, BOOL_OR(NOT b3) AS t6 FROM bool_test --- !query 7 schema +-- !query 11 schema struct --- !query 7 output +-- !query 11 output true true false NULL false true --- !query 8 +-- !query 12 select min(udf(unique1)) from tenk1 --- !query 8 schema +-- !query 12 schema struct --- !query 8 output +-- !query 12 output 0 --- !query 9 +-- !query 13 select udf(max(unique1)) from tenk1 --- !query 9 schema +-- !query 13 schema struct --- !query 9 output +-- !query 13 output 9999 --- !query 10 +-- !query 14 select max(unique1) from tenk1 where udf(unique1) < 42 --- !query 10 schema +-- !query 14 schema struct --- !query 10 output +-- !query 14 output 41 --- !query 11 +-- !query 15 select max(unique1) from tenk1 where unique1 > udf(42) --- !query 11 schema +-- !query 15 schema struct --- !query 11 output +-- !query 15 output 9999 --- !query 12 +-- !query 16 select max(unique1) from tenk1 where udf(unique1) > 42000 --- !query 12 schema +-- !query 16 schema struct --- !query 12 output +-- !query 16 output NULL --- !query 13 +-- !query 17 select max(tenthous) from tenk1 where udf(thousand) = 33 --- !query 13 schema +-- !query 17 schema struct --- !query 13 output +-- !query 17 output 9033 --- !query 14 +-- !query 18 select min(tenthous) from tenk1 where udf(thousand) = 33 --- !query 14 schema +-- !query 18 schema struct --- !query 14 output +-- !query 18 output 33 --- !query 15 +-- !query 19 select distinct max(udf(unique2)) from tenk1 --- !query 15 schema +-- !query 19 schema struct --- !query 15 output +-- !query 19 output 9999 --- !query 16 +-- !query 20 select max(unique2) from tenk1 order by udf(1) --- !query 16 schema +-- !query 20 schema struct --- !query 16 output +-- !query 20 output 9999 --- !query 17 +-- !query 21 select max(unique2) from tenk1 order by max(udf(unique2)) --- !query 17 schema +-- !query 21 schema struct --- !query 17 output +-- !query 21 output 9999 --- !query 18 +-- !query 22 select udf(max(udf(unique2))) from tenk1 order by udf(max(unique2))+1 --- !query 18 schema +-- !query 22 schema struct --- !query 18 output +-- !query 22 output 9999 --- !query 19 +-- !query 23 select t1.max_unique2, udf(g) from (select max(udf(unique2)) as max_unique2 FROM tenk1) t1 LATERAL VIEW explode(array(1,2,3)) t2 AS g order by g desc --- !query 19 schema +-- !query 23 schema struct --- !query 19 output +-- !query 23 output 9999 3 9999 2 9999 1 --- !query 20 +-- !query 24 select udf(max(100)) from tenk1 --- !query 20 schema +-- !query 24 schema struct --- !query 20 output +-- !query 24 output 100 From e91c364a45bdd17f7b3f2b84d426ae737a524ab0 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 21 Oct 2019 10:06:33 +0800 Subject: [PATCH 3/3] imports --- .../sql/catalyst/expressions/aggregate/bitwiseAggregates.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala index b2c4bf92673b..131fa2eb5055 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.catalyst.expressions.aggregate -import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BitwiseAnd, BitwiseOr, ExpectsInputTypes, Expression, ExpressionDescription, If, IsNotNull, IsNull, Literal, Nvl2} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BitwiseAnd, BitwiseOr, ExpectsInputTypes, Expression, ExpressionDescription, If, IsNull, Literal} import org.apache.spark.sql.types.{AbstractDataType, DataType, IntegralType} @ExpressionDescription( @@ -57,7 +57,6 @@ case class BitAndAgg(child: Expression) extends DeclarativeAggregate with Expect If(IsNull(bitAnd.right), bitAnd.left, BitwiseAnd(bitAnd.left, bitAnd.right))) :: Nil override lazy val evaluateExpression: AttributeReference = bitAnd - } @ExpressionDescription(