diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 695267ac130a0..efd760c785b40 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -414,7 +414,6 @@ object FunctionRegistry { expression[MapFromArrays]("map_from_arrays"), expression[MapKeys]("map_keys"), expression[MapValues]("map_values"), - expression[MapEntries]("map_entries"), expression[MapFromEntries]("map_from_entries"), expression[MapConcat]("map_concat"), expression[Size]("size"), @@ -433,13 +432,9 @@ object FunctionRegistry { expression[ArrayRemove]("array_remove"), expression[ArrayDistinct]("array_distinct"), expression[ArrayTransform]("transform"), - expression[MapFilter]("map_filter"), expression[ArrayFilter]("filter"), expression[ArrayExists]("exists"), expression[ArrayAggregate]("aggregate"), - expression[TransformValues]("transform_values"), - expression[TransformKeys]("transform_keys"), - expression[MapZipWith]("map_zip_with"), expression[ZipWith]("zip_with"), CreateStruct.registryEntry, diff --git a/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql index 02ad5e3538689..504306470faea 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql @@ -60,26 +60,3 @@ select zip_with(array('a', 'b', 'c'), array('d', 'e', 'f'), (x, y) -> concat(x, -- Zip with array coalesce select zip_with(array('a'), array('d', null, 'f'), (x, y) -> coalesce(x, y)) as v; - -create or replace temporary view nested as values - (1, map(1, 1, 2, 2, 3, 3)), - (2, map(4, 4, 5, 5, 6, 6)) - as t(x, ys); - --- Identity Transform Keys in a map -select transform_keys(ys, (k, v) -> k) as v from nested; - --- Transform Keys in a map by adding constant -select transform_keys(ys, (k, v) -> k + 1) as v from nested; - --- Transform Keys in a map using values -select transform_keys(ys, (k, v) -> k + v) as v from nested; - --- Identity Transform values in a map -select transform_values(ys, (k, v) -> v) as v from nested; - --- Transform values in a map by adding constant -select transform_values(ys, (k, v) -> v + 1) as v from nested; - --- Transform values in a map using values -select transform_values(ys, (k, v) -> k + v) as v from nested; diff --git a/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out index 32d20d1b73415..cdc509aeb16ca 100644 --- a/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 27 +-- Number of queries: 20 -- !query 0 @@ -189,67 +189,3 @@ select zip_with(array('a'), array('d', null, 'f'), (x, y) -> coalesce(x, y)) as struct> -- !query 19 output ["a",null,"f"] - - --- !query 20 -create or replace temporary view nested as values - (1, map(1, 1, 2, 2, 3, 3)), - (2, map(4, 4, 5, 5, 6, 6)) - as t(x, ys) --- !query 20 schema -struct<> --- !query 20 output - - --- !query 21 -select transform_keys(ys, (k, v) -> k) as v from nested --- !query 21 schema -struct> --- !query 21 output -{1:1,2:2,3:3} -{4:4,5:5,6:6} - - --- !query 22 -select transform_keys(ys, (k, v) -> k + 1) as v from nested --- !query 22 schema -struct> --- !query 22 output -{2:1,3:2,4:3} -{5:4,6:5,7:6} - - --- !query 23 -select transform_keys(ys, (k, v) -> k + v) as v from nested --- !query 23 schema -struct> --- !query 23 output -{10:5,12:6,8:4} -{2:1,4:2,6:3} - - --- !query 24 -select transform_values(ys, (k, v) -> v) as v from nested --- !query 24 schema -struct> --- !query 24 output -{1:1,2:2,3:3} -{4:4,5:5,6:6} - - --- !query 25 -select transform_values(ys, (k, v) -> v + 1) as v from nested --- !query 25 schema -struct> --- !query 25 output -{1:2,2:3,3:4} -{4:5,5:6,6:7} - - --- !query 26 -select transform_values(ys, (k, v) -> k + v) as v from nested --- !query 26 schema -struct> --- !query 26 output -{1:2,2:4,3:6} -{4:8,5:10,6:12} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala index 60ebc5e6cc09b..fef193b3dce92 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala @@ -509,7 +509,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext { ) } - test("map_entries") { + ignore("map_entries") { // Primitive-type elements val idf = Seq( Map[Int, Int](1 -> 100, 2 -> 200, 3 -> 300), @@ -2070,7 +2070,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext { assert(ex3.getMessage.contains("cannot resolve '`a`'")) } - test("map_filter") { + ignore("map_filter") { val dfInts = Seq( Map(1 -> 10, 2 -> 20, 3 -> 30), Map(1 -> -1, 2 -> -2, 3 -> -3), @@ -2449,7 +2449,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext { assert(ex5.getMessage.contains("cannot resolve '`a`'")) } - test("map_zip_with function - map of primitive types") { + ignore("map_zip_with function - map of primitive types") { val df = Seq( (Map(8 -> 6L, 3 -> 5L, 6 -> 2L), Map[Integer, Integer]((6, 4), (8, 2), (3, 2))), (Map(10 -> 6L, 8 -> 3L), Map[Integer, Integer]((8, 4), (4, null))), @@ -2465,7 +2465,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext { Row(null))) } - test("map_zip_with function - map of non-primitive types") { + ignore("map_zip_with function - map of non-primitive types") { val df = Seq( (Map("z" -> "a", "y" -> "b", "x" -> "c"), Map("x" -> "a", "z" -> "c")), (Map("b" -> "a", "c" -> "d"), Map("c" -> "a", "b" -> null, "d" -> "k")), @@ -2481,7 +2481,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext { Row(null))) } - test("map_zip_with function - invalid") { + ignore("map_zip_with function - invalid") { val df = Seq( (Map(1 -> 2), Map(1 -> "a"), Map("a" -> "b"), Map(Map(1 -> 2) -> 2), 1) ).toDF("mii", "mis", "mss", "mmi", "i") @@ -2513,7 +2513,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext { assert(ex5.getMessage.contains("function map_zip_with does not support ordering on type map")) } - test("transform keys function - primitive data types") { + ignore("transform keys function - primitive data types") { val dfExample1 = Seq( Map[Int, Int](1 -> 1, 9 -> 9, 8 -> 8, 7 -> 7) ).toDF("i") @@ -2568,7 +2568,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext { testMapOfPrimitiveTypesCombination() } - test("transform keys function - Invalid lambda functions and exceptions") { + ignore("transform keys function - Invalid lambda functions and exceptions") { val dfExample1 = Seq( Map[String, String]("a" -> null) ).toDF("i") @@ -2600,7 +2600,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext { "data type mismatch: argument 1 requires map type")) } - test("transform values function - test primitive data types") { + ignore("transform values function - test primitive data types") { val dfExample1 = Seq( Map[Int, Int](1 -> 1, 9 -> 9, 8 -> 8, 7 -> 7) ).toDF("i") @@ -2670,7 +2670,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext { testMapOfPrimitiveTypesCombination() } - test("transform values function - test empty") { + ignore("transform values function - test empty") { val dfExample1 = Seq( Map.empty[Integer, Integer] ).toDF("i") @@ -2708,7 +2708,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext { testEmpty() } - test("transform values function - test null values") { + ignore("transform values function - test null values") { val dfExample1 = Seq( Map[Int, Integer](1 -> 1, 2 -> 2, 3 -> 3, 4 -> 4) ).toDF("a") @@ -2732,7 +2732,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext { testNullValue() } - test("transform values function - test invalid functions") { + ignore("transform values function - test invalid functions") { val dfExample1 = Seq( Map[Int, Int](1 -> 1, 9 -> 9, 8 -> 8, 7 -> 7) ).toDF("i") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 826408c7161e9..493f44393a1a0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -101,8 +101,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { /** List of test cases to ignore, in lower cases. */ private val blackList = Set( "blacklist.sql", // Do NOT remove this one. It is here to test the blacklist functionality. - ".DS_Store" // A meta-file that may be created on Mac by Finder App. + ".DS_Store", // A meta-file that may be created on Mac by Finder App. // We should ignore this file from processing. + "mapZipWith.sql" ) // Create all the test cases.