Skip to content

[SPARK-25832][SQL] remove newly added map related functions from FunctionRegistry #22821

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cloud-fan wants to merge 2 commits into from
Closed

[SPARK-25832][SQL] remove newly added map related functions from FunctionRegistry #22821

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7e919e3
remove newly added map related functions from FunctionRegistry
cloud-fan Oct 25, 2018
726fc30
fix tests
cloud-fan Oct 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,6 @@ object FunctionRegistry {
expression[MapFromArrays]("map_from_arrays"),
expression[MapKeys]("map_keys"),
expression[MapValues]("map_values"),
expression[MapEntries]("map_entries"),
Copy link
Member

@dongjoon-hyun dongjoon-hyun Oct 25, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To remove map_entries, we need to R together SPARK-24331. I'll include my PR to you, @cloud-fan .
cc @felixcheung

Copy link
Member

@dongjoon-hyun dongjoon-hyun Oct 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, we need to remove map_entires from functions.scala, too.

Copy link
Member

@dongjoon-hyun dongjoon-hyun Oct 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, cc @HyukjinKwon . We need to remove map_entries in Python.

expression[MapFromEntries]("map_from_entries"),
expression[MapConcat]("map_concat"),
expression[Size]("size"),
Expand All @@ -433,13 +432,9 @@ object FunctionRegistry {
expression[ArrayRemove]("array_remove"),
expression[ArrayDistinct]("array_distinct"),
expression[ArrayTransform]("transform"),
expression[MapFilter]("map_filter"),
expression[ArrayFilter]("filter"),
expression[ArrayExists]("exists"),
expression[ArrayAggregate]("aggregate"),
expression[TransformValues]("transform_values"),
expression[TransformKeys]("transform_keys"),
expression[MapZipWith]("map_zip_with"),
expression[ZipWith]("zip_with"),

CreateStruct.registryEntry,
Expand Down
23 changes: 0 additions & 23 deletions sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,26 +60,3 @@ select zip_with(array('a', 'b', 'c'), array('d', 'e', 'f'), (x, y) -> concat(x,

-- Zip with array coalesce
select zip_with(array('a'), array('d', null, 'f'), (x, y) -> coalesce(x, y)) as v;

create or replace temporary view nested as values
(1, map(1, 1, 2, 2, 3, 3)),
(2, map(4, 4, 5, 5, 6, 6))
as t(x, ys);

-- Identity Transform Keys in a map
select transform_keys(ys, (k, v) -> k) as v from nested;

-- Transform Keys in a map by adding constant
select transform_keys(ys, (k, v) -> k + 1) as v from nested;

-- Transform Keys in a map using values
select transform_keys(ys, (k, v) -> k + v) as v from nested;

-- Identity Transform values in a map
select transform_values(ys, (k, v) -> v) as v from nested;

-- Transform values in a map by adding constant
select transform_values(ys, (k, v) -> v + 1) as v from nested;

-- Transform values in a map using values
select transform_values(ys, (k, v) -> k + v) as v from nested;
66 changes: 1 addition & 65 deletions sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 27
-- Number of queries: 20


-- !query 0
Expand Down Expand Up @@ -189,67 +189,3 @@ select zip_with(array('a'), array('d', null, 'f'), (x, y) -> coalesce(x, y)) as
struct<v:array<string>>
-- !query 19 output
["a",null,"f"]


-- !query 20
create or replace temporary view nested as values
(1, map(1, 1, 2, 2, 3, 3)),
(2, map(4, 4, 5, 5, 6, 6))
as t(x, ys)
-- !query 20 schema
struct<>
-- !query 20 output


-- !query 21
select transform_keys(ys, (k, v) -> k) as v from nested
-- !query 21 schema
struct<v:map<int,int>>
-- !query 21 output
{1:1,2:2,3:3}
{4:4,5:5,6:6}


-- !query 22
select transform_keys(ys, (k, v) -> k + 1) as v from nested
-- !query 22 schema
struct<v:map<int,int>>
-- !query 22 output
{2:1,3:2,4:3}
{5:4,6:5,7:6}


-- !query 23
select transform_keys(ys, (k, v) -> k + v) as v from nested
-- !query 23 schema
struct<v:map<int,int>>
-- !query 23 output
{10:5,12:6,8:4}
{2:1,4:2,6:3}


-- !query 24
select transform_values(ys, (k, v) -> v) as v from nested
-- !query 24 schema
struct<v:map<int,int>>
-- !query 24 output
{1:1,2:2,3:3}
{4:4,5:5,6:6}


-- !query 25
select transform_values(ys, (k, v) -> v + 1) as v from nested
-- !query 25 schema
struct<v:map<int,int>>
-- !query 25 output
{1:2,2:3,3:4}
{4:5,5:6,6:7}


-- !query 26
select transform_values(ys, (k, v) -> k + v) as v from nested
-- !query 26 schema
struct<v:map<int,int>>
-- !query 26 output
{1:2,2:4,3:6}
{4:8,5:10,6:12}
22 changes: 11 additions & 11 deletions sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
)
}

test("map_entries") {
ignore("map_entries") {
// Primitive-type elements
val idf = Seq(
Map[Int, Int](1 -> 100, 2 -> 200, 3 -> 300),
Expand Down Expand Up @@ -2070,7 +2070,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
assert(ex3.getMessage.contains("cannot resolve '`a`'"))
}

test("map_filter") {
ignore("map_filter") {
val dfInts = Seq(
Map(1 -> 10, 2 -> 20, 3 -> 30),
Map(1 -> -1, 2 -> -2, 3 -> -3),
Expand Down Expand Up @@ -2449,7 +2449,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
assert(ex5.getMessage.contains("cannot resolve '`a`'"))
}

test("map_zip_with function - map of primitive types") {
ignore("map_zip_with function - map of primitive types") {
val df = Seq(
(Map(8 -> 6L, 3 -> 5L, 6 -> 2L), Map[Integer, Integer]((6, 4), (8, 2), (3, 2))),
(Map(10 -> 6L, 8 -> 3L), Map[Integer, Integer]((8, 4), (4, null))),
Expand All @@ -2465,7 +2465,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
Row(null)))
}

test("map_zip_with function - map of non-primitive types") {
ignore("map_zip_with function - map of non-primitive types") {
val df = Seq(
(Map("z" -> "a", "y" -> "b", "x" -> "c"), Map("x" -> "a", "z" -> "c")),
(Map("b" -> "a", "c" -> "d"), Map("c" -> "a", "b" -> null, "d" -> "k")),
Expand All @@ -2481,7 +2481,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
Row(null)))
}

test("map_zip_with function - invalid") {
ignore("map_zip_with function - invalid") {
val df = Seq(
(Map(1 -> 2), Map(1 -> "a"), Map("a" -> "b"), Map(Map(1 -> 2) -> 2), 1)
).toDF("mii", "mis", "mss", "mmi", "i")
Expand Down Expand Up @@ -2513,7 +2513,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
assert(ex5.getMessage.contains("function map_zip_with does not support ordering on type map"))
}

test("transform keys function - primitive data types") {
ignore("transform keys function - primitive data types") {
val dfExample1 = Seq(
Map[Int, Int](1 -> 1, 9 -> 9, 8 -> 8, 7 -> 7)
).toDF("i")
Expand Down Expand Up @@ -2568,7 +2568,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
testMapOfPrimitiveTypesCombination()
}

test("transform keys function - Invalid lambda functions and exceptions") {
ignore("transform keys function - Invalid lambda functions and exceptions") {
val dfExample1 = Seq(
Map[String, String]("a" -> null)
).toDF("i")
Expand Down Expand Up @@ -2600,7 +2600,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
"data type mismatch: argument 1 requires map type"))
}

test("transform values function - test primitive data types") {
ignore("transform values function - test primitive data types") {
val dfExample1 = Seq(
Map[Int, Int](1 -> 1, 9 -> 9, 8 -> 8, 7 -> 7)
).toDF("i")
Expand Down Expand Up @@ -2670,7 +2670,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
testMapOfPrimitiveTypesCombination()
}

test("transform values function - test empty") {
ignore("transform values function - test empty") {
val dfExample1 = Seq(
Map.empty[Integer, Integer]
).toDF("i")
Expand Down Expand Up @@ -2708,7 +2708,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
testEmpty()
}

test("transform values function - test null values") {
ignore("transform values function - test null values") {
val dfExample1 = Seq(
Map[Int, Integer](1 -> 1, 2 -> 2, 3 -> 3, 4 -> 4)
).toDF("a")
Expand All @@ -2732,7 +2732,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSQLContext {
testNullValue()
}

test("transform values function - test invalid functions") {
ignore("transform values function - test invalid functions") {
val dfExample1 = Seq(
Map[Int, Int](1 -> 1, 9 -> 9, 8 -> 8, 7 -> 7)
).toDF("i")
Expand Down
3 changes: 2 additions & 1 deletion sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext {
/** List of test cases to ignore, in lower cases. */
private val blackList = Set(
"blacklist.sql", // Do NOT remove this one. It is here to test the blacklist functionality.
".DS_Store" // A meta-file that may be created on Mac by Finder App.
".DS_Store", // A meta-file that may be created on Mac by Finder App.
// We should ignore this file from processing.
"mapZipWith.sql"
)

// Create all the test cases.
Expand Down