Skip to content

[SPARK-24381][TESTING] Add unit tests for NOT IN subquery around null values #21425

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mgyucht wants to merge 5 commits into from
Closed

[SPARK-24381][TESTING] Add unit tests for NOT IN subquery around null values #21425

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d6040ea
Add tests, and small clean-up of the NOT IN pathway
mgyucht May 24, 2018
45962d6
Port to SQLQueryTestSuite
mgyucht May 25, 2018
b37aa3e
Add tests with literals
mgyucht May 25, 2018
4a55c18
Remove broken queries
mgyucht May 25, 2018
8bb8963
Remove one last broken query
mgyucht May 26, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add tests with literals
  • Loading branch information
@mgyucht
mgyucht committed May 25, 2018
commit b37aa3e9f90d953c636af5dd161a4a66e64ff6f1
58 changes: 58 additions & 0 deletions ...esources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
-- Unit tests for simple NOT IN predicate subquery across multiple columns.
--
-- See not-in-single-column-unit-tests.sql for an introduction.
-- This file has the same test cases as not-in-unit-tests-multi-column.sql with literals instead of
-- subqueries. Small changes have been made to the literals to make them typecheck.

CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES
(null, null),
(null, 1.0),
(2, 3.0),
(4, 5.0)
AS m(a, b);

-- Case 1 (not possible to write a literal with no rows, so we ignore it.)
-- (subquery is empty -> row is returned)

-- Case 2
-- (subquery contains a row with null in all columns -> row not returned)
SELECT *
FROM m
WHERE (a, b) NOT IN ((CAST (null AS INT), CAST (null AS DECIMAL(2, 1))));

-- Case 3
-- (probe-side columns are all null -> row not returned)
SELECT *
FROM m
WHERE a IS NULL AND b IS NULL -- Matches only (null, null)
AND (a, b) NOT IN ((0, 1.0), (2, 3.0), (4, CAST(null AS DECIMAL(2, 1))));

-- Case 4
-- (one column null, other column matches a row in the subquery result -> row not returned)
SELECT *
FROM m
WHERE b = 1.0 -- Matches (null, 1.0)
AND (a, b) NOT IN ((0, 1.0), (2, 3.0), (4, CAST(null AS DECIMAL(2, 1))));

-- Case 5
-- (one null column with no match -> row is returned)
SELECT *
FROM m
WHERE b = 1.0 -- Matches (null, 1.0)
AND (a, b) NOT IN ((2, 3.0));

-- Case 6
-- (no null columns with match -> row not returned)
SELECT *
FROM m
WHERE b = 3.0 -- Matches (2, 3.0)
AND (a, b) NOT IN ((2, 3.0))
;

-- Case 7
-- (no null columns with no match -> row is returned)
SELECT *
FROM m
WHERE b = 5.0 -- Matches (4, 5.0)
AND (a, b) NOT IN ((2, 3.0))
;
Copy link
Member

@gatorsmile gatorsmile May 26, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: please also move ; to the above line. The same to all the other cases

42 changes: 42 additions & 0 deletions ...sources/sql-tests/inputs/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
-- Unit tests for simple NOT IN with a literal expression of a single column
--
-- More information can be found in not-in-unit-tests-single-column.sql.
-- This file has the same test cases as not-in-unit-tests-single-column.sql with literals instead of
-- subqueries.

CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES
(null, 1.0),
(2, 3.0),
(4, 5.0)
AS m(a, b);

-- Uncorrelated NOT IN Subquery test cases
-- Case 1 (not possible to write a literal with no rows, so we ignore it.)
-- (empty subquery -> all rows returned)

-- Case 2
-- (subquery includes null -> no rows returned)
SELECT *
FROM m
WHERE a NOT IN (null);

-- Case 3
-- (probe column is null -> row not returned)
SELECT *
FROM m
WHERE b = 1.0 -- Only matches (null, 1.0)
AND a NOT IN (2);

-- Case 4
-- (probe column matches subquery row -> row not returned)
SELECT *
FROM m
WHERE b = 3.0 -- Only matches (2, 3.0)
AND a NOT IN (2);

-- Case 5
-- (probe column does not match subquery row -> row is returned)
SELECT *
FROM m
WHERE b = 3.0 -- Only matches (2, 3.0)
AND a NOT IN (6);
97 changes: 97 additions & 0 deletions ...ces/sql-tests/results/subquery/in-subquery/not-in-unit-tests-multi-column-literal.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 7


-- !query 0
CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES
(null, null),
(null, 1.0),
(2, 3.0),
(4, 5.0)
AS m(a, b)
-- !query 0 schema
struct<>
-- !query 0 output



-- !query 1
-- Case 1 (not possible to write a literal with no rows, so we ignore it.)
-- (subquery is empty -> row is returned)

-- Case 2
-- (subquery contains a row with null in all columns -> row not returned)
SELECT *
FROM m
WHERE (a, b) NOT IN ((CAST (null AS INT), CAST (null AS DECIMAL(2, 1))))
-- !query 1 schema
struct<a:int,b:decimal(2,1)>
-- !query 1 output
2 3
4 5
NULL 1
Copy link
Member

@gatorsmile gatorsmile May 26, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This result is wrong, right?

Copy link
Contributor Author

@mgyucht mgyucht May 26, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup, I added it to the follow-up JIRA.



-- !query 2
-- Case 3
-- (probe-side columns are all null -> row not returned)
SELECT *
FROM m
WHERE a IS NULL AND b IS NULL -- Matches only (null, null)
AND (a, b) NOT IN ((0, 1.0), (2, 3.0), (4, CAST(null AS DECIMAL(2, 1))))
-- !query 2 schema
struct<a:int,b:decimal(2,1)>
-- !query 2 output
NULL NULL
Copy link
Contributor Author

@mgyucht mgyucht May 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is actually a wrong result, it should not be included in the output. cc @hvanhovell @gatorsmile

Copy link
Member

@gatorsmile gatorsmile May 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please open a JIRA in Apache Spark.

Copy link
Contributor Author

@mgyucht mgyucht May 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Filed https://issues.apache.org/jira/browse/SPARK-24395 to fix this.



-- !query 3
-- Case 4
-- (one column null, other column matches a row in the subquery result -> row not returned)
SELECT *
FROM m
WHERE b = 1.0 -- Matches (null, 1.0)
AND (a, b) NOT IN ((0, 1.0), (2, 3.0), (4, CAST(null AS DECIMAL(2, 1))))
-- !query 3 schema
struct<a:int,b:decimal(2,1)>
-- !query 3 output
NULL 1
Copy link
Contributor Author

@mgyucht mgyucht May 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here.

Copy link
Contributor Author

@mgyucht mgyucht May 25, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Filed https://issues.apache.org/jira/browse/SPARK-24395 to fix this.



-- !query 4
-- Case 5
-- (one null column with no match -> row is returned)
SELECT *
FROM m
WHERE b = 1.0 -- Matches (null, 1.0)
AND (a, b) NOT IN ((2, 3.0))
-- !query 4 schema
struct<a:int,b:decimal(2,1)>
-- !query 4 output
NULL 1


-- !query 5
-- Case 6
-- (no null columns with match -> row not returned)
SELECT *
FROM m
WHERE b = 3.0 -- Matches (2, 3.0)
AND (a, b) NOT IN ((2, 3.0))
-- !query 5 schema
struct<a:int,b:decimal(2,1)>
-- !query 5 output



-- !query 6
-- Case 7
-- (no null columns with no match -> row is returned)
SELECT *
FROM m
WHERE b = 5.0 -- Matches (4, 5.0)
AND (a, b) NOT IN ((2, 3.0))
-- !query 6 schema
struct<a:int,b:decimal(2,1)>
-- !query 6 output
4 5
69 changes: 69 additions & 0 deletions ...es/sql-tests/results/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 5


-- !query 0
CREATE TEMPORARY VIEW m AS SELECT * FROM VALUES
(null, 1.0),
(2, 3.0),
(4, 5.0)
AS m(a, b)
-- !query 0 schema
struct<>
-- !query 0 output



-- !query 1
-- Uncorrelated NOT IN Subquery test cases
-- Case 1 (not possible to write a literal with no rows, so we ignore it.)
-- (empty subquery -> all rows returned)

-- Case 2
-- (subquery includes null -> no rows returned)
SELECT *
FROM m
WHERE a NOT IN (null)
-- !query 1 schema
struct<a:int,b:decimal(2,1)>
-- !query 1 output



-- !query 2
-- Case 3
-- (probe column is null -> row not returned)
SELECT *
FROM m
WHERE b = 1.0 -- Only matches (null, 1.0)
AND a NOT IN (2)
-- !query 2 schema
struct<a:int,b:decimal(2,1)>
-- !query 2 output



-- !query 3
-- Case 4
-- (probe column matches subquery row -> row not returned)
SELECT *
FROM m
WHERE b = 3.0 -- Only matches (2, 3.0)
AND a NOT IN (2)
-- !query 3 schema
struct<a:int,b:decimal(2,1)>
-- !query 3 output



-- !query 4
-- Case 5
-- (probe column does not match subquery row -> row is returned)
SELECT *
FROM m
WHERE b = 3.0 -- Only matches (2, 3.0)
AND a NOT IN (6)
-- !query 4 schema
struct<a:int,b:decimal(2,1)>
-- !query 4 output
2 3