Skip to content

Commit 61c443a

Browse files
maropucloud-fan
authored andcommitted
[SPARK-26262][SQL] Runs SQLQueryTestSuite on mixed config sets: WHOLESTAGE_CODEGEN_ENABLED and CODEGEN_FACTORY_MODE
## What changes were proposed in this pull request? For better test coverage, this pr proposed to use the 4 mixed config sets of `WHOLESTAGE_CODEGEN_ENABLED` and `CODEGEN_FACTORY_MODE` when running `SQLQueryTestSuite`: 1. WHOLESTAGE_CODEGEN_ENABLED=true, CODEGEN_FACTORY_MODE=CODEGEN_ONLY 2. WHOLESTAGE_CODEGEN_ENABLED=false, CODEGEN_FACTORY_MODE=CODEGEN_ONLY 3. WHOLESTAGE_CODEGEN_ENABLED=true, CODEGEN_FACTORY_MODE=NO_CODEGEN 4. WHOLESTAGE_CODEGEN_ENABLED=false, CODEGEN_FACTORY_MODE=NO_CODEGEN This pr also moved some existing tests into `ExplainSuite` because explain output results are different between codegen and interpreter modes. ## How was this patch tested? Existing tests. Closes apache#23213 from maropu/InterpreterModeTest. Authored-by: Takeshi Yamamuro <[email protected]> Signed-off-by: Wenchen Fan <[email protected]>
1 parent 08f74ad commit 61c443a

14 files changed

+281
-469
lines changed

sql/core/src/test/resources/sql-tests/inputs/group-by.sql

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,3 @@ SELECT every("true");
141141
SELECT k, v, every(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg;
142142
SELECT k, v, some(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg;
143143
SELECT k, v, any(v) OVER (PARTITION BY k ORDER BY v) FROM test_agg;
144-
145-
-- simple explain of queries having every/some/any agregates. Optimized
146-
-- plan should show the rewritten aggregate expression.
147-
EXPLAIN EXTENDED SELECT k, every(v), some(v), any(v) FROM test_agg GROUP BY k;
148-

sql/core/src/test/resources/sql-tests/inputs/inline-table.sql

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,3 @@ select * from values ("one", count(1)), ("two", 2) as data(a, b);
4949

5050
-- string to timestamp
5151
select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-12-06 01:00:00.0'), timestamp('1991-12-06 12:00:00.0'))) as data(a, b);
52-
53-
-- cross-join inline tables
54-
EXPLAIN EXTENDED SELECT * FROM VALUES ('one', 1), ('three', null) CROSS JOIN VALUES ('one', 1), ('three', null);

sql/core/src/test/resources/sql-tests/inputs/operators.sql

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -29,27 +29,6 @@ select 2 * 5;
2929
select 5 % 3;
3030
select pmod(-7, 3);
3131

32-
-- check operator precedence.
33-
-- We follow Oracle operator precedence in the table below that lists the levels of precedence
34-
-- among SQL operators from high to low:
35-
------------------------------------------------------------------------------------------
36-
-- Operator Operation
37-
------------------------------------------------------------------------------------------
38-
-- +, - identity, negation
39-
-- *, / multiplication, division
40-
-- +, -, || addition, subtraction, concatenation
41-
-- =, !=, <, >, <=, >=, IS NULL, LIKE, BETWEEN, IN comparison
42-
-- NOT exponentiation, logical negation
43-
-- AND conjunction
44-
-- OR disjunction
45-
------------------------------------------------------------------------------------------
46-
explain select 'a' || 1 + 2;
47-
explain select 1 - 2 || 'b';
48-
explain select 2 * 4 + 3 || 'b';
49-
explain select 3 + 1 || 'a' || 4 / 2;
50-
explain select 1 == 1 OR 'a' || 'b' == 'ab';
51-
explain select 'a' || 'c' == 'ac' AND 2 == 3;
52-
5332
-- math functions
5433
select cot(1);
5534
select cot(null);

sql/core/src/test/resources/sql-tests/inputs/sql-compatibility-functions.sql

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,6 @@ SELECT nullif(1, 2.1d), nullif(1, 1.0d);
1212
SELECT nvl(1, 2.1d), nvl(null, 2.1d);
1313
SELECT nvl2(null, 1, 2.1d), nvl2('n', 1, 2.1d);
1414

15-
-- explain for these functions; use range to avoid constant folding
16-
explain extended
17-
select ifnull(id, 'x'), nullif(id, 'x'), nvl(id, 'x'), nvl2(id, 'x', 'y')
18-
from range(2);
19-
2015
-- SPARK-16730 cast alias functions for Hive compatibility
2116
SELECT boolean(1), tinyint(1), smallint(1), int(1), bigint(1);
2217
SELECT float(1), double(1), decimal(1);

sql/core/src/test/resources/sql-tests/inputs/string-functions.sql

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,6 @@ select format_string();
55
-- A pipe operator for string concatenation
66
select 'a' || 'b' || 'c';
77

8-
-- Check if catalyst combine nested `Concat`s
9-
EXPLAIN EXTENDED SELECT (col1 || col2 || col3 || col4) col
10-
FROM (SELECT id col1, id col2, id col3, id col4 FROM range(10));
11-
128
-- replace function
139
select replace('abc', 'b', '123');
1410
select replace('abc', 'b');
@@ -25,29 +21,6 @@ select left(null, -2), left("abcd", -2), left("abcd", 0), left("abcd", 'a');
2521
select right("abcd", 2), right("abcd", 5), right("abcd", '2'), right("abcd", null);
2622
select right(null, -2), right("abcd", -2), right("abcd", 0), right("abcd", 'a');
2723

28-
-- turn off concatBinaryAsString
29-
set spark.sql.function.concatBinaryAsString=false;
30-
31-
-- Check if catalyst combine nested `Concat`s if concatBinaryAsString=false
32-
EXPLAIN SELECT ((col1 || col2) || (col3 || col4)) col
33-
FROM (
34-
SELECT
35-
string(id) col1,
36-
string(id + 1) col2,
37-
encode(string(id + 2), 'utf-8') col3,
38-
encode(string(id + 3), 'utf-8') col4
39-
FROM range(10)
40-
);
41-
42-
EXPLAIN SELECT (col1 || (col3 || col4)) col
43-
FROM (
44-
SELECT
45-
string(id) col1,
46-
encode(string(id + 2), 'utf-8') col3,
47-
encode(string(id + 3), 'utf-8') col4
48-
FROM range(10)
49-
);
50-
5124
-- split function
5225
SELECT split('aa1cc2ee3', '[1-9]+');
5326
SELECT split('aa1cc2ee3', '[1-9]+', 2);

sql/core/src/test/resources/sql-tests/inputs/table-valued-functions.sql

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,3 @@ select * from range(1, null);
2121

2222
-- range call with a mixed-case function name
2323
select * from RaNgE(2);
24-
25-
-- Explain
26-
EXPLAIN select * from RaNgE(2);
27-
28-
-- cross-join table valued functions
29-
EXPLAIN EXTENDED SELECT * FROM range(3) CROSS JOIN range(3);

sql/core/src/test/resources/sql-tests/results/group-by.sql.out

Lines changed: 1 addition & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 47
2+
-- Number of queries: 46
33

44

55
-- !query 0
@@ -459,31 +459,3 @@ struct<k:int,v:boolean,any(v) OVER (PARTITION BY k ORDER BY v ASC NULLS FIRST RA
459459
5 NULL NULL
460460
5 false false
461461
5 true true
462-
463-
464-
-- !query 46
465-
EXPLAIN EXTENDED SELECT k, every(v), some(v), any(v) FROM test_agg GROUP BY k
466-
-- !query 46 schema
467-
struct<plan:string>
468-
-- !query 46 output
469-
== Parsed Logical Plan ==
470-
'Aggregate ['k], ['k, unresolvedalias('every('v), None), unresolvedalias('some('v), None), unresolvedalias('any('v), None)]
471-
+- 'UnresolvedRelation `test_agg`
472-
473-
== Analyzed Logical Plan ==
474-
k: int, every(v): boolean, some(v): boolean, any(v): boolean
475-
Aggregate [k#x], [k#x, every(v#x) AS every(v)#x, some(v#x) AS some(v)#x, any(v#x) AS any(v)#x]
476-
+- SubqueryAlias `test_agg`
477-
+- Project [k#x, v#x]
478-
+- SubqueryAlias `test_agg`
479-
+- LocalRelation [k#x, v#x]
480-
481-
== Optimized Logical Plan ==
482-
Aggregate [k#x], [k#x, min(v#x) AS every(v)#x, max(v#x) AS some(v)#x, max(v#x) AS any(v)#x]
483-
+- LocalRelation [k#x, v#x]
484-
485-
== Physical Plan ==
486-
*HashAggregate(keys=[k#x], functions=[min(v#x), max(v#x)], output=[k#x, every(v)#x, some(v)#x, any(v)#x])
487-
+- Exchange hashpartitioning(k#x, 200)
488-
+- *HashAggregate(keys=[k#x], functions=[partial_min(v#x), partial_max(v#x)], output=[k#x, min#x, max#x])
489-
+- LocalTableScan [k#x, v#x]

sql/core/src/test/resources/sql-tests/results/inline-table.sql.out

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 18
2+
-- Number of queries: 17
33

44

55
-- !query 0
@@ -151,33 +151,3 @@ select * from values (timestamp('1991-12-06 00:00:00.0'), array(timestamp('1991-
151151
struct<a:timestamp,b:array<timestamp>>
152152
-- !query 16 output
153153
1991-12-06 00:00:00 [1991-12-06 01:00:00.0,1991-12-06 12:00:00.0]
154-
155-
156-
-- !query 17
157-
EXPLAIN EXTENDED SELECT * FROM VALUES ('one', 1), ('three', null) CROSS JOIN VALUES ('one', 1), ('three', null)
158-
-- !query 17 schema
159-
struct<plan:string>
160-
-- !query 17 output
161-
== Parsed Logical Plan ==
162-
'Project [*]
163-
+- 'Join Cross
164-
:- 'UnresolvedInlineTable [col1, col2], [List(one, 1), List(three, null)]
165-
+- 'UnresolvedInlineTable [col1, col2], [List(one, 1), List(three, null)]
166-
167-
== Analyzed Logical Plan ==
168-
col1: string, col2: int, col1: string, col2: int
169-
Project [col1#x, col2#x, col1#x, col2#x]
170-
+- Join Cross
171-
:- LocalRelation [col1#x, col2#x]
172-
+- LocalRelation [col1#x, col2#x]
173-
174-
== Optimized Logical Plan ==
175-
Join Cross
176-
:- LocalRelation [col1#x, col2#x]
177-
+- LocalRelation [col1#x, col2#x]
178-
179-
== Physical Plan ==
180-
BroadcastNestedLoopJoin BuildRight, Cross
181-
:- LocalTableScan [col1#x, col2#x]
182-
+- BroadcastExchange IdentityBroadcastMode
183-
+- LocalTableScan [col1#x, col2#x]

0 commit comments

Comments
 (0)