From a8543160e946e1f80b91be8d9d24cb63359541ed Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 16 Nov 2019 21:07:43 +0800 Subject: [PATCH 1/5] [SPARK-29873][SQL] set operations should not escape when regen golden file with --SET --import both specified --- .../sql-tests/inputs/ansi/literals.sql | 3 +++ .../resources/sql-tests/inputs/set-cmd.sql | 3 +++ .../sql-tests/inputs/set-cmd_import.sql | 2 ++ .../sql-tests/results/ansi/literals.sql.out | 10 ++++++- .../sql-tests/results/set-cmd.sql.out | 26 +++++++++++++++++++ .../sql-tests/results/set-cmd_import.sql.out | 26 +++++++++++++++++++ .../apache/spark/sql/SQLQueryTestSuite.scala | 2 +- 7 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql create mode 100644 sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/set-cmd.sql.out create mode 100644 sql/core/src/test/resources/sql-tests/results/set-cmd_import.sql.out diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql index 170690ea699c..7acb627e1856 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql @@ -1,2 +1,5 @@ --- malformed interval literal with ansi mode +-- SET spark.abc=xyz --import literals.sql + +SET spark.abc; diff --git a/sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql b/sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql new file mode 100644 index 000000000000..78900566bd86 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql @@ -0,0 +1,3 @@ +set spark.abc; +set spark.abc=efg; +set spark.abc; \ No newline at end of file diff --git a/sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql b/sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql new file mode 100644 index 000000000000..4c6c5899f06a --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql @@ -0,0 +1,2 @@ +--SET spark.abc=xyz; +--import set-cmd.sql \ No newline at end of file diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out index e43e88c8c72a..93f7d116c60e 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 48 +-- Number of queries: 49 -- !query 0 @@ -472,3 +472,11 @@ select +integer '7' struct<7:int> -- !query 47 output 7 + + +-- !query 48 +SET spark.abc +-- !query 48 schema +struct +-- !query 48 output +spark.abc diff --git a/sql/core/src/test/resources/sql-tests/results/set-cmd.sql.out b/sql/core/src/test/resources/sql-tests/results/set-cmd.sql.out new file mode 100644 index 000000000000..297326c37982 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/set-cmd.sql.out @@ -0,0 +1,26 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 3 + + +-- !query 0 +set spark.abc +-- !query 0 schema +struct +-- !query 0 output +spark.abc + + +-- !query 1 +set spark.abc=efg +-- !query 1 schema +struct +-- !query 1 output +spark.abc efg + + +-- !query 2 +set spark.abc +-- !query 2 schema +struct +-- !query 2 output +spark.abc efg diff --git a/sql/core/src/test/resources/sql-tests/results/set-cmd_import.sql.out b/sql/core/src/test/resources/sql-tests/results/set-cmd_import.sql.out new file mode 100644 index 000000000000..2df3f2eb9935 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/set-cmd_import.sql.out @@ -0,0 +1,26 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 3 + + +-- !query 0 +set spark.abc +-- !query 0 schema +struct +-- !query 0 output +spark.abc xyz; + + +-- !query 1 +set spark.abc=efg +-- !query 1 schema +struct +-- !query 1 output +spark.abc efg + + +-- !query 2 +set spark.abc +-- !query 2 schema +struct +-- !query 2 output +spark.abc efg diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index ffea03474ac4..d98e804a8551 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -285,7 +285,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { // When we are regenerating the golden files, we don't need to set any config as they // all need to return the same result - if (regenerateGoldenFiles || !isTestWithConfigSets) { + if ((regenerateGoldenFiles && importedTestCaseName.isEmpty) || !isTestWithConfigSets) { runQueries(queries, testCase, None) } else { val configSets = { From c791b36fe8dfb8dc5cfbb393aaf44b34c5b408fa Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 16 Nov 2019 21:20:50 +0800 Subject: [PATCH 2/5] rm changes in literals.sql --- .../test/resources/sql-tests/inputs/ansi/literals.sql | 3 --- .../src/test/resources/sql-tests/inputs/set-cmd.sql | 2 +- .../test/resources/sql-tests/inputs/set-cmd_import.sql | 2 +- .../resources/sql-tests/results/ansi/literals.sql.out | 10 +--------- 4 files changed, 3 insertions(+), 14 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql index 7acb627e1856..170690ea699c 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql @@ -1,5 +1,2 @@ --- malformed interval literal with ansi mode --- SET spark.abc=xyz --import literals.sql - -SET spark.abc; diff --git a/sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql b/sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql index 78900566bd86..500f0108cc0a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql @@ -1,3 +1,3 @@ set spark.abc; set spark.abc=efg; -set spark.abc; \ No newline at end of file +set spark.abc; diff --git a/sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql b/sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql index 4c6c5899f06a..c764e162f467 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql @@ -1,2 +1,2 @@ --SET spark.abc=xyz; ---import set-cmd.sql \ No newline at end of file +--import set-cmd.sql diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out index 93f7d116c60e..e43e88c8c72a 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 49 +-- Number of queries: 48 -- !query 0 @@ -472,11 +472,3 @@ select +integer '7' struct<7:int> -- !query 47 output 7 - - --- !query 48 -SET spark.abc --- !query 48 schema -struct --- !query 48 output -spark.abc From 8fb683edf65fdb0c5a452c257b71335724428e02 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Sat, 16 Nov 2019 23:49:02 +0800 Subject: [PATCH 3/5] rm tests --- .../resources/sql-tests/inputs/set-cmd.sql | 3 --- .../sql-tests/inputs/set-cmd_import.sql | 2 -- .../sql-tests/results/set-cmd.sql.out | 26 ------------------- .../sql-tests/results/set-cmd_import.sql.out | 26 ------------------- 4 files changed, 57 deletions(-) delete mode 100644 sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql delete mode 100644 sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql delete mode 100644 sql/core/src/test/resources/sql-tests/results/set-cmd.sql.out delete mode 100644 sql/core/src/test/resources/sql-tests/results/set-cmd_import.sql.out diff --git a/sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql b/sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql deleted file mode 100644 index 500f0108cc0a..000000000000 --- a/sql/core/src/test/resources/sql-tests/inputs/set-cmd.sql +++ /dev/null @@ -1,3 +0,0 @@ -set spark.abc; -set spark.abc=efg; -set spark.abc; diff --git a/sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql b/sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql deleted file mode 100644 index c764e162f467..000000000000 --- a/sql/core/src/test/resources/sql-tests/inputs/set-cmd_import.sql +++ /dev/null @@ -1,2 +0,0 @@ ---SET spark.abc=xyz; ---import set-cmd.sql diff --git a/sql/core/src/test/resources/sql-tests/results/set-cmd.sql.out b/sql/core/src/test/resources/sql-tests/results/set-cmd.sql.out deleted file mode 100644 index 297326c37982..000000000000 --- a/sql/core/src/test/resources/sql-tests/results/set-cmd.sql.out +++ /dev/null @@ -1,26 +0,0 @@ --- Automatically generated by SQLQueryTestSuite --- Number of queries: 3 - - --- !query 0 -set spark.abc --- !query 0 schema -struct --- !query 0 output -spark.abc - - --- !query 1 -set spark.abc=efg --- !query 1 schema -struct --- !query 1 output -spark.abc efg - - --- !query 2 -set spark.abc --- !query 2 schema -struct --- !query 2 output -spark.abc efg diff --git a/sql/core/src/test/resources/sql-tests/results/set-cmd_import.sql.out b/sql/core/src/test/resources/sql-tests/results/set-cmd_import.sql.out deleted file mode 100644 index 2df3f2eb9935..000000000000 --- a/sql/core/src/test/resources/sql-tests/results/set-cmd_import.sql.out +++ /dev/null @@ -1,26 +0,0 @@ --- Automatically generated by SQLQueryTestSuite --- Number of queries: 3 - - --- !query 0 -set spark.abc --- !query 0 schema -struct --- !query 0 output -spark.abc xyz; - - --- !query 1 -set spark.abc=efg --- !query 1 schema -struct --- !query 1 output -spark.abc efg - - --- !query 2 -set spark.abc --- !query 2 schema -struct --- !query 2 output -spark.abc efg From 538a44fbb8bcda1a462c18e82ca033ba76a5c2ce Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 18 Nov 2019 16:34:19 +0800 Subject: [PATCH 4/5] comments, to upper case --- .../inputs/ansi/higher-order-functions.sql | 2 +- .../resources/sql-tests/inputs/ansi/interval.sql | 2 +- .../resources/sql-tests/inputs/ansi/literals.sql | 2 +- .../sql-tests/inputs/interval-display-iso_8601.sql | 2 +- .../inputs/interval-display-sql_standard.sql | 2 +- .../org/apache/spark/sql/SQLQueryTestSuite.scala | 13 +++++++------ 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/higher-order-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/higher-order-functions.sql index 4068a27fcb2a..1e2424fe47ca 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/ansi/higher-order-functions.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/higher-order-functions.sql @@ -1 +1 @@ ---import higher-order-functions.sql +--IMPORT higher-order-functions.sql diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/interval.sql index 215ee7c074fa..087914eebb07 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/ansi/interval.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/interval.sql @@ -1,4 +1,4 @@ ---import interval.sql +--IMPORT interval.sql -- the `interval` keyword can be omitted with ansi mode select 1 year 2 days; diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql index 170690ea699c..698e8fa88630 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/literals.sql @@ -1,2 +1,2 @@ --- malformed interval literal with ansi mode ---import literals.sql +--IMPORT literals.sql diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval-display-iso_8601.sql b/sql/core/src/test/resources/sql-tests/inputs/interval-display-iso_8601.sql index 62f3f43bd2c4..3b63c715a6aa 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/interval-display-iso_8601.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/interval-display-iso_8601.sql @@ -1,3 +1,3 @@ -- tests for interval output style with iso_8601 format --SET spark.sql.intervalOutputStyle = ISO_8601 ---import interval-display.sql +--IMPORT interval-display.sql diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval-display-sql_standard.sql b/sql/core/src/test/resources/sql-tests/inputs/interval-display-sql_standard.sql index 375b4899e760..d96865b160bb 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/interval-display-sql_standard.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/interval-display-sql_standard.sql @@ -1,3 +1,3 @@ -- tests for interval output style with sql standard format --SET spark.sql.intervalOutputStyle = SQL_STANDARD ---import interval-display.sql +--IMPORT interval-display.sql diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 8990ae68b52b..0f0ed809794e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -65,7 +65,7 @@ import org.apache.spark.tags.ExtendedSQLTest * 1. A list of SQL queries separated by semicolon. * 2. Lines starting with -- are treated as comments and ignored. * 3. Lines starting with --SET are used to run the file with the following set of configs. - * 4. Lines starting with --import are used to load queries from another test file. + * 4. Lines starting with --IMPORT are used to load queries from another test file. * * For example: * {{{ @@ -265,9 +265,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { val (comments, code) = input.split("\n").partition(_.trim.startsWith("--")) - // If `--import` found, load code from another test case file, then insert them + // If `--IMPORT` found, load code from another test case file, then insert them // into the head in this test. - val importedTestCaseName = comments.filter(_.startsWith("--import ")).map(_.substring(9)) + val importedTestCaseName = comments.filter(_.startsWith("--IMPORT ")).map(_.substring(9)) val importedCode = importedTestCaseName.flatMap { testCaseName => listTestCases.find(_.name == testCaseName).map { testCase => val input = fileToString(new File(testCase.inputFile)) @@ -283,13 +283,14 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { // Fix misplacement when comment is at the end of the query. .map(_.split("\n").filterNot(_.startsWith("--")).mkString("\n")).map(_.trim).filter(_ != "") - // When we are regenerating the golden files, we don't need to set any config as they - // all need to return the same result + // When we are regenerating the golden files for test cases without '--IMPORT' specified, or + // running test cases against [[ThriftServerQueryTestSuite], we don't need to set any config as + // they all need to return the same result. if ((regenerateGoldenFiles && importedTestCaseName.isEmpty) || !isTestWithConfigSets) { runQueries(queries, testCase, None) } else { val configSets = { - val configLines = comments.filter(_.startsWith("--SET")).map(_.substring(5)) + val configLines = comments.filter(_.startsWith("--SET ")).map(_.substring(6)) val configs = configLines.map(_.split(",").map { confAndValue => val (conf, value) = confAndValue.span(_ != '=') conf.trim -> value.substring(1).trim From 3192e152b34c6a4c9bfe4e7c6f8ffc2f6ca6b153 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 18 Nov 2019 16:51:10 +0800 Subject: [PATCH 5/5] comment --- .../test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 0f0ed809794e..ae0ccf556fbc 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -286,6 +286,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { // When we are regenerating the golden files for test cases without '--IMPORT' specified, or // running test cases against [[ThriftServerQueryTestSuite], we don't need to set any config as // they all need to return the same result. + // When we use '--SET' and '--IMPORT' together for those import queries, we want to run the + // same queries from the original file but with different settings and save the answers. So the + // `--SET` will be respected in this case. if ((regenerateGoldenFiles && importedTestCaseName.isEmpty) || !isTestWithConfigSets) { runQueries(queries, testCase, None) } else {