From 385f6d49e0d472821b3ffa2f691bf2bbd2c068eb Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Sat, 5 Mar 2016 01:49:58 -0800 Subject: [PATCH 1/4] [SPARK-13698] Fix Analysis Exceptions when Using Backticks in Generate --- .../apache/spark/sql/catalyst/parser/CatalystQl.scala | 10 ++++++++-- .../scala/org/apache/spark/sql/hive/HiveQlSuite.scala | 9 +++++++++ .../spark/sql/hive/execution/SQLQuerySuite.scala | 8 ++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala index d2318417e3e6..e3f6ff94fab1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala @@ -999,10 +999,16 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C } val attributes = clauses.collect { - case Token(a, Nil) => UnresolvedAttribute(a.toLowerCase) + case Token(a, Nil) => UnresolvedAttribute(cleanIdentifier(a.toLowerCase)) } - Generate(generator, join = true, outer = outer, Some(alias.toLowerCase), attributes, child) + Generate( + generator, + join = true, + outer = outer, + Some(cleanIdentifier(alias.toLowerCase)), + attributes, + child) } protected def nodeToGenerator(node: ASTNode): Generator = noParseRule("Generator", node) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala index efd33f59416a..50d574bc3182 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala @@ -208,4 +208,13 @@ class HiveQlSuite extends SparkFunSuite with BeforeAndAfterAll { |USING 'cat' AS (`thing1` int, `thing2` string) FROM `default`.`parquet_t1`) AS t """.stripMargin) } + + test("use backticks in output of Generator") { + val plan = parser.parsePlan( + """SELECT `gentab2`.`gencol2` + |FROM `default`.`src` + |LATERAL VIEW explode(array(array(1, 2, 3))) `gentab1` AS `gencol1` + |LATERAL VIEW explode(`gentab1`.`gencol1`) `gentab2` AS `gencol2` + """.stripMargin) + } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index e478bcd0ed65..eb2d02d0624d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -254,6 +254,14 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { checkAnswer( sql("SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) a AS ints"), Row(1) :: Row(2) :: Row(3) :: Nil) + + checkAnswer( + sql("SELECT `ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"), + Row(1) :: Row(2) :: Row(3) :: Nil) + + checkAnswer( + sql("SELECT `a`.`ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"), + Row(1) :: Row(2) :: Row(3) :: Nil) } test("SPARK-4512 Fix attribute reference resolution error when using SORT BY") { From 07af901bccb4f989e7c734d874f1fe6e53a0a9c9 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Tue, 8 Mar 2016 01:22:56 -0800 Subject: [PATCH 2/4] Add escaped backtick test --- .../scala/org/apache/spark/sql/hive/HiveQlSuite.scala | 9 +++++++++ .../apache/spark/sql/hive/execution/SQLQuerySuite.scala | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala index 50d574bc3182..024fc6c2aeef 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala @@ -217,4 +217,13 @@ class HiveQlSuite extends SparkFunSuite with BeforeAndAfterAll { |LATERAL VIEW explode(`gentab1`.`gencol1`) `gentab2` AS `gencol2` """.stripMargin) } + + test("use escaped backticks in output of Generator") { + val plan = parser.parsePlan( + """SELECT `gen``tab2`.`gen``col2` + |FROM `default`.`src` + |LATERAL VIEW explode(array(array(1, 2, 3))) `gen``tab1` AS `gen``col1` + |LATERAL VIEW explode(`gen``tab1`.`gen``col1`) `gen``tab2` AS `gen``col2` + """.stripMargin) + } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index eb2d02d0624d..bf6ab43b0ecc 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -262,6 +262,10 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { checkAnswer( sql("SELECT `a`.`ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"), Row(1) :: Row(2) :: Row(3) :: Nil) + + checkAnswer( + sql("SELECT `weird``tab`.`weird``col` FROM nestedArray LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col`"), + Row(1) :: Row(2) :: Row(3) :: Nil) } test("SPARK-4512 Fix attribute reference resolution error when using SORT BY") { From ba6411afdc8c59dd0fe049b62df55eb137a2dedc Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Tue, 8 Mar 2016 03:01:02 -0800 Subject: [PATCH 3/4] style --- .../org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index bf6ab43b0ecc..b83fa19c8bca 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -264,7 +264,10 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { Row(1) :: Row(2) :: Row(3) :: Nil) checkAnswer( - sql("SELECT `weird``tab`.`weird``col` FROM nestedArray LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col`"), + sql( + """SELECT `weird``tab`.`weird``col` + |FROM nestedArray LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col` + """.stripMargin), Row(1) :: Row(2) :: Row(3) :: Nil) } From 7ed8ef772534254648816788e5e7ca98f7f23529 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Tue, 8 Mar 2016 17:25:19 -0800 Subject: [PATCH 4/4] Fix style --- .../scala/org/apache/spark/sql/hive/HiveQlSuite.scala | 8 +++++--- .../apache/spark/sql/hive/execution/SQLQuerySuite.scala | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala index 024fc6c2aeef..d78bcb2a83a4 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala @@ -211,7 +211,8 @@ class HiveQlSuite extends SparkFunSuite with BeforeAndAfterAll { test("use backticks in output of Generator") { val plan = parser.parsePlan( - """SELECT `gentab2`.`gencol2` + """ + |SELECT `gentab2`.`gencol2` |FROM `default`.`src` |LATERAL VIEW explode(array(array(1, 2, 3))) `gentab1` AS `gencol1` |LATERAL VIEW explode(`gentab1`.`gencol1`) `gentab2` AS `gencol2` @@ -220,9 +221,10 @@ class HiveQlSuite extends SparkFunSuite with BeforeAndAfterAll { test("use escaped backticks in output of Generator") { val plan = parser.parsePlan( - """SELECT `gen``tab2`.`gen``col2` + """ + |SELECT `gen``tab2`.`gen``col2` |FROM `default`.`src` - |LATERAL VIEW explode(array(array(1, 2, 3))) `gen``tab1` AS `gen``col1` + |LATERAL VIEW explode(array(array(1, 2, 3))) `gen``tab1` AS `gen``col1` |LATERAL VIEW explode(`gen``tab1`.`gen``col1`) `gen``tab2` AS `gen``col2` """.stripMargin) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index b83fa19c8bca..4fbd64201779 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -265,8 +265,10 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { checkAnswer( sql( - """SELECT `weird``tab`.`weird``col` - |FROM nestedArray LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col` + """ + |SELECT `weird``tab`.`weird``col` + |FROM nestedArray + |LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col` """.stripMargin), Row(1) :: Row(2) :: Row(3) :: Nil) }