Skip to content

[SPARK-12719][SQL] [WIP] SQL generation support for generators, including UDTF #11596

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
dilipbiswal wants to merge 6 commits into from
Closed

[SPARK-12719][SQL] [WIP] SQL generation support for generators, including UDTF #11596

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1088c30
[SPARK-12719] SQL generation support for generators (including UDTF)
dilipbiswal Mar 7, 2016
153b876
Code review comments
dilipbiswal Mar 7, 2016
37d17cf
Review comments from Reynold
dilipbiswal Mar 8, 2016
cd0c001
Alternative approach.
dilipbiswal Mar 8, 2016
53e9934
scala style fix
dilipbiswal Mar 9, 2016
bea871f
Address review comments
dilipbiswal Mar 11, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Code review comments
  • Loading branch information
@dilipbiswal
dilipbiswal committed Mar 11, 2016
commit 153b876767b2001e5847ee306d686db5ebfa48e6
4 changes: 2 additions & 2 deletions sql/hive/src/main/scala/org/apache/spark/sql/hive/SQLBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi

/* This function handles the SQL generation when generators are specified in the
* LATERAL VIEW clause. SQL generation of generators specified in projection lists
* are handled in projectToSQL.
* is handled in projectToSQL.
* sample plan :
* +- Project [mycol2#192]
* +- Generate explode(myCol#191), true, false, Some(mytable2), [mycol2#192]
Expand All @@ -348,7 +348,7 @@ class SQLBuilder(logicalPlan: LogicalPlan, sqlContext: SQLContext) extends Loggi
val outerClause = if (plan.outer) "OUTER" else ""
build(
toSQL(plan.child),
"LATERAL VIEW ",
"LATERAL VIEW",
outerClause,
plan.generator.sql,
quoteIdentifier(generatorAlias),
Expand Down
36 changes: 17 additions & 19 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/LogicalPlanToSQLSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ class LogicalPlanToSQLSuite extends SQLBuilderTest with SQLTestUtils {
sql("DROP TABLE IF EXISTS parquet_t0")
sql("DROP TABLE IF EXISTS parquet_t1")
sql("DROP TABLE IF EXISTS parquet_t2")
sql("DROP TABLE IF EXISTS parquet_t3")
sql("DROP TABLE IF EXISTS t0")
sql("DROP TABLE IF EXISTS t3")
sql("DROP TABLE IF EXISTS t4")
sql("DROP TABLE IF EXISTS t1")

val tuples: Seq[(String, String)] =
("1", """{"f1": "value1", "f2": "value2", "f3": 3, "f5": 5.23}""") ::
Expand All @@ -57,19 +57,17 @@ class LogicalPlanToSQLSuite extends SQLBuilderTest with SQLTestUtils {
.write
.saveAsTable("parquet_t2")

sqlContext.range(10).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd).write.saveAsTable("t2")

tuples.toDF("key", "jstring").write.saveAsTable("t3")
sql("CREATE TABLE t4 as select key, array(value) as value from parquet_t1 limit 20")
tuples.toDF("key", "jstring").write.saveAsTable("parquet_t3")
sql("CREATE TABLE t1 as select key, array(value) as value from parquet_t1 limit 20")
}

override protected def afterAll(): Unit = {
sql("DROP TABLE IF EXISTS parquet_t0")
sql("DROP TABLE IF EXISTS parquet_t1")
sql("DROP TABLE IF EXISTS parquet_t2")
sql("DROP TABLE IF EXISTS parquet_t3")
sql("DROP TABLE IF EXISTS t0")
sql("DROP TABLE IF EXISTS t3")
sql("DROP TABLE IF EXISTS t4")
sql("DROP TABLE IF EXISTS t1")
}

private def checkHiveQl(hiveQl: String): Unit = {
Expand Down Expand Up @@ -589,40 +587,40 @@ class LogicalPlanToSQLSuite extends SQLBuilderTest with SQLTestUtils {
checkHiveQl("select explode(array(1,2,3)) AS gencol")

// Explode with columns other than generated columns in projection list
checkHiveQl("SELECT key as c1, explode(array(1,2,3)) as c2, value as c3 FROM t4")
checkHiveQl("SELECT key as c1, explode(array(1,2,3)) as c2, value as c3 FROM t1")

// Explode with a column reference as input to generator
checkHiveQl("SELECT key, value from t4 LATERAL VIEW explode(value) gentab")
checkHiveQl("SELECT key, value from t1 LATERAL VIEW explode(value) gentab")

// json_tuple
checkHiveQl("SELECT key, json_tuple(jstring, 'f1', 'f2', 'f3', 'f4', 'f5') FROM t3")
checkHiveQl("SELECT key, json_tuple(jstring, 'f1', 'f2', 'f3', 'f4', 'f5') FROM parquet_t3")

// udtf
checkHiveQl("SELECT key, gencol FROM t4 LATERAL VIEW udtf_count2(value) gentab AS gencol")
checkHiveQl("SELECT key, gencol FROM t1 LATERAL VIEW udtf_count2(value) gentab AS gencol")

// udtf
checkHiveQl("SELECT udtf_count2(c1) FROM (SELECT 1 AS c1 FROM t4 LIMIT 3) g1")
checkHiveQl("SELECT udtf_count2(c1) FROM (SELECT 1 AS c1 FROM t1 LIMIT 3) g1")

// Filter and OUTER clause
checkHiveQl(
"""SELECT key, value
|FROM t4 LATERAL VIEW OUTER explode(value) gentab as gencol
|FROM t1 LATERAL VIEW OUTER explode(value) gentab as gencol
|WHERE key = 1
""".stripMargin
)

// single lateral view
checkHiveQl(
"""SELECT *
|FROM t4 LATERAL VIEW explode(array(1,2,3)) gentab AS gencol
|FROM t1 LATERAL VIEW explode(array(1,2,3)) gentab AS gencol
|SORT BY key ASC, gencol ASC LIMIT 1
""".stripMargin
)

// multiple lateral views
checkHiveQl(
"""SELECT gentab2.*
|FROM t4
|FROM t1
|LATERAL VIEW explode(array(array(1,2,3))) gentab1 AS gencol1
|LATERAL VIEW explode(gentab1.gencol1) gentab2 AS gencol2 LIMIT 3
""".stripMargin
Expand All @@ -632,20 +630,20 @@ class LogicalPlanToSQLSuite extends SQLBuilderTest with SQLTestUtils {
checkHiveQl(
"""SELECT subq.gencol
|FROM
|(SELECT * from t4 LATERAL VIEW explode(value) gentab AS gencol) subq
|(SELECT * from t1 LATERAL VIEW explode(value) gentab AS gencol) subq
""".stripMargin)

checkHiveQl(
"""SELECT subq.key
|FROM
|(SELECT key, value from t4 LATERAL VIEW explode(value) gentab AS gencol) subq
|(SELECT key, value from t1 LATERAL VIEW explode(value) gentab AS gencol) subq
""".stripMargin
)

checkHiveQl(
"""SELECT gentab.*
|FROM
|t4 LATERAL VIEW explode(map('key1', 100, 'key2', 200)) gentab limit 2
|t1 LATERAL VIEW explode(map('key1', 100, 'key2', 200)) gentab limit 2
""".stripMargin
)
sql("DROP TEMPORARY FUNCTION udtf_count2")
Expand Down