Skip to content

[SPARK-26493][SQL] Allow multiple spark.sql.extensions #23398

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jamisonbennett wants to merge 7 commits into from
Closed

[SPARK-26493][SQL] Allow multiple spark.sql.extensions #23398

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
cef8eb0
[SPARK-26493][SQL] Allow multiple spark.sql.extensions
jamisonbennett Dec 28, 2018
689a4d2
Address comments from dongjoon-hyun, hvanhovell, beliefer, gatorsmile
jamisonbennett Dec 29, 2018
d89cfd9
Address comment from hvanhovell
jamisonbennett Dec 31, 2018
fb4ad34
Address comments from HyukjinKwon
jamisonbennett Jan 5, 2019
65a5f3f
Address comments from HyukjinKwon, vanzin
jamisonbennett Jan 9, 2019
9c0181d
Fix a typo
jamisonbennett Jan 9, 2019
deaf73e
Address comments from vanzin
jamisonbennett Jan 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Address comments from HyukjinKwon, vanzin 
This addresses the comments for #23398
  • Loading branch information
@jamisonbennett
jamisonbennett committed Jan 9, 2019
commit 65a5f3f6c05c4e07da494be2f7ad5eaddd33dc5c
2 changes: 1 addition & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ class SimpleFunctionRegistry extends FunctionRegistry with Logging {
functionBuilders.put(normalizedName, newFunction) match {
case Some(previousFunction) if previousFunction != newFunction =>
logWarning(s"The function $normalizedName replaced a previously registered function.")
case _ => Unit
case _ =>
}
}

Expand Down
4 changes: 2 additions & 2 deletions sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
* <li>(External) Catalog listeners.</li>
* </ul>
*
* The extensions can be used by calling withExtension on the [[SparkSession.Builder]], for
* The extensions can be used by calling `withExtension` on the [[SparkSession.Builder]], for
* example:
* {{{
* SparkSession.builder()
Expand All @@ -62,7 +62,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
* }}}
*
* The extensions can also be used by setting the Spark SQL configuration property
* spark.sql.extensions, for example:
* `spark.sql.extensions`. Multiple extensions can be set using a comma-separated list. For example:
* {{{
* SparkSession.builder()
* .master("...")
Expand Down
19 changes: 9 additions & 10 deletions sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class SparkSessionExtensionSuite extends SparkFunSuite {

test("inject parser") {
val extension = create { extensions =>
extensions.injectParser((_, _) => CatalystSqlParser)
extensions.injectParser((_: SparkSession, _: ParserInterface) => CatalystSqlParser)
}
withSession(extension) { session =>
assert(session.sessionState.sqlParser == CatalystSqlParser)
Expand All @@ -88,15 +88,15 @@ class SparkSessionExtensionSuite extends SparkFunSuite {

test("inject multiple rules") {
withSession(Seq(_.injectOptimizerRule(MyRule),
_.injectPlannerStrategy(MySparkStrategy))) { session =>
_.injectPlannerStrategy(MySparkStrategy))) { session =>
assert(session.sessionState.optimizer.batches.flatMap(_.rules).contains(MyRule(session)))
assert(session.sessionState.planner.strategies.contains(MySparkStrategy(session)))
}
}

test("inject stacked parsers") {
val extension = create { extensions =>
extensions.injectParser((_, _) => CatalystSqlParser)
extensions.injectParser((_: SparkSession, _: ParserInterface) => CatalystSqlParser)
extensions.injectParser(MyParser)
extensions.injectParser(MyParser)
}
Expand Down Expand Up @@ -198,8 +198,8 @@ class SparkSessionExtensionSuite extends SparkFunSuite {
val lastRegistered = session.sessionState.functionRegistry
.lookupFunction(FunctionIdentifier("myFunction2"))
assert(lastRegistered.isDefined)
assert(lastRegistered.get.getExtended != MyExtensions2.myFunction._2.getExtended)
assert(lastRegistered.get.getExtended == MyExtensions2Duplicate.myFunction._2.getExtended)
assert(lastRegistered.get != MyExtensions2.myFunction._2)
assert(lastRegistered.get == MyExtensions2Duplicate.myFunction._2)
} finally {
stop(session)
}
Expand Down Expand Up @@ -242,8 +242,7 @@ object MyExtensions {

val myFunction = (FunctionIdentifier("myFunction"),
new ExpressionInfo("noClass", "myDb", "myFunction", "usage", "extended usage"),
(myArgs: Seq[Expression]) => Literal(5, IntegerType))

(_: Seq[Expression]) => Literal(5, IntegerType))
}

class MyExtensions extends (SparkSessionExtensions => Unit) {
Expand Down Expand Up @@ -273,7 +272,7 @@ case class MySparkStrategy2(spark: SparkSession) extends SparkStrategy {
object MyExtensions2 {

val myFunction = (FunctionIdentifier("myFunction2"),
new ExpressionInfo("noClass", "myDb", "myFunction2", "usage", "extended usage" ),
new ExpressionInfo("noClass", "myDb", "myFunction2", "usage", "extended usage"),
(_: Seq[Expression]) => Literal(5, IntegerType))
}

Expand All @@ -284,15 +283,15 @@ class MyExtensions2 extends (SparkSessionExtensions => Unit) {
e.injectPostHocResolutionRule(MyRule2)
e.injectCheckRule(MyCheckRule2)
e.injectOptimizerRule(MyRule2)
e.injectParser((_, _) => CatalystSqlParser)
e.injectParser((_: SparkSession, _: ParserInterface) => CatalystSqlParser)
e.injectFunction(MyExtensions2.myFunction)
}
}

object MyExtensions2Duplicate {

val myFunction = (FunctionIdentifier("myFunction2"),
new ExpressionInfo("noClass", "myDb", "myFunction2", "usage", "last wins" ),
new ExpressionInfo("noClass", "myDb", "myFunction2", "usage", "extended usage"),
(_: Seq[Expression]) => Literal(5, IntegerType))
}

Expand Down