-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-21499] [SQL] Support creating persistent function for Spark UDAF(UserDefinedAggregateFunction) #18700
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
4028155
f634043
a65607c
12cefc2
bd5ae26
7251be9
d3fbdc5
57607b5
05e8168
aff8f9e
7d9aabd
8ea4ad1
50224a7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -41,6 +41,7 @@ import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParserInterface} | |
| import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias, View} | ||
| import org.apache.spark.sql.catalyst.util.StringUtils | ||
| import org.apache.spark.sql.internal.SQLConf | ||
| import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION | ||
| import org.apache.spark.sql.types.StructType | ||
| import org.apache.spark.util.Utils | ||
|
|
||
|
|
@@ -129,6 +130,13 @@ class SessionCatalog( | |
| if (conf.caseSensitiveAnalysis) name else name.toLowerCase(Locale.ROOT) | ||
| } | ||
|
|
||
| /** | ||
| * Checks whether the Hive metastore is being used | ||
| */ | ||
| private def isUsingHiveMetastore: Boolean = { | ||
| conf.getConf(CATALOG_IMPLEMENTATION).toLowerCase(Locale.ROOT) == "hive" | ||
| } | ||
|
|
||
| private val tableRelationCache: Cache[QualifiedTableName, LogicalPlan] = { | ||
| val cacheSize = conf.tableRelationCacheSize | ||
| CacheBuilder.newBuilder().maximumSize(cacheSize).build[QualifiedTableName, LogicalPlan]() | ||
|
|
@@ -1094,27 +1102,24 @@ class SessionCatalog( | |
| // ---------------------------------------------------------------- | ||
|
|
||
| /** | ||
| * Construct a [[FunctionBuilder]] based on the provided class that represents a function. | ||
| * | ||
| * This performs reflection to decide what type of [[Expression]] to return in the builder. | ||
| * Constructs a [[FunctionBuilder]] based on the provided class that represents a function. | ||
| */ | ||
| protected def makeFunctionBuilder(name: String, functionClassName: String): FunctionBuilder = { | ||
|
||
| val clazz = Utils.classForName(functionClassName) | ||
| (children: Seq[Expression]) => { | ||
| try { | ||
| makeFunctionExpression(name, Utils.classForName(functionClassName), children).getOrElse { | ||
|
||
| throw new UnsupportedOperationException("Use sqlContext.udf.register(...) instead.") | ||
| val extraMsg = | ||
| if (!isUsingHiveMetastore) "Use sparkSession.udf.register(...) instead." else "" | ||
| throw new AnalysisException( | ||
| s"No handler for UDF/UDAF/UDTF '${clazz.getCanonicalName}'. $extraMsg") | ||
| } | ||
| } catch { | ||
| case NonFatal(exception) => | ||
| val e = exception match { | ||
| // Since we are using shim, the exceptions thrown by the underlying method of | ||
| // Method.invoke() are wrapped by InvocationTargetException | ||
| case i: InvocationTargetException => i.getCause | ||
| case o => o | ||
| } | ||
| case ae: AnalysisException => | ||
| throw ae | ||
| case NonFatal(e) => | ||
| val analysisException = | ||
| new AnalysisException(s"No handler for UDAF '${clazz.getCanonicalName}': $e") | ||
| new AnalysisException(s"No handler for UDF/UDAF/UDTF '${clazz.getCanonicalName}': $e") | ||
| analysisException.setStackTrace(e.getStackTrace) | ||
| throw analysisException | ||
| } | ||
|
|
@@ -1123,6 +1128,8 @@ class SessionCatalog( | |
|
|
||
| /** | ||
| * Construct a [[FunctionBuilder]] based on the provided class that represents a function. | ||
| * | ||
| * This performs reflection to decide what type of [[Expression]] to return in the builder. | ||
| */ | ||
| protected def makeFunctionExpression( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. seems we need to catch exception for this method anyway, how about we just make this method return |
||
| name: String, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
do we still need this?