-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-18016][SQL] Code Generation: Constant Pool Limit - reduce entries for mutable state #19811
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
d441794
870d106
24d7087
3eb5842
074d711
eafa3f8
90d15f3
c456c07
9ca5ab3
5b36c61
fd51d75
effe918
9df109c
634d494
d3438fd
f4f3754
f1e1fca
0937ef2
4bfcc1a
49119a9
24f49c5
15e967e
d6c1a97
a9d40e9
31914c0
0e45c19
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -177,9 +177,9 @@ class CodegenContext { | |
| * the list of default imports available. | ||
| * Also, generic type arguments are accepted but ignored. | ||
| * @param variableName Name of the field. | ||
| * @param codeFunctions Function includes statement(s) to put into the init() method to | ||
| * initialize this field. An argument is the name of the mutable state variable | ||
| * If left blank, the field will be default-initialized. | ||
| * @param initFunc Function includes statement(s) to put into the init() method to initialize | ||
| * this field. An argument is the name of the mutable state variable. | ||
| * If left blank, the field will be default-initialized. | ||
| * @param inline whether the declaration and initialization code may be inlined rather than | ||
|
||
| * compacted. | ||
| * @param useFreshName If false and inline is true, the name is not changed | ||
|
|
@@ -197,7 +197,7 @@ class CodegenContext { | |
| def addMutableState( | ||
| javaType: String, | ||
| variableName: String, | ||
| codeFunctions: String => String = _ => "", | ||
| initFunc: String => String = _ => "", | ||
| inline: Boolean = false, | ||
| useFreshName: Boolean = true): String = { | ||
| val varName = if (useFreshName) freshName(variableName) else variableName | ||
|
||
|
|
@@ -208,7 +208,7 @@ class CodegenContext { | |
| (mutableStates.length < CodeGenerator.OUTER_CLASS_VARIABLES_THRESHOLD) || | ||
| // type is multi-dimensional array | ||
|
||
| javaType.contains("[][]")) { | ||
| val initCode = codeFunctions(varName) | ||
| val initCode = initFunc(varName) | ||
| mutableStates += ((javaType, varName, initCode)) | ||
| varName | ||
| } else { | ||
|
|
@@ -231,7 +231,7 @@ class CodegenContext { | |
| mutableStateArrayCurrentNames(javaType) = arrayName | ||
| mutableStateArrayIdx((javaType, arrayName)) = newIdx | ||
|
|
||
| val initCode = codeFunctions(s"$arrayName[$newIdx]") | ||
| val initCode = initFunc(s"$arrayName[$newIdx]") | ||
| mutableStateArrayInitCodes += initCode | ||
| s"$arrayName[$newIdx]" | ||
| } | ||
|
|
@@ -279,7 +279,7 @@ class CodegenContext { | |
| // `TypedAggregateExpression`, we should call `distinct` here to remove the duplicated ones. | ||
| val initCodes = mutableStates.map(_._3).distinct.map(_ + "\n") | ||
| // statements for array element initialization | ||
| val arrayInitCodes = mutableStateArrayInitCodes.distinct.map(_ + "\n") | ||
| val arrayInitCodes = mutableStateArrayInitCodes.distinct | ||
|
|
||
| // The generated initialization code may exceed 64kb function size limit in JVM if there are too | ||
| // many mutable states, so split it into multiple functions. | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: indentation
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit:
The Argument...