-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-18016][SQL] Code Generation: Constant Pool Limit - reduce entries for mutable state #19811
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
d441794
870d106
24d7087
3eb5842
074d711
eafa3f8
90d15f3
c456c07
9ca5ab3
5b36c61
fd51d75
effe918
9df109c
634d494
d3438fd
f4f3754
f1e1fca
0937ef2
4bfcc1a
49119a9
24f49c5
15e967e
d6c1a97
a9d40e9
31914c0
0e45c19
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
code cleanup
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -154,19 +154,17 @@ class CodegenContext { | |
| val mutableStates: mutable.ArrayBuffer[(String, String, String)] = | ||
| mutable.ArrayBuffer.empty[(String, String, String)] | ||
|
|
||
| // An array keyed by the tuple of mutable states' types and array name, holds the | ||
| // current max index of the array | ||
| // An map keyed by the tuple of mutable states' types and array name, holds the current max | ||
| // index of the array | ||
| var mutableStateArrayIdx: mutable.Map[(String, String), Int] = | ||
| mutable.Map.empty[(String, String), Int] | ||
|
|
||
| // An array keyed by the tuple of mutable states' types, holds the | ||
| // current name of the mutableStateArray into which state of the given key will be compacted | ||
| // An map keyed by mutable states' types holds the current name of the mutableStateArray | ||
| // into which state of the given key will be compacted | ||
| var mutableStateArrayCurrentNames: mutable.Map[String, String] = | ||
|
||
| mutable.Map.empty[String, String] | ||
|
|
||
| // An array keyed by the tuple of mutable states' types, array names, array index, and | ||
| // initialization code, holds the code that will initialize the mutableStateArray when | ||
| // initialized in loops | ||
| // An array holds the code that will initialize each element of the mutableStateArray | ||
| var mutableStateArrayInitCodes: mutable.ArrayBuffer[String] = | ||
|
||
| mutable.ArrayBuffer.empty[String] | ||
|
|
||
|
|
@@ -220,21 +218,21 @@ class CodegenContext { | |
| // a mutableStateArray for the given type and name has already been declared, | ||
| // update the max index of the array and return an array element | ||
| val idx = prevIdx + 1 | ||
| mutableStateArrayIdx.update((javaType, arrayName), idx) | ||
| val initCode = codeFunctions(s"$arrayName[$idx]") | ||
| mutableStateArrayInitCodes += initCode | ||
| mutableStateArrayIdx.update((javaType, arrayName), idx) | ||
| s"$arrayName[$idx]" | ||
| } else { | ||
| // mutableStateArray has not been declared yet for the given type and name. | ||
| // Create a new name for the array, and add an entry to keep track of current array name | ||
| // for type and initialized code. In addition, init code is stored for code generation | ||
| val arrayName = freshName("mutableStateArray") | ||
| mutableStateArrayCurrentNames += javaType -> arrayName | ||
| // for type. In addition, init code is stored for code generation | ||
| val newArrayName = freshName("mutableStateArray") | ||
| mutableStateArrayCurrentNames += javaType -> newArrayName | ||
| val idx = 0 | ||
| val initCode = codeFunctions(s"$arrayName[$idx]") | ||
| mutableStateArrayIdx += (javaType, newArrayName) -> idx | ||
| val initCode = codeFunctions(s"$newArrayName[$idx]") | ||
| mutableStateArrayInitCodes += initCode | ||
| mutableStateArrayIdx += (javaType, arrayName) -> idx | ||
| s"$arrayName[$idx]" | ||
| s"$newArrayName[$idx]" | ||
| } | ||
| } | ||
| } | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
From the below code, looks like this is keyed by
(javaType, arrayName), instead of(javaType, initCode)?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good catch, old comment still exists there. Thanks