-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-23711][SQL] Add fallback generator for UnsafeProjection #21106
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
fe2a1cd
36f90cf
68213fb
6cdaec4
32e2766
aeef468
1f5cc17
4c04d14
67f8701
e01fe97
de88f88
f883c2b
716d88f
8c56ae4
5a735af
5569646
ed525f6
c9ec817
5527595
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions | |
| import org.codehaus.commons.compiler.CompileException | ||
| import org.codehaus.janino.InternalCompilerException | ||
|
|
||
| import org.apache.spark.TaskContext | ||
| import org.apache.spark.sql.internal.SQLConf | ||
| import org.apache.spark.util.Utils | ||
|
|
||
|
|
@@ -34,23 +35,36 @@ object CodegenError { | |
| } | ||
| } | ||
|
|
||
| /** | ||
| * Defines values for `SQLConf` config of fallback mode. Use for test only. | ||
| */ | ||
| object CodegenObjectFactoryMode extends Enumeration { | ||
| val AUTO, CODEGEN_ONLY, NO_CODEGEN = Value | ||
|
|
||
| def currentMode: CodegenObjectFactoryMode.Value = { | ||
| // If we weren't on task execution, accesses that config. | ||
| if (TaskContext.get == null) { | ||
| val config = SQLConf.get.getConf(SQLConf.CODEGEN_OBJECT_FALLBACK) | ||
| CodegenObjectFactoryMode.withName(config) | ||
| } else { | ||
| CodegenObjectFactoryMode.AUTO | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * A factory which can be used to create objects that have both codegen and interpreted | ||
| * implementations. This tries to create codegen object first, if any compile error happens, | ||
| * it fallbacks to interpreted version. | ||
| */ | ||
| abstract class CodegenObjectFactory[IN, OUT] { | ||
|
||
|
|
||
| // Creates wanted object. First trying codegen implementation. If any compile error happens, | ||
| // fallbacks to interpreted version. | ||
| def createObject(in: IN): OUT = { | ||
| val fallbackMode = SQLConf.get.getConf(SQLConf.CODEGEN_OBJECT_FALLBACK) | ||
| // Only in tests, we can use `SQLConf.CODEGEN_OBJECT_FALLBACK` to choose codegen/interpreted | ||
| // only path. | ||
| if (Utils.isTesting && fallbackMode != "fallback") { | ||
| fallbackMode match { | ||
| case "codegen-only" => createCodeGeneratedObject(in) | ||
| case "interpreted-only" => createInterpretedObject(in) | ||
| // We are allowed to choose codegen-only or no-codegen modes if under tests. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm rethinking about it. Do you think this conf is useful if there is a perf problem in codegen and users want to use interpreted version?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmm, possible. If users are aware of codegen objects like unsafe projection can be controlled too. Currently when we fallback from wholestage codegen to interpreted mode, we still use codegen unsafe projection if it doesn't fail compile. If it has perf problem, users can't do anything.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So do we want to have this conf for not only test but also production? |
||
| if (Utils.isTesting && CodegenObjectFactoryMode.currentMode != CodegenObjectFactoryMode.AUTO) { | ||
| CodegenObjectFactoryMode.currentMode match { | ||
| case CodegenObjectFactoryMode.CODEGEN_ONLY => createCodeGeneratedObject(in) | ||
| case CodegenObjectFactoryMode.NO_CODEGEN => createInterpretedObject(in) | ||
| } | ||
| } else { | ||
| try { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -32,6 +32,7 @@ import org.apache.spark.internal.Logging | |
| import org.apache.spark.internal.config._ | ||
| import org.apache.spark.network.util.ByteUnit | ||
| import org.apache.spark.sql.catalyst.analysis.Resolver | ||
| import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode | ||
| import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator | ||
| import org.apache.spark.util.Utils | ||
|
|
||
|
|
@@ -688,14 +689,14 @@ object SQLConf { | |
|
|
||
| val CODEGEN_OBJECT_FALLBACK = buildConf("spark.sql.test.codegenObject.fallback") | ||
|
||
| .doc("Determines the behavior of any factories extending `CodegenObjectFactory`" + | ||
| " during tests. `fallback` means trying codegen first and then fallbacking to" + | ||
| "interpreted if any compile error happens. Disabling fallback if `codegen-only`." + | ||
| "`interpreted-only` skips codegen and goes interpreted path always. Note that" + | ||
| "this config works only for tests. In production it always runs with `fallback` mode") | ||
| " during tests. `AUTO` means trying codegen first and then fallbacking to" + | ||
| "interpreted if any compile error happens. Disabling fallback if `CODEGEN_ONLY`." + | ||
| "`NO_CODEGEN` skips codegen and goes interpreted path always. Note that" + | ||
| "this config works only for tests.") | ||
| .internal() | ||
| .stringConf | ||
| .checkValues(Set("fallback", "codegen-only", "interpreted-only")) | ||
| .createWithDefault("fallback") | ||
| .checkValues(CodegenObjectFactoryMode.values.map(_.toString)) | ||
| .createWithDefault(CodegenObjectFactoryMode.AUTO.toString) | ||
|
|
||
| val CODEGEN_FALLBACK = buildConf("spark.sql.codegen.fallback") | ||
| .internal() | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -197,7 +197,7 @@ trait ExpressionEvalHelper extends GeneratorDrivenPropertyChecks with PlanTestBa | |
| expression: Expression, | ||
| expected: Any, | ||
| inputRow: InternalRow = EmptyRow): Unit = { | ||
| for (fallbackMode <- Seq("codegen-only", "interpreted-only")) { | ||
| for (fallbackMode <- Seq("CODEGEN_ONLY", "NO_CODEGEN")) { | ||
|
||
| withSQLConf(SQLConf.CODEGEN_OBJECT_FALLBACK.key -> fallbackMode) { | ||
| val factory = UnsafeProjection | ||
| val unsafeRow = evaluateWithUnsafeProjection(expression, inputRow, factory) | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This SQL config can only be useful for unit test against codegen and interpreted unsafe projection.