Skip to content

Commit 6bf5f01

Browse files
gatorsmilemaropu
authored andcommitted
[SPARK-31477][SQL] Dump codegen and compile time in BenchmarkQueryTest
### What changes were proposed in this pull request? This PR is to dump the codegen and compilation time for benchmark query tests. ### Why are the changes needed? Measure the codegen and compilation time costs in TPC-DS queries ### Does this PR introduce any user-facing change? No ### How was this patch tested? Manual test in my local laptop: ``` 23:13:12.845 WARN org.apache.spark.sql.TPCDSQuerySuite: === Metrics of Whole-stage Codegen === Total code generation time: 21.275102261 seconds Total compilation time: 12.223771828 seconds ``` Closes apache#28252 from gatorsmile/testMastercode. Authored-by: gatorsmile <[email protected]> Signed-off-by: Takeshi Yamamuro <[email protected]>
1 parent 77cb7cd commit 6bf5f01

File tree

4 files changed

+22
-8
lines changed

4 files changed

+22
-8
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1324,7 +1324,7 @@ object CodeGenerator extends Logging {
13241324

13251325
// Reset compile time.
13261326
// Visible for testing
1327-
def resetCompileTime: Unit = _compileTime.reset()
1327+
def resetCompileTime(): Unit = _compileTime.reset()
13281328

13291329
/**
13301330
* Compile the Java source code into a Java class, using Janino.

sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -586,7 +586,7 @@ object WholeStageCodegenExec {
586586

587587
// Reset generation time of Java source code.
588588
// Visible for testing
589-
def resetCodeGenTime: Unit = _codeGenTime.set(0L)
589+
def resetCodeGenTime(): Unit = _codeGenTime.set(0L)
590590
}
591591

592592
/**

sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package org.apache.spark.sql
2020
import org.apache.spark.internal.config.Tests.IS_TESTING
2121
import org.apache.spark.sql.catalyst.expressions.codegen.{ByteCodeStats, CodeFormatter, CodeGenerator}
2222
import org.apache.spark.sql.catalyst.rules.RuleExecutor
23+
import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND
2324
import org.apache.spark.sql.execution.{SparkPlan, WholeStageCodegenExec}
2425
import org.apache.spark.sql.test.SharedSparkSession
2526
import org.apache.spark.util.Utils
@@ -36,7 +37,17 @@ abstract class BenchmarkQueryTest extends QueryTest with SharedSparkSession {
3637
protected override def afterAll(): Unit = {
3738
try {
3839
// For debugging dump some statistics about how much time was spent in various optimizer rules
40+
// code generation, and compilation.
3941
logWarning(RuleExecutor.dumpTimeSpent())
42+
val codeGenTime = WholeStageCodegenExec.codeGenTime.toDouble / NANOS_PER_SECOND
43+
val compileTime = CodeGenerator.compileTime.toDouble / NANOS_PER_SECOND
44+
val codegenInfo =
45+
s"""
46+
|=== Metrics of Whole-stage Codegen ===
47+
|Total code generation time: $codeGenTime seconds
48+
|Total compile time: $compileTime seconds
49+
""".stripMargin
50+
logWarning(codegenInfo)
4051
spark.sessionState.catalog.reset()
4152
} finally {
4253
super.afterAll()
@@ -46,6 +57,8 @@ abstract class BenchmarkQueryTest extends QueryTest with SharedSparkSession {
4657
override def beforeAll(): Unit = {
4758
super.beforeAll()
4859
RuleExecutor.resetMetrics()
60+
CodeGenerator.resetCompileTime()
61+
WholeStageCodegenExec.resetCodeGenTime()
4962
}
5063

5164
protected def checkGeneratedCode(plan: SparkPlan, checkMethodCodeSize: Boolean = true): Unit = {

sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -683,8 +683,8 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
683683
// Add Locale setting
684684
Locale.setDefault(Locale.US)
685685
RuleExecutor.resetMetrics()
686-
CodeGenerator.resetCompileTime
687-
WholeStageCodegenExec.resetCodeGenTime
686+
CodeGenerator.resetCompileTime()
687+
WholeStageCodegenExec.resetCodeGenTime()
688688
}
689689

690690
override def afterAll(): Unit = {
@@ -696,12 +696,13 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
696696
// For debugging dump some statistics about how much time was spent in various optimizer rules
697697
logWarning(RuleExecutor.dumpTimeSpent())
698698

699-
val generateJavaTime = WholeStageCodegenExec.codeGenTime
699+
val codeGenTime = WholeStageCodegenExec.codeGenTime.toDouble / NANOS_PER_SECOND
700+
val compileTime = CodeGenerator.compileTime.toDouble / NANOS_PER_SECOND
700701
val codegenInfo =
701702
s"""
702-
|=== Metrics of Whole-Stage Codegen ===
703-
|Total code generation time: ${generateJavaTime.toDouble / NANOS_PER_SECOND} seconds
704-
|Total compile time: ${CodeGenerator.compileTime.toDouble / NANOS_PER_SECOND} seconds
703+
|=== Metrics of Whole-stage Codegen ===
704+
|Total code generation time: $codeGenTime seconds
705+
|Total compile time: $compileTime seconds
705706
""".stripMargin
706707
logWarning(codegenInfo)
707708
} finally {

0 commit comments

Comments
 (0)