diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 3042a27cdb83..1cc7836e93d3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -1324,7 +1324,7 @@ object CodeGenerator extends Logging { // Reset compile time. // Visible for testing - def resetCompileTime: Unit = _compileTime.reset() + def resetCompileTime(): Unit = _compileTime.reset() /** * Compile the Java source code into a Java class, using Janino. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala index 9f6e4fcdbdd2..024454205461 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala @@ -586,7 +586,7 @@ object WholeStageCodegenExec { // Reset generation time of Java source code. // Visible for testing - def resetCodeGenTime: Unit = _codeGenTime.set(0L) + def resetCodeGenTime(): Unit = _codeGenTime.set(0L) } /** diff --git a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala index 07afd4195c3d..2c3b37a1498e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql import org.apache.spark.internal.config.Tests.IS_TESTING import org.apache.spark.sql.catalyst.expressions.codegen.{ByteCodeStats, CodeFormatter, CodeGenerator} import org.apache.spark.sql.catalyst.rules.RuleExecutor +import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND import org.apache.spark.sql.execution.{SparkPlan, WholeStageCodegenExec} import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.util.Utils @@ -36,7 +37,17 @@ abstract class BenchmarkQueryTest extends QueryTest with SharedSparkSession { protected override def afterAll(): Unit = { try { // For debugging dump some statistics about how much time was spent in various optimizer rules + // code generation, and compilation. logWarning(RuleExecutor.dumpTimeSpent()) + val codeGenTime = WholeStageCodegenExec.codeGenTime.toDouble / NANOS_PER_SECOND + val compileTime = CodeGenerator.compileTime.toDouble / NANOS_PER_SECOND + val codegenInfo = + s""" + |=== Metrics of Whole-stage Codegen === + |Total code generation time: $codeGenTime seconds + |Total compile time: $compileTime seconds + """.stripMargin + logWarning(codegenInfo) spark.sessionState.catalog.reset() } finally { super.afterAll() @@ -46,6 +57,8 @@ abstract class BenchmarkQueryTest extends QueryTest with SharedSparkSession { override def beforeAll(): Unit = { super.beforeAll() RuleExecutor.resetMetrics() + CodeGenerator.resetCompileTime() + WholeStageCodegenExec.resetCodeGenTime() } protected def checkGeneratedCode(plan: SparkPlan, checkMethodCodeSize: Boolean = true): Unit = { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index bc9b4d85aacd..2b977e74ebd1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -683,8 +683,8 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { // Add Locale setting Locale.setDefault(Locale.US) RuleExecutor.resetMetrics() - CodeGenerator.resetCompileTime - WholeStageCodegenExec.resetCodeGenTime + CodeGenerator.resetCompileTime() + WholeStageCodegenExec.resetCodeGenTime() } override def afterAll(): Unit = { @@ -696,12 +696,13 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession { // For debugging dump some statistics about how much time was spent in various optimizer rules logWarning(RuleExecutor.dumpTimeSpent()) - val generateJavaTime = WholeStageCodegenExec.codeGenTime + val codeGenTime = WholeStageCodegenExec.codeGenTime.toDouble / NANOS_PER_SECOND + val compileTime = CodeGenerator.compileTime.toDouble / NANOS_PER_SECOND val codegenInfo = s""" - |=== Metrics of Whole-Stage Codegen === - |Total code generation time: ${generateJavaTime.toDouble / NANOS_PER_SECOND} seconds - |Total compile time: ${CodeGenerator.compileTime.toDouble / NANOS_PER_SECOND} seconds + |=== Metrics of Whole-stage Codegen === + |Total code generation time: $codeGenTime seconds + |Total compile time: $compileTime seconds """.stripMargin logWarning(codegenInfo) } finally {