From c70e29f83dd1a0deac28653586ed90876a8a9108 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 24 Jul 2020 13:04:20 +0900 Subject: [PATCH 1/3] Use python3 executable instead of python3.6 in IntegratedUDFTestUtils --- .../scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala index 6391d5664d5c..80346b350c14 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala @@ -197,7 +197,7 @@ object IntegratedUDFTestUtils extends SQLHelper { lazy val pythonExec: String = { val pythonExec = sys.env.getOrElse( - "PYSPARK_DRIVER_PYTHON", sys.env.getOrElse("PYSPARK_PYTHON", "python3.6")) + "PYSPARK_DRIVER_PYTHON", sys.env.getOrElse("PYSPARK_PYTHON", "python3")) if (TestUtils.testCommandAvailable(pythonExec)) { pythonExec } else { From 826689a36efbf1b73a0744824080ac01da3a5f60 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 24 Jul 2020 13:58:28 +0900 Subject: [PATCH 2/3] Revert "[SPARK-32253][INFRA] Show errors only for the sbt tests of github actions" This reverts commit 026b0b926dfd40038f2cee932f38b917eb25b77e. --- dev/run-tests.py | 3 --- project/SparkBuild.scala | 7 ------- 2 files changed, 10 deletions(-) diff --git a/dev/run-tests.py b/dev/run-tests.py index 8e29f89c3a0d..065a27c0e853 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -653,9 +653,6 @@ def main(): # If we're running the tests in Github Actions, attempt to detect and test # only the affected modules. if test_env == "github_actions": - # Set the log level of sbt as ERROR to make the output more readable. - if build_tool == "sbt": - extra_profiles.append("--error") if os.environ["GITHUB_BASE_REF"] != "": # Pull requests changed_files = identify_changed_files_from_git_commits( diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index c05e41cfcc0c..5ff9d14a5aa1 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1031,13 +1031,6 @@ object TestSettings { }.getOrElse(Nil): _*), // Show full stack trace and duration in test cases. testOptions in Test += Tests.Argument("-oDF"), - // Show only the failed test cases with full stack traces in github action to make the log more - // readable. - // Check https://www.scalatest.org/user_guide/using_the_runner for the details of options . - testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest, - sys.env.get("GITHUB_ACTIONS").map { _ => - Seq("-eNCXEHLOPQMDF") - }.getOrElse(Nil): _*), testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"), // Required to detect Junit tests for each project, see also https://github.com/sbt/junit-interface/issues/35 crossPaths := false, From d72c4c12bf61aab4dfd93824cef801d7d93e7a4c Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Fri, 24 Jul 2020 13:58:59 +0900 Subject: [PATCH 3/3] Revert "Revert "[SPARK-32253][INFRA] Show errors only for the sbt tests of github actions"" This reverts commit 826689a36efbf1b73a0744824080ac01da3a5f60. --- dev/run-tests.py | 3 +++ project/SparkBuild.scala | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/dev/run-tests.py b/dev/run-tests.py index 065a27c0e853..8e29f89c3a0d 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -653,6 +653,9 @@ def main(): # If we're running the tests in Github Actions, attempt to detect and test # only the affected modules. if test_env == "github_actions": + # Set the log level of sbt as ERROR to make the output more readable. + if build_tool == "sbt": + extra_profiles.append("--error") if os.environ["GITHUB_BASE_REF"] != "": # Pull requests changed_files = identify_changed_files_from_git_commits( diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 5ff9d14a5aa1..c05e41cfcc0c 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1031,6 +1031,13 @@ object TestSettings { }.getOrElse(Nil): _*), // Show full stack trace and duration in test cases. testOptions in Test += Tests.Argument("-oDF"), + // Show only the failed test cases with full stack traces in github action to make the log more + // readable. + // Check https://www.scalatest.org/user_guide/using_the_runner for the details of options . + testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest, + sys.env.get("GITHUB_ACTIONS").map { _ => + Seq("-eNCXEHLOPQMDF") + }.getOrElse(Nil): _*), testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"), // Required to detect Junit tests for each project, see also https://github.com/sbt/junit-interface/issues/35 crossPaths := false,