Close SparkContext after the Main method has finished, to allow SparkApplication on K8S to complete

kotlovs · kotlovs · commit e3946b459533 · 2021-04-08T14:56:54.000+03:00
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -954,7 +954,7 @@ private[spark] class SparkSubmit extends Logging {
         throw findCause(t)
     } finally {
       try {
-        org.apache.spark.SparkContext.getOrCreate().stop()
+        SparkContext.getActive.foreach(_.stop())
       } catch {
         case e: Throwable => logError(s"Failed to close SparkContext: $e")
       }

Original file line number	Diff line number	Diff line change
`@@ -954,7 +954,7 @@ private[spark] class SparkSubmit extends Logging {`
`954`	`954`	`throw findCause(t)`
`955`	`955`	`} finally {`
`956`	`956`	`try {`
`957`		`- org.apache.spark.SparkContext.getOrCreate().stop()`
	`957`	`+ SparkContext.getActive.foreach(_.stop())`
`958`	`958`	`} catch {`
`959`	`959`	`case e: Throwable => logError(s"Failed to close SparkContext: $e")`
`960`	`960`	`}`