From da38bc6839f1fa118d861ed84fb446ac808219b1 Mon Sep 17 00:00:00 2001 From: jerryshao Date: Mon, 14 Mar 2016 16:41:26 +0800 Subject: [PATCH 1/2] Changed the default application exit state --- .../spark/deploy/yarn/ApplicationMaster.scala | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 7d7bf88b9eb1..da7e2d4df2b4 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -152,13 +152,14 @@ private[spark] class ApplicationMaster( val isLastAttempt = client.getAttemptId().getAttemptId() >= maxAppAttempts if (!finished) { - // This happens when the user application calls System.exit(). We have the choice - // of either failing or succeeding at this point. We report success to avoid - // retrying applications that have succeeded (System.exit(0)), which means that - // applications that explicitly exit with a non-zero status will also show up as - // succeeded in the RM UI. + // The default state of ApplicationMaster is failed if it is invoked by shut down hook. + // This behavior is different compared to 1.x version, we guarantee user will not call + // System.exit() at the end of application, so state will be updated before calling + // into shutdown hook. + // If user application is exited ahead of time by calling System.exit(), here mark + // this application as failed with EXIT_EARLY. finish(finalStatus, - ApplicationMaster.EXIT_SUCCESS, + ApplicationMaster.EXIT_EARLY, "Shutdown hook called before final status was reported.") } @@ -209,7 +210,7 @@ private[spark] class ApplicationMaster( */ final def getDefaultFinalStatus(): FinalApplicationStatus = { if (isClusterMode) { - FinalApplicationStatus.SUCCEEDED + FinalApplicationStatus.FAILED } else { FinalApplicationStatus.UNDEFINED } @@ -653,6 +654,7 @@ object ApplicationMaster extends Logging { private val EXIT_SC_NOT_INITED = 13 private val EXIT_SECURITY = 14 private val EXIT_EXCEPTION_USER_CLASS = 15 + private val EXIT_EARLY = 16 private var master: ApplicationMaster = _ From c41739c74bee24429f547bc6546ca25be6fc76bf Mon Sep 17 00:00:00 2001 From: jerryshao Date: Tue, 15 Mar 2016 09:09:19 +0800 Subject: [PATCH 2/2] Update the comments --- .../org/apache/spark/deploy/yarn/ApplicationMaster.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index da7e2d4df2b4..cd179cf32805 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -153,11 +153,10 @@ private[spark] class ApplicationMaster( if (!finished) { // The default state of ApplicationMaster is failed if it is invoked by shut down hook. - // This behavior is different compared to 1.x version, we guarantee user will not call - // System.exit() at the end of application, so state will be updated before calling - // into shutdown hook. - // If user application is exited ahead of time by calling System.exit(), here mark - // this application as failed with EXIT_EARLY. + // This behavior is different compared to 1.x version. + // If user application is exited ahead of time by calling System.exit(N), here mark + // this application as failed with EXIT_EARLY. For a good shutdown, user shouldn't call + // System.exit(0) to terminate the application. finish(finalStatus, ApplicationMaster.EXIT_EARLY, "Shutdown hook called before final status was reported.")