Don’t display pending stages for completed jobs.

core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala

@@ -22,13 +22,13 @@ import scala.xml.{NodeSeq, Node}
 
 import javax.servlet.http.HttpServletRequest
 
+import org.apache.spark.JobExecutionStatus
 import org.apache.spark.scheduler.StageInfo
 import org.apache.spark.ui.{UIUtils, WebUIPage}
 
 /** Page showing statistics and stage list for a given job */
 private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") {
   private val listener = parent.listener
-  private val sc = parent.sc
 
   def render(request: HttpServletRequest): Seq[Node] = {
     listener.synchronized {
@@ -43,6 +43,7 @@ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") {
           s"Details for Job $jobId", content, parent)
       }
       val jobData = jobDataOption.get
+      val isComplete = jobData.status != JobExecutionStatus.RUNNING
       val stages = jobData.stageIds.map { stageId =>
         // This could be empty if the JobProgressListener hasn't received information about the
         // stage or if the stage information has been garbage collected
@@ -55,7 +56,7 @@ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") {
       val completedStages = mutable.Buffer[StageInfo]()
       val failedStages = mutable.Buffer[StageInfo]()
       for (stage <- stages) {
-        if (stage.submissionTime.isEmpty) {
+        if (!isComplete && stage.submissionTime.isEmpty) {
           pendingStages += stage
         } else if (stage.completionTime.isDefined) {
           if (stage.failureReason.isDefined) {

core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala

@@ -221,4 +221,21 @@ class UISeleniumSuite extends FunSuite with WebBrowser with Matchers {
       }
     }
   }
+
+  test("stages that aren't run do not show up in 'pending stages' after a job finishes") {
+    withSpark(newSparkContext()) { sc =>
+      // Create an RDD that involves multiple stages:
+      val rdd =
+        sc.parallelize(Seq(1, 2, 3)).map(identity).groupBy(identity).map(identity).groupBy(identity)
+      // Run it twice; this will cause the second job to have two "phantom" stages that were
+      // mentioned in its job start event but which were never actually executed:
+      rdd.count()
+      rdd.count()
+      // Check that the "pending stages" section is empty:
+      eventually(timeout(10 seconds), interval(50 milliseconds)) {
+        go to (sc.ui.get.appUIAddress.stripSuffix("/") + "/jobs/job/?id=1")
+        find(id("pending")).get.text should be ("Pending Stages (0)")
+      }
+    }
+  }
 }