Skip to content

[SPARK-23475][UI] Show also skipped stages #20651

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mgaido91 wants to merge 3 commits into from
Closed

[SPARK-23475][UI] Show also skipped stages #20651

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5f5d0da
[SPARK-23475][UI] Show also skipped stages
mgaido91 Feb 21, 2018
5cedd74
add collapseTablePageLoad
mgaido91 Feb 22, 2018
e04bf24
added UT
mgaido91 Feb 22, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/src/main/resources/org/apache/spark/ui/static/webui.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ $(function() {
collapseTablePageLoad('collapse-aggregated-allActiveStages','aggregated-allActiveStages');
collapseTablePageLoad('collapse-aggregated-allPendingStages','aggregated-allPendingStages');
collapseTablePageLoad('collapse-aggregated-allCompletedStages','aggregated-allCompletedStages');
collapseTablePageLoad('collapse-aggregated-allSkippedStages','aggregated-allSkippedStages');
collapseTablePageLoad('collapse-aggregated-allFailedStages','aggregated-allFailedStages');
collapseTablePageLoad('collapse-aggregated-activeStages','aggregated-activeStages');
collapseTablePageLoad('collapse-aggregated-pendingOrSkippedStages','aggregated-pendingOrSkippedStages');
Expand Down
27 changes: 27 additions & 0 deletions core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {

val activeStages = allStages.filter(_.status == StageStatus.ACTIVE)
val pendingStages = allStages.filter(_.status == StageStatus.PENDING)
val skippedStages = allStages.filter(_.status == StageStatus.SKIPPED)
val completedStages = allStages.filter(_.status == StageStatus.COMPLETE)
val failedStages = allStages.filter(_.status == StageStatus.FAILED).reverse

Expand All @@ -51,6 +52,9 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
val completedStagesTable =
new StageTableBase(parent.store, request, completedStages, "completed", "completedStage",
parent.basePath, subPath, parent.isFairScheduler, false, false)
val skippedStagesTable =
new StageTableBase(parent.store, request, skippedStages, "skipped", "skippedStage",
parent.basePath, subPath, parent.isFairScheduler, false, false)
val failedStagesTable =
new StageTableBase(parent.store, request, failedStages, "failed", "failedStage",
parent.basePath, subPath, parent.isFairScheduler, false, true)
Expand All @@ -66,6 +70,7 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
val shouldShowActiveStages = activeStages.nonEmpty
val shouldShowPendingStages = pendingStages.nonEmpty
val shouldShowCompletedStages = completedStages.nonEmpty
val shouldShowSkippedStages = skippedStages.nonEmpty
val shouldShowFailedStages = failedStages.nonEmpty

val appSummary = parent.store.appSummary()
Expand Down Expand Up @@ -102,6 +107,14 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
</li>
}
}
{
if (shouldShowSkippedStages) {
<li id="completed-summary">
<a href="#skipped"><strong>Skipped Stages:</strong></a>
{skippedStages.size}
</li>
}
}
{
if (shouldShowFailedStages) {
<li>
Expand Down Expand Up @@ -172,6 +185,20 @@ private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
{completedStagesTable.toNodeSeq}
</div>
}
if (shouldShowSkippedStages) {
content ++=
<span id="skipped" class="collapse-aggregated-allSkippedStages collapse-table"
Copy link
Contributor

@vanzin vanzin Feb 21, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see things like collapse-aggregated-allCompletedStages are mentioned in webui.js, so probably this one should be added there.

This is kinda past the point where this block should be refactored (it's repeated 5 times now), but probably ok for a 2.3 change.

Copy link
Contributor Author

@mgaido91 mgaido91 Feb 22, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks, I added the UT. I think we can create a new JIRA/followup PR to resolve the code redundancy issue you rightly pointed out, what do you think?

Copy link
Contributor

@vanzin vanzin Feb 22, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good to clean things up in a follow-up.

onClick="collapseTable('collapse-aggregated-allSkippedStages',
'aggregated-allSkippedStages')">
<h4>
<span class="collapse-table-arrow arrow-open"></span>
<a>Skipped Stages ({skippedStages.size})</a>
</h4>
</span> ++
<div class="aggregated-allSkippedStages collapsible-table">
{skippedStagesTable.toNodeSeq}
</div>
}
if (shouldShowFailedStages) {
content ++=
<span id ="failed" class="collapse-aggregated-allFailedStages collapse-table"
Expand Down
17 changes: 17 additions & 0 deletions core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -707,6 +707,23 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
}
}

test("stages page should show skipped stages") {
withSpark(newSparkContext()) { sc =>
val rdd = sc.parallelize(0 to 100, 100).repartition(10).cache()
rdd.count()
rdd.count()

eventually(timeout(5 seconds), interval(50 milliseconds)) {
goToUi(sc, "/stages")
find(id("skipped")).get.text should be("Skipped Stages (1)")
}
val stagesJson = getJson(sc.ui.get, "stages")
stagesJson.children.size should be (4)
val stagesStatus = stagesJson.children.map(_ \ "status")
stagesStatus.count(_ == JString(StageStatus.SKIPPED.name())) should be (1)
}
}

def goToUi(sc: SparkContext, path: String): Unit = {
goToUi(sc.ui.get, path)
}
Expand Down