Skip to content

[SPARK-9103][WIP] Track Netty memory usage - take two #17762

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jsoltren wants to merge 12 commits into from
Closed

[SPARK-9103][WIP] Track Netty memory usage - take two #17762

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
1e51c73
Netty network layer memory usage on webUI
jsoltren Mar 28, 2017
543cd88
more fixups to get things compiling
jsoltren Apr 4, 2017
b794b86
fix JsonProtocolSuite
jsoltren Apr 4, 2017
941eef0
Some fixups related to internal code review. More are coming.
jsoltren Apr 11, 2017
72ed41e
delete stragglers
jsoltren Apr 11, 2017
7cddd6c
Doc changes and minor tweaks in preparation for PR
jsoltren Apr 12, 2017
1bb14b4
s/Happen Time/Peak Time
jsoltren Apr 12, 2017
f3e5704
fix merge conflict
jsoltren Apr 12, 2017
577d442
Respond to some code review feedback. Make executorMetrics an Option.
jsoltren Apr 17, 2017
0ebcec0
Get tests to compile after making ExecutorMetrics an Option
jsoltren Apr 24, 2017
f3d8aa5
More SQL test fixes
jsoltren Apr 24, 2017
550c2d8
Remove UI parts of this change set
jsoltren Apr 25, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Remove UI parts of this change set 
Refactor MemoryListener to core from UI.
  • Loading branch information
@jsoltren
jsoltren committed Apr 25, 2017
commit 550c2d8bd57a55e2801f9e5569b0425a62558255
10 changes: 1 addition & 9 deletions ...rg/apache/spark/ui/memory/MemoryTab.scala → .../apache/spark/memory/MemoryListener.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,14 @@
* limitations under the License.
*/

package org.apache.spark.ui.memory
package org.apache.spark.memory

import scala.collection.mutable.HashMap

import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.executor.{ExecutorMetrics, TransportMetrics}
import org.apache.spark.scheduler._
import org.apache.spark.scheduler.cluster.ExecutorInfo
import org.apache.spark.ui.{SparkUI, SparkUITab}

private[ui] class MemoryTab(parent: SparkUI) extends SparkUITab(parent, "memory") {
val memoryListener = parent.memoryListener
val progressListener = parent.jobProgressListener
attachPage(new MemoryPage(this))
attachPage(new StageMemoryPage(this))
}

/**
* :: DeveloperApi ::
Expand Down
9 changes: 2 additions & 7 deletions core/src/main/scala/org/apache/spark/ui/SparkUI.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ import org.apache.spark.ui.JettyUtils._
import org.apache.spark.ui.env.{EnvironmentListener, EnvironmentTab}
import org.apache.spark.ui.exec.{ExecutorsListener, ExecutorsTab}
import org.apache.spark.ui.jobs.{JobProgressListener, JobsTab, StagesTab}
import org.apache.spark.ui.memory.{MemoryListener, MemoryTab}
import org.apache.spark.ui.scope.RDDOperationGraphListener
import org.apache.spark.ui.storage.{StorageListener, StorageTab}
import org.apache.spark.util.Utils
Expand All @@ -48,7 +47,6 @@ private[spark] class SparkUI private (
val executorsListener: ExecutorsListener,
val jobProgressListener: JobProgressListener,
val storageListener: StorageListener,
val memoryListener: MemoryListener,
val operationGraphListener: RDDOperationGraphListener,
var appName: String,
val basePath: String,
Expand All @@ -73,7 +71,6 @@ private[spark] class SparkUI private (
attachTab(new StorageTab(this))
attachTab(new EnvironmentTab(this))
attachTab(new ExecutorsTab(this))
attachTab(new MemoryTab(this))
attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static"))
attachHandler(createRedirectHandler("/", "/jobs/", basePath = basePath))
attachHandler(ApiRootResource.getServletHandler(this))
Expand Down Expand Up @@ -213,18 +210,16 @@ private[spark] object SparkUI {
val storageStatusListener = new StorageStatusListener(conf)
val executorsListener = new ExecutorsListener(storageStatusListener, conf)
val storageListener = new StorageListener(storageStatusListener)
val memoryListener = new MemoryListener
val operationGraphListener = new RDDOperationGraphListener(conf)

listenerBus.addListener(environmentListener)
listenerBus.addListener(storageStatusListener)
listenerBus.addListener(executorsListener)
listenerBus.addListener(storageListener)
listenerBus.addListener(memoryListener)
listenerBus.addListener(operationGraphListener)

new SparkUI(sc, conf, securityManager, environmentListener, storageStatusListener,
executorsListener, _jobProgressListener, storageListener, memoryListener,
operationGraphListener, appName, basePath, startTime)
executorsListener, _jobProgressListener, storageListener, operationGraphListener,
appName, basePath, startTime)
}
}
109 changes: 0 additions & 109 deletions core/src/main/scala/org/apache/spark/ui/memory/MemoryPage.scala
View file
Open in desktop

This file was deleted.

143 changes: 0 additions & 143 deletions core/src/main/scala/org/apache/spark/ui/memory/MemoryTable.scala
View file
Open in desktop

This file was deleted.

52 changes: 0 additions & 52 deletions core/src/main/scala/org/apache/spark/ui/memory/StageMemoryPage.scala
View file
Open in desktop

This file was deleted.

1 change: 1 addition & 0 deletions core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import org.apache.spark._
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.internal.Logging
import org.apache.spark.io._
import org.apache.spark.memory.{MemoryListener, MemTime}
import org.apache.spark.util.{JsonProtocol, Utils}

/**
Expand Down
1 change: 1 addition & 0 deletions core/src/test/scala/org/apache/spark/ui/memory/MemoryListenerSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package org.apache.spark.ui.memory

import org.apache.spark._
import org.apache.spark.executor._
import org.apache.spark.memory.{MemoryListener, MemTime, TransportMemSize}
import org.apache.spark.scheduler._
import org.apache.spark.scheduler.cluster._

Expand Down