-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-1103] [WIP] Automatic garbage collection of RDD, shuffle and broadcast data #126
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
1e752f1
80dd977
e427a9e
8512612
a24fefc
cb0a5a6
ae9da88
e61daa0
a7260d3
892b952
e1fba5f
f2881fd
620eca3
a007307
d2f8b97
6c9dcf6
c7ccef1
ba52e00
d0edef3
544ac86
e95479c
f201a8d
c92e4d9
0d17060
34f436f
fbfeec8
88904a3
e442246
8557c12
7edbc98
634a097
7ed72fb
5016375
f0aabb1
762a4d8
a6460d4
c5b1d98
a2cc8bc
ada45f0
cd72d19
b27f8e8
a430f06
104a89a
6222697
41c9ece
2b95b5e
4d05314
cff023c
d25a86e
f489fdc
61b8d6e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -150,7 +150,7 @@ class BlockManagerMasterActor(val isLocal: Boolean, conf: SparkConf) extends Act | |
| private def removeShuffle(shuffleId: Int) { | ||
| // Nothing to do in the BlockManagerMasterActor data structures | ||
| val removeMsg = RemoveShuffle(shuffleId) | ||
| blockManagerInfo.values.map { bm => | ||
| blockManagerInfo.values.foreach { bm => | ||
| bm.slaveActor ! removeMsg | ||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this can be foreach |
||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -51,11 +51,11 @@ private[spark] class TimeStampedWeakValueHashMap[A, B]() | |
| /** Counter for counting the number of inserts */ | ||
| private val insertCounts = new AtomicInteger(0) | ||
|
|
||
| protected[util] val internalJavaMap: util.Map[A, TimeStampedWeakValue[B]] = { | ||
| private[util] val internalJavaMap: util.Map[A, TimeStampedWeakValue[B]] = { | ||
| new ConcurrentHashMap[A, TimeStampedWeakValue[B]]() | ||
| } | ||
|
|
||
| protected[util] def newInstance[K1, V1](): WrappedJavaHashMap[K1, V1, _, _] = { | ||
| private[util] def newInstance[K1, V1](): WrappedJavaHashMap[K1, V1, _, _] = { | ||
| new TimeStampedWeakValueHashMap[K1, V1]() | ||
| } | ||
|
|
||
|
|
@@ -68,15 +68,12 @@ private[spark] class TimeStampedWeakValueHashMap[A, B]() | |
| } | ||
|
|
||
| override def get(key: A): Option[B] = { | ||
| Option(internalJavaMap.get(key)) match { | ||
| case Some(weakValue) => | ||
| val value = weakValue.weakValue.get | ||
| if (value == null) { | ||
| internalJavaMap.remove(key) | ||
| } | ||
| Option(value) | ||
| case None => | ||
| None | ||
| Option(internalJavaMap.get(key)).flatMap { weakValue => | ||
| val value = weakValue.weakValue.get | ||
| if (value == null) { | ||
| internalJavaMap.remove(key) | ||
| } | ||
| Option(value) | ||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How about ?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not the same logic. This when value is null this makes the function return Some(null) instead of None. Changing map to flatMap is the solution. |
||
| } | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this should be private