Skip to content
Closed
Changes from 1 commit
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
3e1cc29
rename: executorMemory -> executorMemoryMB
ryan-williams Feb 10, 2015
cba802c
rename: amMemory -> amMemoryMB
ryan-williams Feb 10, 2015
0a717d7
rename: driverMemory -> driverMemoryMB
ryan-williams Feb 10, 2015
6635945
rename: DEFAULT_MEMORY -> DEFAULT_MEMORY_MB
ryan-williams Feb 10, 2015
96dfec0
rename: amMemoryOverhead -> amMemoryOverheadMB
ryan-williams Feb 10, 2015
5a3e4b8
rename: executorMemoryOverhead -> executorMemoryOverheadMB
ryan-williams Feb 10, 2015
63086eb
rename: amMemoryOverhead -> amMemoryOverheadMB
ryan-williams Feb 10, 2015
f54b0ce
rename: executorMemoryOverhead -> executorMemoryOverheadMB
ryan-williams Feb 10, 2015
fa3d69f
rename: MEMORY_OVERHEAD_MIN -> MEMORY_OVERHEAD_MIN_MB
ryan-williams Feb 10, 2015
f265d15
fix deprecation warning
ryan-williams Feb 9, 2015
c29da1d
rename: executorMemory -> executorMemoryMB
ryan-williams Feb 10, 2015
23a77be
rename: executorMemory -> executorMemoryMB
ryan-williams Feb 10, 2015
5057bd3
rename: memoryOverhead -> memoryOverheadMB
ryan-williams Feb 10, 2015
14bd3d5
rename: memory -> memoryMB
ryan-williams Feb 10, 2015
6e69b08
rename: mem -> memMB
ryan-williams Feb 10, 2015
48c5115
memoryStringToMb can have default scale specified
ryan-williams Nov 13, 2014
dc03bf2
move getMaxResultSize from Utils to SparkConf
ryan-williams Feb 10, 2015
40ac6ce
privatize amMemoryOverheadConf
ryan-williams Feb 10, 2015
dd9be85
refactor memory-size order-of-magnitude constants
ryan-williams Feb 10, 2015
bb66b22
add memory-string-parsing helpers to Utils
ryan-williams Feb 10, 2015
960b525
add `getMemory`, `getMB` helpers to SparkConf
ryan-williams Feb 10, 2015
e038867
use SparkConf.getMB helper in Yarn memory parsing
ryan-williams Feb 10, 2015
2ebb55a
update docs about YARN memory overhead parameters
ryan-williams Feb 10, 2015
50f0f52
code review feedback
ryan-williams Feb 10, 2015
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
rename: amMemoryOverhead -> amMemoryOverheadMB
  • Loading branch information
ryan-williams committed Feb 10, 2015
commit 96dfec088e3151126dcefeedb376fa02391bcf3c
10 changes: 5 additions & 5 deletions yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ private[spark] class Client(
private val yarnClient = YarnClient.createYarnClient
private val yarnConf = new YarnConfiguration(hadoopConf)
private val credentials = UserGroupInformation.getCurrentUser.getCredentials
private val amMemoryOverhead = args.amMemoryOverhead // MB
private val amMemoryOverheadMB = args.amMemoryOverhead
private val executorMemoryOverhead = args.executorMemoryOverhead // MB
private val distCacheMgr = new ClientDistributedCacheManager()
private val isClusterMode = args.isClusterMode
Expand Down Expand Up @@ -126,7 +126,7 @@ private[spark] class Client(
"Cluster's default value will be used.")
}
val capability = Records.newRecord(classOf[Resource])
capability.setMemory(args.amMemoryMB + amMemoryOverhead)
capability.setMemory(args.amMemoryMB + amMemoryOverheadMB)
capability.setVirtualCores(args.amCores)
appContext.setResource(capability)
appContext
Expand Down Expand Up @@ -162,14 +162,14 @@ private[spark] class Client(
throw new IllegalArgumentException(s"Required executor memory (${args.executorMemoryMB}" +
s"+$executorMemoryOverhead MB) is above the max threshold ($maxMem MB) of this cluster!")
}
val amMem = args.amMemoryMB + amMemoryOverhead
val amMem = args.amMemoryMB + amMemoryOverheadMB
if (amMem > maxMem) {
throw new IllegalArgumentException(s"Required AM memory (${args.amMemoryMB}" +
s"+$amMemoryOverhead MB) is above the max threshold ($maxMem MB) of this cluster!")
s"+$amMemoryOverheadMB MB) is above the max threshold ($maxMem MB) of this cluster!")
}
logInfo("Will allocate AM container, with %d MB memory including %d MB overhead".format(
amMem,
amMemoryOverhead))
amMemoryOverheadMB))

// We could add checks to make sure the entire cluster has enough resources but that involves
// getting all the node reports and computing ourselves.
Expand Down