Skip to content
Closed
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
b34ec0c
make master support multiple executors per worker
CodingCat May 4, 2014
a5d629a
java doc
CodingCat Jan 27, 2015
a26096d
stylistic fix
CodingCat Jan 27, 2015
e5efabb
more java docs and consolidate canUse function
CodingCat Jan 27, 2015
ec7d421
test commit
CodingCat Jan 27, 2015
5b81466
remove outdated comments
CodingCat Jan 27, 2015
19d3da7
address the comments
CodingCat Feb 22, 2015
0b64fea
fix compilation issue
CodingCat Feb 22, 2015
35c462c
address Andrew's comments
CodingCat Feb 22, 2015
387f4ec
bug fix
CodingCat Feb 23, 2015
f64a28d
typo fix
CodingCat Feb 23, 2015
878402c
change the launching executor code
CodingCat Feb 23, 2015
497ec2c
address andrew's comments
CodingCat Mar 27, 2015
2c2bcc5
fix wrong usage info
CodingCat Mar 27, 2015
ff011e2
start multiple executors on the worker by rewriting startExeuctor logic
CodingCat Apr 5, 2015
4cf61f1
improve the code and docs
CodingCat Apr 5, 2015
63b3df9
change the description of the parameter in the submit script
CodingCat Apr 5, 2015
f595bd6
recover some unintentional changes
CodingCat Apr 5, 2015
d9c1685
remove unused var
CodingCat Apr 5, 2015
f035423
stylistic fix
CodingCat Apr 5, 2015
12a1b32
change the semantic of coresPerExecutor to exact core number
CodingCat Apr 9, 2015
2eeff77
stylistic fixes
CodingCat Apr 10, 2015
45967b4
remove unused method
CodingCat Apr 10, 2015
b8ca561
revert a change
CodingCat Apr 10, 2015
940cb42
avoid unnecessary allocation
CodingCat Apr 10, 2015
fbeb7e5
address the comments
CodingCat Apr 14, 2015
6dee808
change filter predicate
CodingCat Apr 14, 2015
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
bug fix
  • Loading branch information
CodingCat committed Apr 9, 2015
commit 387f4ecfcda4671b4ac2461fcf6d543c697074c9
21 changes: 14 additions & 7 deletions core/src/main/scala/org/apache/spark/deploy/master/Master.scala
Original file line number Diff line number Diff line change
Expand Up @@ -564,21 +564,28 @@ private[master] class Master(
// each worker. The first index refers to the usable worker, and the second index
// refers to the executor launched on that worker.
val assigned = Array.fill[Array[Int]](numUsable)(Array.fill[Int](maxExecutorPerWorker)(0))
val workerPointer = Array.fill[Int](numUsable)(0)
val executorNumberOnWorker = Array.fill[Int](numUsable)(0)
val assignedSum = Array.fill[Int](numUsable)(0)
var pos = 0
while (maxCoresLeft > 0) {
if ((usableWorkers(pos).coresFree - assignedSum(pos) > 0) &&
(usableWorkers(pos).memoryFree >= memoryPerExecutor * (workerPointer(pos) + 1))) {
val memoryDemand = {
if (app.desc.maxCorePerExecutor.isDefined) {
executorNumberOnWorker(pos) + 1
} else {
memoryPerExecutor
}
}
if ((usableWorkers(pos).coresFree - assignedSum(pos) > 0) &&
(usableWorkers(pos).memoryFree >= memoryDemand)) {
val coreToAssign = math.min(math.min(usableWorkers(pos).coresFree - assignedSum(pos),
maxCoreAllocationPerRound), maxCoresLeft)
val workerAllocationArray = assigned(pos)
workerAllocationArray(workerPointer(pos)) += coreToAssign
workerAllocationArray(executorNumberOnWorker(pos)) += coreToAssign
assignedSum(pos) += coreToAssign
maxCoresLeft -= coreToAssign
if (app.desc.maxCorePerExecutor.isDefined) {
if (app.desc.maxCorePerExecutor.isDefined || executorNumberOnWorker(pos) == 0) {
// if starting multiple executors on the worker, we move to the next executor
workerPointer(pos) += 1
executorNumberOnWorker(pos) += 1
}
}
pos = (pos + 1) % numUsable
Expand All @@ -587,7 +594,7 @@ private[master] class Master(
// Now that we've decided how many executors and the core number for each to
// give on each node, let's actually give them
for (pos <- 0 until numUsable) {
for (execIdx <- 0 until workerPointer(pos)) {
for (execIdx <- 0 until executorNumberOnWorker(pos)) {
val exec = app.addExecutor(usableWorkers(pos), assigned(pos)(execIdx))
launchExecutor(usableWorkers(pos), exec)
app.state = ApplicationState.RUNNING
Expand Down