From dde05f38bdab590bec23c2b4a5b8062e62991036 Mon Sep 17 00:00:00 2001 From: Dario Rexin Date: Thu, 10 Jul 2014 18:39:41 +0200 Subject: [PATCH] mesos executor ids now consist of the slave id and a counter to fix duplicate id problems --- .../cluster/mesos/MesosSchedulerBackend.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala index c717e7c621a8..3f417d73d406 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala @@ -62,6 +62,8 @@ private[spark] class MesosSchedulerBackend( var classLoader: ClassLoader = null + var _nextExecutorId = 0 + override def start() { synchronized { classLoader = Thread.currentThread.getContextClassLoader @@ -250,7 +252,7 @@ private[spark] class MesosSchedulerBackend( MesosTaskInfo.newBuilder() .setTaskId(taskId) .setSlaveId(SlaveID.newBuilder().setValue(slaveId).build()) - .setExecutor(createExecutorInfo(slaveId)) + .setExecutor(createExecutorInfo(nextExecutorId(slaveId))) .setName(task.name) .addResources(cpuResource) .setData(ByteString.copyFrom(task.serializedTask)) @@ -333,4 +335,10 @@ private[spark] class MesosSchedulerBackend( // TODO: query Mesos for number of cores override def defaultParallelism() = sc.conf.getInt("spark.default.parallelism", 8) + + def nextExecutorId(slaveId: String): String = { + val id = s"$slaveId-${_nextExecutorId}" + _nextExecutorId += 1 + id + } }