From c213acc1f5728a2cb8dc1cc68f83e13692c6eb5e Mon Sep 17 00:00:00 2001 From: toddwan Date: Sun, 22 Nov 2015 00:21:40 +0800 Subject: [PATCH 1/3] SparkContext accepts invalid Master URLs like 'zk://host:port' for a multi-master Mesos cluster using ZooKeeper --- .../main/scala/org/apache/spark/SparkContext.scala | 11 +++++------ .../spark/SparkContextSchedulerCreationSuite.scala | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index af4456c05b0a1..aadcfd569ab0d 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -2708,15 +2708,14 @@ object SparkContext extends Logging { scheduler.initialize(backend) (backend, scheduler) - case mesosUrl @ MESOS_REGEX(_) => + case MESOS_REGEX(mesosUrl) => MesosNativeLibrary.load() val scheduler = new TaskSchedulerImpl(sc) val coarseGrained = sc.conf.getBoolean("spark.mesos.coarse", defaultValue = true) - val url = mesosUrl.stripPrefix("mesos://") // strip scheme from raw Mesos URLs val backend = if (coarseGrained) { - new CoarseMesosSchedulerBackend(scheduler, sc, url, sc.env.securityManager) + new CoarseMesosSchedulerBackend(scheduler, sc, mesosUrl, sc.env.securityManager) } else { - new MesosSchedulerBackend(scheduler, sc, url) + new MesosSchedulerBackend(scheduler, sc, mesosUrl) } scheduler.initialize(backend) (backend, scheduler) @@ -2745,8 +2744,8 @@ private object SparkMasterRegex { val LOCAL_CLUSTER_REGEX = """local-cluster\[\s*([0-9]+)\s*,\s*([0-9]+)\s*,\s*([0-9]+)\s*]""".r // Regular expression for connecting to Spark deploy clusters val SPARK_REGEX = """spark://(.*)""".r - // Regular expression for connection to Mesos cluster by mesos:// or zk:// url - val MESOS_REGEX = """(mesos|zk)://.*""".r + // Regular expression for connection to Mesos cluster by mesos:// or mesos://zk:// url + val MESOS_REGEX = """mesos://(.*)""".r // Regular expression for connection to Simr cluster val SIMR_REGEX = """simr://(.*)""".r } diff --git a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala index e5a14a69ef05f..c6b0ecbfd68a9 100644 --- a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala @@ -175,6 +175,6 @@ class SparkContextSchedulerCreationSuite } test("mesos with zookeeper") { - testMesos("zk://localhost:1234,localhost:2345", classOf[MesosSchedulerBackend], coarse = false) + testMesos("mesos://zk://localhost:1234,localhost:2345", classOf[MesosSchedulerBackend], coarse = false) } } From 8896264a586f2a2bde2858ff076b19273a5a96c5 Mon Sep 17 00:00:00 2001 From: toddwan Date: Tue, 24 Nov 2015 23:50:24 +0800 Subject: [PATCH 2/3] Warn when the given mesos Master URL starts with zk:// --- core/src/main/scala/org/apache/spark/SparkContext.scala | 5 +++++ .../apache/spark/SparkContextSchedulerCreationSuite.scala | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index aadcfd569ab0d..a057a06d26d5c 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -2726,6 +2726,11 @@ object SparkContext extends Logging { scheduler.initialize(backend) (backend, scheduler) + case zkUrl if zkUrl.startsWith("zk://") => + logWarning("Master URL for a multi-master Mesos cluster managed by ZooKeeper should be " + + "in the form mesos://zk://host:port.") + createTaskScheduler(sc, "mesos://" + zkUrl) + case _ => throw new SparkException("Could not parse Master URL: '" + master + "'") } diff --git a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala index c6b0ecbfd68a9..20cc4b874cbaf 100644 --- a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala @@ -177,4 +177,8 @@ class SparkContextSchedulerCreationSuite test("mesos with zookeeper") { testMesos("mesos://zk://localhost:1234,localhost:2345", classOf[MesosSchedulerBackend], coarse = false) } + + test("mesos with zookeeper and Master URL starting with zk://") { + testMesos("zk://localhost:1234,localhost:2345", classOf[MesosSchedulerBackend], coarse = false) + } } From 1eeb933cff4451744a65e16c4a4694e2212cbcbc Mon Sep 17 00:00:00 2001 From: toddwan Date: Wed, 25 Nov 2015 13:20:54 +0800 Subject: [PATCH 3/3] Fix style test error & explicit warning message --- core/src/main/scala/org/apache/spark/SparkContext.scala | 2 +- .../org/apache/spark/SparkContextSchedulerCreationSuite.scala | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index a057a06d26d5c..ba4d5fc162a7f 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -2728,7 +2728,7 @@ object SparkContext extends Logging { case zkUrl if zkUrl.startsWith("zk://") => logWarning("Master URL for a multi-master Mesos cluster managed by ZooKeeper should be " + - "in the form mesos://zk://host:port.") + "in the form mesos://zk://host:port. Current Master URL will stop working in Spark 2.0.") createTaskScheduler(sc, "mesos://" + zkUrl) case _ => diff --git a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala index 20cc4b874cbaf..d18e0782c0392 100644 --- a/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkContextSchedulerCreationSuite.scala @@ -175,7 +175,8 @@ class SparkContextSchedulerCreationSuite } test("mesos with zookeeper") { - testMesos("mesos://zk://localhost:1234,localhost:2345", classOf[MesosSchedulerBackend], coarse = false) + testMesos("mesos://zk://localhost:1234,localhost:2345", + classOf[MesosSchedulerBackend], coarse = false) } test("mesos with zookeeper and Master URL starting with zk://") {