[SNAPPYDATA] Allow configurable MemoryManager

To plugin SnappyData's Unified Memory Manager (SnappyUnifiedMemoryManager), the
MemoryManager is made configurable using "spark.memory.manager" configuration property
that is set to the SnappyData's manager in embedded mode.

[SNAP-1256] (#41)
set the memory manager as spark's UnifiedMemoryManager, if spark.memory.manager is set as default

[SNAP-2084] (#86)
If SnappyUMM is found in classpath , SparkEnv will assign the memory manager to SnappyUMM.
If user has explicitly set the memory manager that will take precedence.

Author: Shirish Deshmukh

core/src/main/scala/org/apache/spark/SparkEnv.scala

@@ -342,10 +342,18 @@ object SparkEnv extends Logging {
 
     val useLegacyMemoryManager = conf.getBoolean("spark.memory.useLegacyMode", false)
     val memoryManager: MemoryManager =
-      if (useLegacyMemoryManager) {
-        new StaticMemoryManager(conf, numUsableCores)
-      } else {
-        UnifiedMemoryManager(conf, numUsableCores)
+      conf.getOption("spark.memory.manager").filterNot(_.equalsIgnoreCase("default"))
+          .map(Utils.classForName(_)
+              .getConstructor(classOf[SparkConf], classOf[Int])
+              .newInstance(conf, Int.box(numUsableCores))
+              .asInstanceOf[MemoryManager]).getOrElse {
+        SparkSnappyUtils.loadSnappyManager(conf, numUsableCores).getOrElse {
+          if (useLegacyMemoryManager) {
+            new StaticMemoryManager(conf, numUsableCores)
+          } else {
+            UnifiedMemoryManager(conf, numUsableCores)
+          }
+        }
       }
 
     val blockManagerPort = if (isDriver) {

core/src/main/scala/org/apache/spark/SparkSnappyUtils.scala

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Copyright (c) 2017 SnappyData, Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You
+ * may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License. See accompanying
+ * LICENSE file.
+ */
+
+package org.apache.spark
+
+import org.apache.spark.memory.MemoryManager
+import org.apache.spark.util.Utils
+
+
+object SparkSnappyUtils {
+
+  val SNAPPY_UNIFIED_MEMORY_MANAGER_CLASS = "org.apache.spark.memory.SnappyUnifiedMemoryManager"
+
+  def loadSnappyManager(conf: SparkConf, numUsableCores: Int): Option[MemoryManager] = {
+    try {
+      Some(Utils.classForName(SNAPPY_UNIFIED_MEMORY_MANAGER_CLASS)
+          .getConstructor(classOf[SparkConf], classOf[Int])
+          .newInstance(conf, Int.box(numUsableCores))
+          .asInstanceOf[MemoryManager])
+    } catch {
+      case ex: ClassNotFoundException => None
+    }
+  }
+
+}