[SPARK-5932] Updated spark.shuffle.file.buffer.kb

Author: Ilya Ganelin

core/src/main/scala/org/apache/spark/SparkConf.scala

@@ -480,7 +480,11 @@ private[spark] object SparkConf extends Logging {
       DeprecatedConfig("spark.kryoserializer.buffer.mb", "1.4",
         "Please use spark.kryoserializer.buffer instead."),
       DeprecatedConfig("spark.kryoserializer.buffer.max.mb", "1.4",
-        "Please use spark.kryoserializer.buffer.max instead."))
+        "Please use spark.kryoserializer.buffer.max instead."),
+      DeprecatedConfig("spark.shuffle.file.buffer.kb", "1.4",
+        "Please use spark.shuffle.file.buffer instead."))
+
+    
 
 
     Map(configs.map { cfg => (cfg.key -> cfg) }:_*)
@@ -512,7 +516,9 @@ private[spark] object SparkConf extends Logging {
     "spark.kryoserializer.buffer" -> Seq(
       AlternateConfig("spark.kryoserializer.buffer.mb", "1.4")),
     "spark.kryoserializer.buffer.max" -> Seq(
-      AlternateConfig("spark.kryoserializer.buffer.max.mb", "1.4"))
+      AlternateConfig("spark.kryoserializer.buffer.max.mb", "1.4")),
+    "spark.shuffle.file.buffer" -> Seq(
+      AlternateConfig("spark.shuffle.file.buffer.kb", "1.4"))
   )
 
   /**

core/src/main/scala/org/apache/spark/shuffle/FileShuffleBlockManager.scala

@@ -78,7 +78,7 @@ class FileShuffleBlockManager(conf: SparkConf)
   private val consolidateShuffleFiles =
     conf.getBoolean("spark.shuffle.consolidateFiles", false)
 
-  private val bufferSize = conf.getInt("spark.shuffle.file.buffer.kb", 32) * 1024
+  private val bufferSize = conf.getSizeAsKb("spark.shuffle.file.buffer", "32k") * 1024
 
   /**
    * Contains all the state related to a particular shuffle. This includes a pool of unused

core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala

@@ -90,7 +90,7 @@ class ExternalAppendOnlyMap[K, V, C](
   // Number of bytes spilled in total
   private var _diskBytesSpilled = 0L
 
-  private val fileBufferSize = sparkConf.getInt("spark.shuffle.file.buffer.kb", 32) * 1024
+  private val fileBufferSize = sparkConf.getSizeAsKb("spark.shuffle.file.buffer", "32k") * 1024
 
   // Write metrics for current spill
   private var curWriteMetrics: ShuffleWriteMetrics = _

core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala

@@ -97,7 +97,7 @@ private[spark] class ExternalSorter[K, V, C](
 
   private val conf = SparkEnv.get.conf
   private val spillingEnabled = conf.getBoolean("spark.shuffle.spill", true)
-  private val fileBufferSize = conf.getInt("spark.shuffle.file.buffer.kb", 32) * 1024
+  private val fileBufferSize = conf.getSizeAsKb("spark.shuffle.file.buffer", "32k") * 1024
   private val transferToEnabled = conf.getBoolean("spark.file.transferTo", true)
 
   // Size of object batches when reading/writing from serializers.