merge upstream

sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala

@@ -138,17 +138,18 @@ case class SetCommand(kv: Option[(String, Option[String])]) extends RunnableComm
     // Queries all key-value pairs that are set in the SQLConf of the sqlContext.
     case None =>
       val runFunc = (sqlContext: SQLContext) => {
-        sqlContext.getAllConfs.map { case (k, v) => Row(k, v) }.toSeq
+        sqlContext.getAllConfs.toSeq.sortBy(_._1).map { case (k, v) => Row(k, v) } ++
+          getEnvList(withDoc = false)
       }
       (keyValueOutput, runFunc)
 
     // Queries all properties along with their default values and docs that are defined in the
     // SQLConf of the sqlContext.
     case Some(("-v", None)) =>
       val runFunc = (sqlContext: SQLContext) => {
-        sqlContext.conf.getAllDefinedConfs.map { case (key, defaultValue, doc) =>
+        sqlContext.conf.getAllDefinedConfs.sortBy(_._1).map { case (key, defaultValue, doc) =>
           Row(key, defaultValue, doc)
-        }
+        } ++ getEnvList(withDoc = true)
       }
       val schema = StructType(
         StructField("key", StringType, nullable = false) ::
@@ -182,4 +183,18 @@ case class SetCommand(kv: Option[(String, Option[String])]) extends RunnableComm
 
   override def run(sqlContext: SQLContext): Seq[Row] = runFunc(sqlContext)
 
+  /**
+   * get the system environment properties as a sequence
+   *
+   * @param withDoc whether the result has a doc column or not
+   * @return the sequence of the rows containing the key/value pair of system properties
+   */
+  private def getEnvList(withDoc: Boolean) = {
+    sys.env.toSeq.sortBy(_._1).map {
+      case (k, v) => if (withDoc) Row(s"env:$k", v, "") else Row(s"env:$k", v)
+    } ++
+      sys.props.toSeq.sortBy(_._1).map {
+        case (k, v) => if (withDoc) Row(s"system:$k", v, "") else Row(s"system:$k", v)
+      }
+  }
 }

sql/core/src/main/scala/org/apache/spark/sql/execution/command/commands.scala

@@ -73,170 +73,6 @@ private[sql] case class ExecutedCommand(cmd: RunnableCommand) extends SparkPlan
 }
 
 
-case class SetCommand(kv: Option[(String, Option[String])]) extends RunnableCommand with Logging {
-
-  private def keyValueOutput: Seq[Attribute] = {
-    val schema = StructType(
-      StructField("key", StringType, false) ::
-        StructField("value", StringType, false) :: Nil)
-    schema.toAttributes
-  }
-
-  private val (_output, runFunc): (Seq[Attribute], SQLContext => Seq[Row]) = kv match {
-    // Configures the deprecated "mapred.reduce.tasks" property.
-    case Some((SQLConf.Deprecated.MAPRED_REDUCE_TASKS, Some(value))) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        logWarning(
-          s"Property ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS} is deprecated, " +
-            s"automatically converted to ${SQLConf.SHUFFLE_PARTITIONS.key} instead.")
-        if (value.toInt < 1) {
-          val msg =
-            s"Setting negative ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS} for automatically " +
-              "determining the number of reducers is not supported."
-          throw new IllegalArgumentException(msg)
-        } else {
-          sqlContext.setConf(SQLConf.SHUFFLE_PARTITIONS.key, value)
-          Seq(Row(SQLConf.SHUFFLE_PARTITIONS.key, value))
-        }
-      }
-      (keyValueOutput, runFunc)
-
-    case Some((SQLConf.Deprecated.EXTERNAL_SORT, Some(value))) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        logWarning(
-          s"Property ${SQLConf.Deprecated.EXTERNAL_SORT} is deprecated and will be ignored. " +
-            s"External sort will continue to be used.")
-        Seq(Row(SQLConf.Deprecated.EXTERNAL_SORT, "true"))
-      }
-      (keyValueOutput, runFunc)
-
-    case Some((SQLConf.Deprecated.USE_SQL_AGGREGATE2, Some(value))) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        logWarning(
-          s"Property ${SQLConf.Deprecated.USE_SQL_AGGREGATE2} is deprecated and " +
-            s"will be ignored. ${SQLConf.Deprecated.USE_SQL_AGGREGATE2} will " +
-            s"continue to be true.")
-        Seq(Row(SQLConf.Deprecated.USE_SQL_AGGREGATE2, "true"))
-      }
-      (keyValueOutput, runFunc)
-
-    case Some((SQLConf.Deprecated.TUNGSTEN_ENABLED, Some(value))) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        logWarning(
-          s"Property ${SQLConf.Deprecated.TUNGSTEN_ENABLED} is deprecated and " +
-            s"will be ignored. Tungsten will continue to be used.")
-        Seq(Row(SQLConf.Deprecated.TUNGSTEN_ENABLED, "true"))
-      }
-      (keyValueOutput, runFunc)
-
-    case Some((SQLConf.Deprecated.CODEGEN_ENABLED, Some(value))) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        logWarning(
-          s"Property ${SQLConf.Deprecated.CODEGEN_ENABLED} is deprecated and " +
-            s"will be ignored. Codegen will continue to be used.")
-        Seq(Row(SQLConf.Deprecated.CODEGEN_ENABLED, "true"))
-      }
-      (keyValueOutput, runFunc)
-
-    case Some((SQLConf.Deprecated.UNSAFE_ENABLED, Some(value))) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        logWarning(
-          s"Property ${SQLConf.Deprecated.UNSAFE_ENABLED} is deprecated and " +
-            s"will be ignored. Unsafe mode will continue to be used.")
-        Seq(Row(SQLConf.Deprecated.UNSAFE_ENABLED, "true"))
-      }
-      (keyValueOutput, runFunc)
-
-    case Some((SQLConf.Deprecated.SORTMERGE_JOIN, Some(value))) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        logWarning(
-          s"Property ${SQLConf.Deprecated.SORTMERGE_JOIN} is deprecated and " +
-            s"will be ignored. Sort merge join will continue to be used.")
-        Seq(Row(SQLConf.Deprecated.SORTMERGE_JOIN, "true"))
-      }
-      (keyValueOutput, runFunc)
-
-    case Some((SQLConf.Deprecated.PARQUET_UNSAFE_ROW_RECORD_READER_ENABLED, Some(value))) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        logWarning(
-          s"Property ${SQLConf.Deprecated.PARQUET_UNSAFE_ROW_RECORD_READER_ENABLED} is " +
-            s"deprecated and will be ignored. Vectorized parquet reader will be used instead.")
-        Seq(Row(SQLConf.PARQUET_VECTORIZED_READER_ENABLED, "true"))
-      }
-      (keyValueOutput, runFunc)
-
-    // Configures a single property.
-    case Some((key, Some(value))) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        sqlContext.setConf(key, value)
-        Seq(Row(key, value))
-      }
-      (keyValueOutput, runFunc)
-
-    // (In Hive, "SET" returns all changed properties while "SET -v" returns all properties.)
-    // Queries all key-value pairs that are set in the SQLConf of the sqlContext.
-    case None =>
-      val runFunc = (sqlContext: SQLContext) => {
-        sqlContext.getAllConfs.toSeq.sortBy(_._1).map { case (k, v) => Row(k, v) } ++
-          getEnvList(withDoc = false)
-      }
-      (keyValueOutput, runFunc)
-
-    // Queries all properties along with their default values and docs that are defined in the
-    // SQLConf of the sqlContext.
-    case Some(("-v", None)) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        sqlContext.conf.getAllDefinedConfs.sortBy(_._1).map { case (key, defaultValue, doc) =>
-          Row(key, defaultValue, doc)
-        } ++ getEnvList(withDoc = true)
-      }
-      val schema = StructType(
-        StructField("key", StringType, false) ::
-          StructField("default", StringType, false) ::
-          StructField("meaning", StringType, false) :: Nil)
-      (schema.toAttributes, runFunc)
-
-    // Queries the deprecated "mapred.reduce.tasks" property.
-    case Some((SQLConf.Deprecated.MAPRED_REDUCE_TASKS, None)) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        logWarning(
-          s"Property ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS} is deprecated, " +
-            s"showing ${SQLConf.SHUFFLE_PARTITIONS.key} instead.")
-        Seq(Row(SQLConf.SHUFFLE_PARTITIONS.key, sqlContext.conf.numShufflePartitions.toString))
-      }
-      (keyValueOutput, runFunc)
-
-    // Queries a single property.
-    case Some((key, None)) =>
-      val runFunc = (sqlContext: SQLContext) => {
-        val value =
-          try sqlContext.getConf(key) catch {
-            case _: NoSuchElementException => "<undefined>"
-          }
-        Seq(Row(key, value))
-      }
-      (keyValueOutput, runFunc)
-  }
-
-  override val output: Seq[Attribute] = _output
-
-  override def run(sqlContext: SQLContext): Seq[Row] = runFunc(sqlContext)
-
-  /**
-   * get the system environment properties as a sequence
-   * @param withDoc whether the result has a doc column or not
-   * @return the sequence of the rows containing the key/value pair of system properties
-   */
-  private def getEnvList(withDoc: Boolean) = {
-    sys.env.toSeq.sortBy(_._1).map {
-      case (k, v) => if (withDoc) Row(s"env:$k", v, "") else Row(s"env:$k", v)
-    } ++
-    sys.props.toSeq.sortBy(_._1).map {
-      case (k, v) => if (withDoc) Row(s"system:$k", v, "") else Row(s"system:$k", v)
-    }
-  }
-}
-
 /**
  * An explain command for users to see how a command will be executed.
  *