Merge branch 'master' into SPARK-35112

Author: AngersZhuuuu

.github/PULL_REQUEST_TEMPLATE

@@ -8,6 +8,8 @@ Thanks for sending a pull request!  Here are some tips for you:
   6. If possible, provide a concise example to reproduce the issue for a faster review.
   7. If you want to add a new configuration, please read the guideline first for naming configurations in
      'core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala'.
+  8. If you want to add or modify an error message, please read the guideline first:
+     https://spark.apache.org/error-message-guidelines.html
 -->
 
 ### What changes were proposed in this pull request?

.gitignore

@@ -15,7 +15,9 @@
 .ensime_cache/
 .ensime_lucene
 .generated-mima*
-.idea/
+# The star is required for further !.idea/ to work, see https://git-scm.com/docs/gitignore
+.idea/*
+!.idea/vcs.xml
 .idea_modules/
 .project
 .pydevproject

.idea/vcs.xml

@@ -17,10 +17,14 @@ export R_HOME=/home/username/R
 
 #### Build Spark
 
-Build Spark with [Maven](https://spark.apache.org/docs/latest/building-spark.html#buildmvn) and include the `-Psparkr` profile to build the R package. For example to use the default Hadoop versions you can run
+Build Spark with [Maven](https://spark.apache.org/docs/latest/building-spark.html#buildmvn) or [SBT](https://spark.apache.org/docs/latest/building-spark.html#building-with-sbt), and include the `-Psparkr` profile to build the R package. For example to use the default Hadoop versions you can run
 
 ```bash
+# Maven
 ./build/mvn -DskipTests -Psparkr package
+
+# SBT
+./build/sbt -Psparkr package
 ```
 
 #### Running sparkR

R/README.md

@@ -20,6 +20,7 @@ Depends:
 Suggests:
     knitr,
     rmarkdown,
+    markdown,
     testthat,
     e1071,
     survival,

R/pkg/DESCRIPTION

@@ -38,14 +38,14 @@ test_that("spark.svmLinear", {
   expect_true(class(summary$coefficients[, 1]) == "numeric")
 
   coefs <- summary$coefficients[, "Estimate"]
-  expected_coefs <- c(-0.06004978, -0.1563083, -0.460648, 0.2276626, 1.055085)
+  expected_coefs <- c(-6.8823988, -0.6154984, -1.5135447, 1.9694126, 3.3736856)
   expect_true(all(abs(coefs - expected_coefs) < 0.1))
 
   # Test prediction with string label
   prediction <- predict(model, training)
   expect_equal(typeof(take(select(prediction, "prediction"), 1)$prediction), "character")
-  expected <- c("versicolor", "versicolor", "versicolor", "virginica",  "virginica",
-                "virginica",  "virginica",  "virginica",  "virginica",  "virginica")
+  expected <- c("versicolor", "versicolor", "versicolor", "versicolor",  "versicolor",
+                "versicolor",  "versicolor",  "versicolor",  "versicolor",  "versicolor")
   expect_equal(sort(as.list(take(select(prediction, "prediction"), 10))[[1]]), expected)
 
   # Test model save and load

R/pkg/tests/fulltests/test_mllib_classification.R

@@ -31,7 +31,7 @@ _COMPILE_JVM_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g"
 ## Arg2 - Tarball Name
 ## Arg3 - Checkable Binary
 install_app() {
-  local remote_tarball="$1/$2"
+  local remote_tarball="$1"
   local local_tarball="${_DIR}/$2"
   local binary="${_DIR}/$3"
 
@@ -71,19 +71,20 @@ install_mvn() {
     local MVN_DETECTED_VERSION="$(mvn --version | head -n1 | awk '{print $3}')"
   fi
   if [ $(version $MVN_DETECTED_VERSION) -lt $(version $MVN_VERSION) ]; then
-    local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua?action=download&filename='}
-        
+    local FILE_PATH="maven/maven-3/${MVN_VERSION}/binaries/apache-maven-${MVN_VERSION}-bin.tar.gz"
+    local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua'}
+    local MIRROR_URL="${APACHE_MIRROR}/${FILE_PATH}?action=download"
+
     if [ $(command -v curl) ]; then
-      local TEST_MIRROR_URL="${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries/apache-maven-${MVN_VERSION}-bin.tar.gz"
-      if ! curl -L --output /dev/null --silent --head --fail "$TEST_MIRROR_URL" ; then
+      if ! curl -L --output /dev/null --silent --head --fail "${MIRROR_URL}" ; then
         # Fall back to archive.apache.org for older Maven
         echo "Falling back to archive.apache.org to download Maven"
-        APACHE_MIRROR="https://archive.apache.org/dist"
+        MIRROR_URL="https://archive.apache.org/dist/${FILE_PATH}"
       fi
     fi
 
     install_app \
-      "${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries" \
+      "${MIRROR_URL}" \
       "apache-maven-${MVN_VERSION}-bin.tar.gz" \
       "apache-maven-${MVN_VERSION}/bin/mvn"
 
@@ -102,7 +103,7 @@ install_scala() {
   local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.lightbend.com}
 
   install_app \
-    "${TYPESAFE_MIRROR}/scala/${scala_version}" \
+    "${TYPESAFE_MIRROR}/scala/${scala_version}/scala-${scala_version}.tgz" \
     "scala-${scala_version}.tgz" \
     "scala-${scala_version}/bin/scala"
 

build/mvn

@@ -22,10 +22,12 @@ log4j.appender.console.target=System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
 log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
 
-# Set the default spark-shell log level to WARN. When running the spark-shell, the
-# log level for this class is used to overwrite the root logger's log level, so that
-# the user can have different defaults for the shell and regular Spark apps.
+# Set the default spark-shell/spark-sql log level to WARN. When running the
+# spark-shell/spark-sql, the log level for these classes is used to overwrite
+# the root logger's log level, so that the user can have different defaults
+# for the shell and regular Spark apps.
 log4j.logger.org.apache.spark.repl.Main=WARN
+log4j.logger.org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver=WARN
 
 # Settings to quiet third party logs that are too verbose
 log4j.logger.org.sparkproject.jetty=WARN

conf/log4j.properties.template

@@ -43,6 +43,23 @@ $.extend( $.fn.dataTable.ext.type.order, {
         a = ConvertDurationString( a );
         b = ConvertDurationString( b );
         return ((a < b) ? 1 : ((a > b) ? -1 : 0));
+    },
+
+    "size-pre": function (data) {
+        var floatValue = parseFloat(data)
+        return isNaN(floatValue) ? 0 : floatValue;
+    },
+
+    "size-asc": function (a, b) {
+        a = parseFloat(a);
+        b = parseFloat(b);
+        return ((a < b) ? -1 : ((a > b) ? 1 : 0));
+    },
+
+    "size-desc": function (a, b) {
+        a = parseFloat(a);
+        b = parseFloat(b);
+        return ((a < b) ? 1 : ((a > b) ? -1 : 0));
     }
 } );
 
@@ -562,10 +579,27 @@ $(document).ready(function () {
                         }
                     ],
                     "columnDefs": [
-                        { "visible": false, "targets": 15 },
-                        { "visible": false, "targets": 16 },
-                        { "visible": false, "targets": 17 },
-                        { "visible": false, "targets": 18 }
+                        // SPARK-35087 [type:size] means String with structures like : 'size / records',
+                        // they should be sorted as numerical-order instead of lexicographical-order by default.
+                        // The targets: $id represents column id which comes from stagespage-template.html
+                        // #summary-executor-table.If the relative position of the columns in the table
+                        // #summary-executor-table has changed,please be careful to adjust the column index here
+                        // Input Size / Records
+                        {"type": "size", "targets": 9},
+                        // Output Size / Records
+                        {"type": "size", "targets": 10},
+                        // Shuffle Read Size / Records
+                        {"type": "size", "targets": 11},
+                        // Shuffle Write Size / Records
+                        {"type": "size", "targets": 12},
+                        // Peak JVM Memory OnHeap / OffHeap
+                        {"visible": false, "targets": 15},
+                        // Peak Execution Memory OnHeap / OffHeap
+                        {"visible": false, "targets": 16},
+                        // Peak Storage Memory OnHeap / OffHeap
+                        {"visible": false, "targets": 17},
+                        // Peak Pool Memory Direct / Mapped
+                        {"visible": false, "targets": 18}
                     ],
                     "deferRender": true,
                     "order": [[0, "asc"]],
@@ -746,7 +780,7 @@ $(document).ready(function () {
                     "paging": true,
                     "info": true,
                     "processing": true,
-                    "lengthMenu": [[20, 40, 60, 100, totalTasksToShow], [20, 40, 60, 100, "All"]],
+                    "lengthMenu": [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]],
                     "orderMulti": false,
                     "bAutoWidth": false,
                     "ajax": {
@@ -762,6 +796,9 @@ $(document).ready(function () {
                             data.numTasks = totalTasksToShow;
                             data.columnIndexToSort = columnIndexToSort;
                             data.columnNameToSort = columnNameToSort;
+                            if (data.length === -1) {
+                                data.length = totalTasksToShow;
+                            }
                         },
                         "dataSrc": function (jsons) {
                             var jsonStr = JSON.stringify(jsons);

core/src/main/resources/org/apache/spark/ui/static/stagepage.js

@@ -290,11 +290,11 @@ private[spark] class ExecutorAllocationManager(
    * under the current load to satisfy all running and pending tasks, rounded up.
    */
   private[spark] def maxNumExecutorsNeededPerResourceProfile(rpId: Int): Int = {
-    val pending = listener.totalPendingTasksPerResourceProfile(rpId)
+    val pendingTask = listener.pendingTasksPerResourceProfile(rpId)
     val pendingSpeculative = listener.pendingSpeculativeTasksPerResourceProfile(rpId)
     val unschedulableTaskSets = listener.pendingUnschedulableTaskSetsPerResourceProfile(rpId)
     val running = listener.totalRunningTasksPerResourceProfile(rpId)
-    val numRunningOrPendingTasks = pending + running
+    val numRunningOrPendingTasks = pendingTask + pendingSpeculative + running
     val rp = resourceProfileManager.resourceProfileFromId(rpId)
     val tasksPerExecutor = rp.maxTasksPerExecutor(conf)
     logDebug(s"max needed for rpId: $rpId numpending: $numRunningOrPendingTasks," +
@@ -916,18 +916,6 @@ private[spark] class ExecutorAllocationManager(
       hasPendingSpeculativeTasks || hasPendingRegularTasks
     }
 
-    def totalPendingTasksPerResourceProfile(rp: Int): Int = {
-      pendingTasksPerResourceProfile(rp) + pendingSpeculativeTasksPerResourceProfile(rp)
-    }
-
-    /**
-     * The number of tasks currently running across all stages.
-     * Include running-but-zombie stage attempts
-     */
-    def totalRunningTasks(): Int = {
-      stageAttemptToNumRunningTask.values.sum
-    }
-
     def totalRunningTasksPerResourceProfile(rp: Int): Int = {
       val attempts = resourceProfileIdToStageAttempt.getOrElse(rp, Set.empty).toSeq
       // attempts is a Set, change to Seq so we keep all values