[SPARK-18724][ML] Add TuningSummary for TrainValidationSplit and CrossValidator

examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaCrossValidationExample.scala

@@ -112,6 +112,7 @@ object ModelSelectionViaCrossValidationExample {
       .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) =>
         println(s"($id, $text) --> prob=$prob, prediction=$prediction")
       }
+    cvModel.tuningSummary.show()
     // $example off$
 
     spark.stop()

examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaTrainValidationSplitExample.scala

@@ -74,6 +74,7 @@ object ModelSelectionViaTrainValidationSplitExample {
     model.transform(test)
       .select("features", "label", "prediction")
       .show()
+    model.tuningSummary.show()
     // $example off$
 
     spark.stop()

mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala

@@ -229,6 +229,13 @@ class CrossValidatorModel private[ml] (
     bestModel.transformSchema(schema)
   }
 
+  /**
+   * Summary of grid search tuning in the format of DataFrame. Each row contains one candidate
+   * paramMap and the corresponding metric of trained model.
+   */
+  @Since("2.3.0")
+  lazy val tuningSummary: DataFrame = this.getTuningSummaryDF(avgMetrics)
+
   @Since("1.4.0")
   override def copy(extra: ParamMap): CrossValidatorModel = {
     val copied = new CrossValidatorModel(

mllib/src/main/scala/org/apache/spark/ml/tuning/TrainValidationSplit.scala

@@ -220,6 +220,13 @@ class TrainValidationSplitModel private[ml] (
     bestModel.transformSchema(schema)
   }
 
+  /**
+   * Summary of grid search tuning in the format of DataFrame. Each row contains one candidate
+   * paramMap and the corresponding metric of trained model.
+   */
+  @Since("2.3.0")
+  lazy val tuningSummary: DataFrame = this.getTuningSummaryDF(validationMetrics)
+
   @Since("1.5.0")
   override def copy(extra: ParamMap): TrainValidationSplitModel = {
     val copied = new TrainValidationSplitModel (

mllib/src/main/scala/org/apache/spark/ml/tuning/ValidatorParams.scala

@@ -23,12 +23,13 @@ import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.SparkContext
 import org.apache.spark.ml.{Estimator, Model}
-import org.apache.spark.ml.evaluation.Evaluator
+import org.apache.spark.ml.evaluation.{BinaryClassificationEvaluator, Evaluator, MulticlassClassificationEvaluator, RegressionEvaluator}
 import org.apache.spark.ml.param.{Param, ParamMap, ParamPair, Params}
 import org.apache.spark.ml.param.shared.HasSeed
 import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.DefaultParamsReader.Metadata
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.{DataFrame, Row, SparkSession}
+import org.apache.spark.sql.types.{StringType, StructField, StructType}
 
 /**
  * Common params for [[TrainValidationSplitParams]] and [[CrossValidatorParams]].
@@ -85,6 +86,32 @@ private[ml] trait ValidatorParams extends HasSeed with Params {
     instrumentation.logNamedValue("evaluator", $(evaluator).getClass.getCanonicalName)
     instrumentation.logNamedValue("estimatorParamMapsLength", $(estimatorParamMaps).length)
   }
+
+
+  /**
+   * Summary of grid search tuning in the format of DataFrame. Each row contains one candidate
+   * paramMap and the corresponding metric of trained model.
+   */
+  protected def getTuningSummaryDF(metrics: Array[Double]): DataFrame = {
+    val params = $(estimatorParamMaps)
+    require(params.nonEmpty, "estimator param maps should not be empty")
+    require(params.length == metrics.length, "estimator param maps number should match metrics")
+    val metricName = $(evaluator) match {
+      case b: BinaryClassificationEvaluator => b.getMetricName
+      case m: MulticlassClassificationEvaluator => m.getMetricName
+      case r: RegressionEvaluator => r.getMetricName
+      case _ => "metrics"
+    }
+    val spark = SparkSession.builder().getOrCreate()
+    val sc = spark.sparkContext
+    val fields = params(0).toSeq.sortBy(_.param.name).map(_.param.name) ++ Seq(metricName)
+    val schema = new StructType(fields.map(name => StructField(name, StringType)).toArray)
+    val rows = sc.parallelize(params.zip(metrics)).map { case (param, metric) =>
+      val values = param.toSeq.sortBy(_.param.name).map(_.value.toString) ++ Seq(metric.toString)
+      Row.fromSeq(values)
+    }
+    spark.createDataFrame(rows, schema)
+  }
 }
 
 private[ml] object ValidatorParams {

mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala

@@ -29,7 +29,7 @@ import org.apache.spark.ml.param.shared.HasInputCol
 import org.apache.spark.ml.regression.LinearRegression
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTestingUtils}
 import org.apache.spark.mllib.util.{LinearDataGenerator, MLlibTestSparkContext}
-import org.apache.spark.sql.Dataset
+import org.apache.spark.sql.{Dataset, Row}
 import org.apache.spark.sql.types.StructType
 
 class CrossValidatorSuite
@@ -66,6 +66,26 @@ class CrossValidatorSuite
     assert(cvModel.avgMetrics.length === lrParamMaps.length)
   }
 
+  test("cross validation with tuning summary") {
+    val lr = new LogisticRegression
+    val lrParamMaps = new ParamGridBuilder()
+      .addGrid(lr.regParam, Array(0.001, 1.0, 1000.0))
+      .addGrid(lr.maxIter, Array(0, 2))
+      .build()
+    val eval = new BinaryClassificationEvaluator
+    val cv = new CrossValidator()
+      .setEstimator(lr)
+      .setEstimatorParamMaps(lrParamMaps)
+      .setEvaluator(eval)
+      .setNumFolds(3)
+    val cvModel = cv.fit(dataset)
+    val expected = lrParamMaps.zip(cvModel.avgMetrics).map { case (map, metric) =>
+      Row.fromSeq(map.toSeq.sortBy(_.param.name).map(_.value.toString) ++ Seq(metric.toString))
+    }
+    assert(cvModel.tuningSummary.collect().toSet === expected.toSet)
+    assert(cvModel.tuningSummary.columns.last === eval.getMetricName)
+  }
+
   test("cross validation with linear regression") {
     val dataset = sc.parallelize(
       LinearDataGenerator.generateLinearInput(

mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala

@@ -23,12 +23,12 @@ import org.apache.spark.ml.classification.{LogisticRegression, LogisticRegressio
 import org.apache.spark.ml.classification.LogisticRegressionSuite.generateLogisticInput
 import org.apache.spark.ml.evaluation.{BinaryClassificationEvaluator, Evaluator, RegressionEvaluator}
 import org.apache.spark.ml.linalg.Vectors
-import org.apache.spark.ml.param.{ParamMap}
+import org.apache.spark.ml.param.ParamMap
 import org.apache.spark.ml.param.shared.HasInputCol
 import org.apache.spark.ml.regression.LinearRegression
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTestingUtils}
 import org.apache.spark.mllib.util.{LinearDataGenerator, MLlibTestSparkContext}
-import org.apache.spark.sql.Dataset
+import org.apache.spark.sql.{Dataset, Row}
 import org.apache.spark.sql.types.StructType
 
 class TrainValidationSplitSuite
@@ -59,6 +59,26 @@ class TrainValidationSplitSuite
     assert(tvsModel.validationMetrics.length === lrParamMaps.length)
   }
 
+  test("train validation split with tuning summary") {
+    val dataset = sc.parallelize(generateLogisticInput(1.0, 1.0, 100, 42), 2).toDF()
+    val lr = new LogisticRegression
+    val lrParamMaps = new ParamGridBuilder()
+      .addGrid(lr.regParam, Array(0.001, 1.0, 1000.0))
+      .addGrid(lr.maxIter, Array(0, 2))
+      .build()
+    val eval = new BinaryClassificationEvaluator
+    val tvs = new TrainValidationSplit()
+      .setEstimator(lr)
+      .setEstimatorParamMaps(lrParamMaps)
+      .setEvaluator(eval)
+    val tvsModel = tvs.fit(dataset)
+    val expected = lrParamMaps.zip(tvsModel.validationMetrics).map { case (map, metric) =>
+      Row.fromSeq(map.toSeq.sortBy(_.param.name).map(_.value.toString) ++ Seq(metric.toString))
+    }
+    assert(tvsModel.tuningSummary.collect().toSet === expected.toSet)
+    assert(tvsModel.tuningSummary.columns.last === eval.getMetricName)
+  }
+
   test("train validation with linear regression") {
     val dataset = sc.parallelize(
       LinearDataGenerator.generateLinearInput(
@@ -86,7 +106,7 @@ class TrainValidationSplitSuite
     assert(parent.getMaxIter === 10)
     assert(tvsModel.validationMetrics.length === lrParamMaps.length)
 
-      eval.setMetricName("r2")
+    eval.setMetricName("r2")
     val tvsModel2 = tvs.fit(dataset)
     val parent2 = tvsModel2.bestModel.parent.asInstanceOf[LinearRegression]
     assert(parent2.getRegParam === 0.001)