remove TuningSummary

Author: YY-OnCall

examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaCrossValidationExample.scala

@@ -112,7 +112,7 @@ object ModelSelectionViaCrossValidationExample {
       .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) =>
         println(s"($id, $text) --> prob=$prob, prediction=$prediction")
       }
-    cvModel.summary.trainingMetrics.show()
+    cvModel.tuningSummary.show()
     // $example off$
 
     spark.stop()

examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaTrainValidationSplitExample.scala

@@ -74,7 +74,7 @@ object ModelSelectionViaTrainValidationSplitExample {
     model.transform(test)
       .select("features", "label", "prediction")
       .show()
-    model.summary.trainingMetrics.show()
+    model.tuningSummary.show()
     // $example off$
 
     spark.stop()

mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala

@@ -25,7 +25,6 @@ import com.github.fommil.netlib.F2jBLAS
 import org.apache.hadoop.fs.Path
 import org.json4s.DefaultFormats
 
-import org.apache.spark.SparkException
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml._
@@ -134,10 +133,7 @@ class CrossValidator @Since("1.2.0") (@Since("1.4.0") override val uid: String)
     logInfo(s"Best cross-validation metric: $bestMetric.")
     val bestModel = est.fit(dataset, epm(bestIndex)).asInstanceOf[Model[_]]
     instr.logSuccess(bestModel)
-    val model = new CrossValidatorModel(uid, bestModel, metrics).setParent(this)
-    val summary = new TuningSummary(epm, metrics, bestIndex)
-    model.setSummary(Some(summary))
-    copyValues(model)
+    copyValues(new CrossValidatorModel(uid, bestModel, metrics).setParent(this))
   }
 
   @Since("1.4.0")
@@ -233,28 +229,12 @@ class CrossValidatorModel private[ml] (
     bestModel.transformSchema(schema)
   }
 
-  private var trainingSummary: Option[TuningSummary] = None
-
-  private[tuning] def setSummary(summary: Option[TuningSummary]): this.type = {
-    this.trainingSummary = summary
-    this
-  }
-
-  /**
-   * Return true if there exists summary of model.
-   */
-  @Since("2.3.0")
-  def hasSummary: Boolean = trainingSummary.nonEmpty
-
   /**
-   * Gets summary of model on training set. An exception is
-   * thrown if `trainingSummary == None`.
+   * Summary of grid search tuning in the format of DataFrame. Each row contains one candidate
+   * paramMap and the corresponding metric of trained model.
    */
   @Since("2.3.0")
-  def summary: TuningSummary = trainingSummary.getOrElse {
-    throw new SparkException(
-      s"No training summary available for the ${this.getClass.getSimpleName}")
-  }
+  lazy val tuningSummary: DataFrame = this.getTuningSummaryDF(avgMetrics)
 
   @Since("1.4.0")
   override def copy(extra: ParamMap): CrossValidatorModel = {

mllib/src/main/scala/org/apache/spark/ml/tuning/TrainValidationSplit.scala

@@ -25,7 +25,6 @@ import scala.language.existentials
 import org.apache.hadoop.fs.Path
 import org.json4s.DefaultFormats
 
-import org.apache.spark.SparkException
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.{Estimator, Model}
@@ -129,10 +128,7 @@ class TrainValidationSplit @Since("1.5.0") (@Since("1.5.0") override val uid: St
     logInfo(s"Best train validation split metric: $bestMetric.")
     val bestModel = est.fit(dataset, epm(bestIndex)).asInstanceOf[Model[_]]
     instr.logSuccess(bestModel)
-    val model = copyValues(new TrainValidationSplitModel(uid, bestModel, metrics).setParent(this))
-    val summary = new TuningSummary(epm, metrics, bestIndex)
-    model.setSummary(Some(summary))
-    model
+    copyValues(new TrainValidationSplitModel(uid, bestModel, metrics).setParent(this))
   }
 
   @Since("1.5.0")
@@ -224,6 +220,13 @@ class TrainValidationSplitModel private[ml] (
     bestModel.transformSchema(schema)
   }
 
+  /**
+   * Summary of grid search tuning in the format of DataFrame. Each row contains one candidate
+   * paramMap and the corresponding metric of trained model.
+   */
+  @Since("2.3.0")
+  lazy val tuningSummary: DataFrame = this.getTuningSummaryDF(validationMetrics)
+
   @Since("1.5.0")
   override def copy(extra: ParamMap): TrainValidationSplitModel = {
     val copied = new TrainValidationSplitModel (
@@ -235,29 +238,6 @@ class TrainValidationSplitModel private[ml] (
 
   @Since("2.0.0")
   override def write: MLWriter = new TrainValidationSplitModel.TrainValidationSplitModelWriter(this)
-
-  private var trainingSummary: Option[TuningSummary] = None
-
-  private[tuning] def setSummary(summary: Option[TuningSummary]): this.type = {
-    this.trainingSummary = summary
-    this
-  }
-
-  /**
-   * Return true if there exists summary of model.
-   */
-  @Since("2.3.0")
-  def hasSummary: Boolean = trainingSummary.nonEmpty
-
-  /**
-   * Gets summary of model on training set. An exception is
-   * thrown if `trainingSummary == None`.
-   */
-  @Since("2.3.0")
-  def summary: TuningSummary = trainingSummary.getOrElse {
-    throw new SparkException(
-      s"No training summary available for the ${this.getClass.getSimpleName}")
-  }
 }
 
 @Since("2.0.0")

mllib/src/main/scala/org/apache/spark/ml/tuning/TuningSummary.scala

@@ -23,12 +23,13 @@ import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.SparkContext
 import org.apache.spark.ml.{Estimator, Model}
-import org.apache.spark.ml.evaluation.Evaluator
+import org.apache.spark.ml.evaluation.{BinaryClassificationEvaluator, Evaluator, MulticlassClassificationEvaluator, RegressionEvaluator}
 import org.apache.spark.ml.param.{Param, ParamMap, ParamPair, Params}
 import org.apache.spark.ml.param.shared.HasSeed
 import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.DefaultParamsReader.Metadata
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.{DataFrame, Row, SparkSession}
+import org.apache.spark.sql.types.{StringType, StructField, StructType}
 
 /**
  * Common params for [[TrainValidationSplitParams]] and [[CrossValidatorParams]].
@@ -85,6 +86,32 @@ private[ml] trait ValidatorParams extends HasSeed with Params {
     instrumentation.logNamedValue("evaluator", $(evaluator).getClass.getCanonicalName)
     instrumentation.logNamedValue("estimatorParamMapsLength", $(estimatorParamMaps).length)
   }
+
+
+  /**
+   * Summary of grid search tuning in the format of DataFrame. Each row contains one candidate
+   * paramMap and the corresponding metric of trained model.
+   */
+  protected def getTuningSummaryDF(metrics: Array[Double]): DataFrame = {
+    val params = $(estimatorParamMaps)
+    require(params.nonEmpty, "estimator param maps should not be empty")
+    require(params.length == metrics.length, "estimator param maps number should match metrics")
+    val metricName = $(evaluator) match {
+      case b: BinaryClassificationEvaluator => b.getMetricName
+      case m: MulticlassClassificationEvaluator => m.getMetricName
+      case r: RegressionEvaluator => r.getMetricName
+      case _ => "metrics"
+    }
+    val spark = SparkSession.builder().getOrCreate()
+    val sc = spark.sparkContext
+    val fields = params(0).toSeq.sortBy(_.param.name).map(_.param.name) ++ Seq(metricName)
+    val schema = new StructType(fields.map(name => StructField(name, StringType)).toArray)
+    val rows = sc.parallelize(params.zip(metrics)).map { case (param, metric) =>
+      val values = param.toSeq.sortBy(_.param.name).map(_.value.toString) ++ Seq(metric.toString)
+      Row.fromSeq(values)
+    }
+    spark.createDataFrame(rows, schema)
+  }
 }
 
 private[ml] object ValidatorParams {

mllib/src/main/scala/org/apache/spark/ml/tuning/ValidatorParams.scala

@@ -79,14 +79,11 @@ class CrossValidatorSuite
       .setEvaluator(eval)
       .setNumFolds(3)
     val cvModel = cv.fit(dataset)
-    assert(cvModel.hasSummary)
-    assert(cvModel.summary.params === lrParamMaps)
-    assert(cvModel.summary.trainingMetrics.count() === lrParamMaps.length)
-
     val expected = lrParamMaps.zip(cvModel.avgMetrics).map { case (map, metric) =>
       Row.fromSeq(map.toSeq.sortBy(_.param.name).map(_.value.toString) ++ Seq(metric.toString))
     }
-    assert(cvModel.summary.trainingMetrics.collect().toSet === expected.toSet)
+    assert(cvModel.tuningSummary.collect().toSet === expected.toSet)
+    assert(cvModel.tuningSummary.columns.last === eval.getMetricName)
   }
 
   test("cross validation with linear regression") {

mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala

@@ -72,14 +72,11 @@ class TrainValidationSplitSuite
       .setEstimatorParamMaps(lrParamMaps)
       .setEvaluator(eval)
     val tvsModel = tvs.fit(dataset)
-    assert(tvsModel.hasSummary)
-    assert(tvsModel.summary.params === lrParamMaps)
-    assert(tvsModel.summary.trainingMetrics.count() === lrParamMaps.length)
-
     val expected = lrParamMaps.zip(tvsModel.validationMetrics).map { case (map, metric) =>
       Row.fromSeq(map.toSeq.sortBy(_.param.name).map(_.value.toString) ++ Seq(metric.toString))
     }
-    assert(tvsModel.summary.trainingMetrics.collect().toSet === expected.toSet)
+    assert(tvsModel.tuningSummary.collect().toSet === expected.toSet)
+    assert(tvsModel.tuningSummary.columns.last === eval.getMetricName)
   }
 
   test("train validation with linear regression") {