From 9d09227567d9baf234a9660603343ced55efd054 Mon Sep 17 00:00:00 2001 From: Ehsun Behravesh Date: Thu, 16 Mar 2017 15:47:25 +0800 Subject: [PATCH 1/2] first commit for test - one empty line --- mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala b/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala index 6bbe7e1cb213..1956386a498e 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala @@ -15,6 +15,7 @@ * limitations under the License. */ + package org.apache.spark.ml.ann import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV} From 7c01d11c7394a9ad9c37026100feb5368f3e30eb Mon Sep 17 00:00:00 2001 From: Ehsun Behravesh Date: Thu, 16 Mar 2017 18:41:44 +0800 Subject: [PATCH 2/2] JIRA SPARK-15790: @Since annotation added for classes at package org.apache.spark.ml --- .../scala/org/apache/spark/ml/Model.scala | 3 ++- .../org/apache/spark/ml/ann/BreezeUtil.scala | 3 ++- .../scala/org/apache/spark/ml/ann/Layer.scala | 21 ++++++++++++++- .../apache/spark/ml/ann/LossFunction.scala | 7 ++++- .../spark/ml/attribute/AttributeGroup.scala | 5 ++-- .../spark/ml/attribute/AttributeKeys.scala | 3 +++ .../spark/ml/attribute/AttributeType.scala | 4 ++- .../spark/ml/attribute/attributes.scala | 12 ++++++++- .../spark/ml/classification/Classifier.scala | 5 +++- .../ProbabilisticClassifier.scala | 7 ++++- .../ml/clustering/ClusteringSummary.scala | 3 ++- .../apache/spark/ml/feature/Instance.scala | 2 ++ .../org/apache/spark/ml/feature/LSH.scala | 4 +++ .../spark/ml/feature/RFormulaParser.scala | 5 +++- .../spark/ml/linalg/JsonVectorConverter.scala | 2 ++ .../apache/spark/ml/linalg/MatrixUDT.scala | 2 ++ .../apache/spark/ml/linalg/VectorUDT.scala | 2 ++ .../IterativelyReweightedLeastSquares.scala | 3 +++ .../spark/ml/optim/NormalEquationSolver.scala | 7 ++++- .../spark/ml/optim/WeightedLeastSquares.scala | 4 +++ .../ml/param/shared/SharedParamsCodeGen.scala | 4 +++ .../spark/ml/param/shared/sharedParams.scala | 26 +++++++++++++++++++ .../org/apache/spark/ml/python/MLSerDe.scala | 4 +-- .../ml/r/AFTSurvivalRegressionWrapper.scala | 4 +++ .../org/apache/spark/ml/r/ALSWrapper.scala | 4 +++ .../spark/ml/r/BisectingKMeansWrapper.scala | 3 +++ .../spark/ml/r/GBTClassificationWrapper.scala | 4 +++ .../spark/ml/r/GBTRegressionWrapper.scala | 4 +++ .../GeneralizedLinearRegressionWrapper.scala | 4 +++ .../ml/r/IsotonicRegressionWrapper.scala | 4 +++ .../org/apache/spark/ml/r/KMeansWrapper.scala | 4 +++ .../org/apache/spark/ml/r/KSTestWrapper.scala | 4 +++ .../org/apache/spark/ml/r/LDAWrapper.scala | 5 +++- .../apache/spark/ml/r/LinearSVCWrapper.scala | 4 +++ .../ml/r/LogisticRegressionWrapper.scala | 4 +++ ...ultilayerPerceptronClassifierWrapper.scala | 4 +++ .../apache/spark/ml/r/NaiveBayesWrapper.scala | 4 +++ .../org/apache/spark/ml/r/RWrapperUtils.scala | 3 +++ .../org/apache/spark/ml/r/RWrappers.scala | 3 +++ .../r/RandomForestClassificationWrapper.scala | 4 +++ .../ml/r/RandomForestRegressionWrapper.scala | 4 +++ .../recommendation/TopByKeyAggregator.scala | 3 ++- .../spark/ml/regression/Regressor.scala | 5 +++- .../ml/source/libsvm/LibSVMDataSource.scala | 3 +++ .../ml/source/libsvm/LibSVMOptions.scala | 2 ++ .../ml/source/libsvm/LibSVMRelation.scala | 5 ++-- .../scala/org/apache/spark/ml/tree/Node.scala | 7 +++++ .../spark/ml/tree/impl/BaggedPoint.scala | 3 ++- .../ml/tree/impl/DTStatsAggregator.scala | 3 ++- .../ml/tree/impl/DecisionTreeMetadata.scala | 4 ++- .../ml/tree/impl/GradientBoostedTrees.scala | 3 ++- .../spark/ml/tree/impl/NodeIdCache.scala | 7 ++--- .../spark/ml/tree/impl/RandomForest.scala | 5 ++-- .../spark/ml/tree/impl/TimeTracker.scala | 2 ++ .../apache/spark/ml/tree/impl/TreePoint.scala | 4 +++ .../org/apache/spark/ml/tree/treeModels.scala | 8 ++++-- .../org/apache/spark/ml/tree/treeParams.scala | 19 +++++++++++++- .../spark/ml/tuning/ValidatorParams.scala | 3 ++- .../apache/spark/ml/util/Identifiable.scala | 4 ++- .../spark/ml/util/Instrumentation.scala | 4 +++ .../apache/spark/ml/util/MetadataUtils.scala | 3 +++ .../apache/spark/ml/util/SchemaUtils.scala | 3 +++ .../apache/spark/ml/util/stopwatches.scala | 6 +++++ .../python/GaussianMixtureModelWrapper.scala | 3 +++ 64 files changed, 283 insertions(+), 34 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/Model.scala b/mllib/src/main/scala/org/apache/spark/ml/Model.scala index c581fed17727..d9796e35c0c6 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/Model.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/Model.scala @@ -17,7 +17,7 @@ package org.apache.spark.ml -import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.annotation.{DeveloperApi, Since} import org.apache.spark.ml.param.ParamMap /** @@ -27,6 +27,7 @@ import org.apache.spark.ml.param.ParamMap * @tparam M model type */ @DeveloperApi +@Since("1.2.0") abstract class Model[M <: Model[M]] extends Transformer { /** * The parent estimator that produced this model. diff --git a/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala b/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala index 1956386a498e..ef65d8e817c1 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/ann/BreezeUtil.scala @@ -15,15 +15,16 @@ * limitations under the License. */ - package org.apache.spark.ml.ann +import org.apache.spark.annotation.Since import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV} import com.github.fommil.netlib.BLAS.{getInstance => NativeBLAS} /** * In-place DGEMM and DGEMV for Breeze */ +@Since("1.5.0") private[ann] object BreezeUtil { // TODO: switch to MLlib BLAS interface diff --git a/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala b/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala index e7e0dae0b5a0..ad34dfb378ad 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/ann/Layer.scala @@ -20,7 +20,7 @@ package org.apache.spark.ml.ann import java.util.Random import breeze.linalg.{*, axpy => Baxpy, DenseMatrix => BDM, DenseVector => BDV, Vector => BV} - +import org.apache.spark.annotation.Since import org.apache.spark.ml.linalg.{Vector, Vectors} import org.apache.spark.mllib.linalg.{Vector => OldVector, Vectors => OldVectors} import org.apache.spark.mllib.linalg.VectorImplicits._ @@ -34,6 +34,7 @@ import org.apache.spark.util.random.XORShiftRandom * Implements Layer instantiation. * */ +@Since("1.5.0") private[ann] trait Layer extends Serializable { /** @@ -83,6 +84,7 @@ private[ann] trait Layer extends Serializable { * Implements functions needed for forward propagation, computing delta and gradient. * Can return weights in Vector format. */ +@Since("1.5.0") private[ann] trait LayerModel extends Serializable { val weights: BDV[Double] @@ -130,6 +132,7 @@ private[ann] trait LayerModel extends Serializable { * @param numIn number of inputs * @param numOut number of outputs */ +@Since("1.5.0") private[ann] class AffineLayer(val numIn: Int, val numOut: Int) extends Layer { override val weightSize = numIn * numOut + numOut @@ -150,6 +153,7 @@ private[ann] class AffineLayer(val numIn: Int, val numOut: Int) extends Layer { * @param weights weights * @param layer layer properties */ +@Since("1.5.0") private[ann] class AffineLayerModel private[ann] ( val weights: BDV[Double], val layer: AffineLayer) extends LayerModel { @@ -185,6 +189,7 @@ private[ann] class AffineLayerModel private[ann] ( /** * Fabric for Affine layer models */ +@Since("1.5.0") private[ann] object AffineLayerModel { /** @@ -229,6 +234,7 @@ private[ann] object AffineLayerModel { /** * Trait for functions and their derivatives for functional layers */ +@Since("1.5.0") private[ann] trait ActivationFunction extends Serializable { /** @@ -245,6 +251,7 @@ private[ann] trait ActivationFunction extends Serializable { /** * Implements in-place application of functions in the arrays */ +@Since("1.5.0") private[ann] object ApplyInPlace { // TODO: use Breeze UFunc @@ -281,6 +288,7 @@ private[ann] object ApplyInPlace { /** * Implements Sigmoid activation function */ +@Since("1.5.0") private[ann] class SigmoidFunction extends ActivationFunction { override def eval: (Double) => Double = x => 1.0 / (1 + math.exp(-x)) @@ -312,6 +320,7 @@ private[ann] class FunctionalLayer (val activationFunction: ActivationFunction) * * @param layer functional layer */ +@Since("1.5.0") private[ann] class FunctionalLayerModel private[ann] (val layer: FunctionalLayer) extends LayerModel { @@ -336,6 +345,7 @@ private[ann] class FunctionalLayerModel private[ann] (val layer: FunctionalLayer /** * Trait for the artificial neural network (ANN) topology properties */ +@Since("1.5.0") private[ann] trait Topology extends Serializable { def model(weights: Vector): TopologyModel def model(seed: Long): TopologyModel @@ -344,6 +354,7 @@ private[ann] trait Topology extends Serializable { /** * Trait for ANN topology model */ +@Since("1.5.0") private[ann] trait TopologyModel extends Serializable { val weights: Vector @@ -391,6 +402,7 @@ private[ann] trait TopologyModel extends Serializable { * * @param layers Array of layers */ +@Since("1.5.0") private[ann] class FeedForwardTopology private(val layers: Array[Layer]) extends Topology { override def model(weights: Vector): TopologyModel = FeedForwardModel(this, weights) @@ -400,6 +412,7 @@ private[ann] class FeedForwardTopology private(val layers: Array[Layer]) extends /** * Factory for some of the frequently-used topologies */ +@Since("1.5.0") private[ml] object FeedForwardTopology { /** * Creates a feed forward topology from the array of layers @@ -448,6 +461,7 @@ private[ml] object FeedForwardTopology { * @param weights network weights * @param topology network topology */ +@Since("1.5.0") private[ml] class FeedForwardModel private( val weights: Vector, val topology: FeedForwardTopology) extends TopologyModel { @@ -535,6 +549,7 @@ private[ml] class FeedForwardModel private( /** * Fabric for feed forward ANN models */ +@Since("1.5.0") private[ann] object FeedForwardModel { /** @@ -579,6 +594,7 @@ private[ann] object FeedForwardModel { * @param topology topology * @param dataStacker data stacker */ +@Since("1.5.0") private[ann] class ANNGradient(topology: Topology, dataStacker: DataStacker) extends Gradient { override def compute( data: OldVector, @@ -601,6 +617,7 @@ private[ann] class ANNGradient(topology: Topology, dataStacker: DataStacker) ext * @param inputSize size of the input vectors * @param outputSize size of the output vectors */ +@Since("1.5.0") private[ann] class DataStacker(stackSize: Int, inputSize: Int, outputSize: Int) extends Serializable { @@ -655,6 +672,7 @@ private[ann] class DataStacker(stackSize: Int, inputSize: Int, outputSize: Int) /** * Simple updater */ +@Since("1.5.0") private[ann] class ANNUpdater extends Updater { override def compute( @@ -677,6 +695,7 @@ private[ann] class ANNUpdater extends Updater { * @param inputSize input size * @param outputSize output size */ +@Since("1.5.0") private[ml] class FeedForwardTrainer( topology: Topology, val inputSize: Int, diff --git a/mllib/src/main/scala/org/apache/spark/ml/ann/LossFunction.scala b/mllib/src/main/scala/org/apache/spark/ml/ann/LossFunction.scala index 32d78e9b226e..774e244ba63e 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/ann/LossFunction.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/ann/LossFunction.scala @@ -18,13 +18,14 @@ package org.apache.spark.ml.ann import java.util.Random - +import org.apache.spark.annotation.Since import breeze.linalg.{sum => Bsum, DenseMatrix => BDM, DenseVector => BDV} import breeze.numerics.{log => brzlog} /** * Trait for loss function */ +@Since("2.0.0") private[ann] trait LossFunction { /** * Returns the value of loss function. @@ -41,6 +42,7 @@ private[ann] trait LossFunction { def loss(output: BDM[Double], target: BDM[Double], delta: BDM[Double]): Double } +@Since("2.0.0") private[ann] class SigmoidLayerWithSquaredError extends Layer { override val weightSize = 0 override val inPlace = true @@ -52,6 +54,7 @@ private[ann] class SigmoidLayerWithSquaredError extends Layer { new SigmoidLayerModelWithSquaredError() } +@Since("2.0.0") private[ann] class SigmoidLayerModelWithSquaredError extends FunctionalLayerModel(new FunctionalLayer(new SigmoidFunction)) with LossFunction { override def loss(output: BDM[Double], target: BDM[Double], delta: BDM[Double]): Double = { @@ -62,6 +65,7 @@ private[ann] class SigmoidLayerModelWithSquaredError } } +@Since("2.0.0") private[ann] class SoftmaxLayerWithCrossEntropyLoss extends Layer { override val weightSize = 0 override val inPlace = true @@ -73,6 +77,7 @@ private[ann] class SoftmaxLayerWithCrossEntropyLoss extends Layer { new SoftmaxLayerModelWithCrossEntropyLoss() } +@Since("2.0.0") private[ann] class SoftmaxLayerModelWithCrossEntropyLoss extends LayerModel with LossFunction { // loss layer models do not have weights diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala index 21a246e454c8..7fa110d1f520 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala @@ -18,8 +18,7 @@ package org.apache.spark.ml.attribute import scala.collection.mutable.ArrayBuffer - -import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.annotation.{DeveloperApi, Since} import org.apache.spark.ml.linalg.VectorUDT import org.apache.spark.sql.types.{Metadata, MetadataBuilder, StructField} @@ -34,6 +33,7 @@ import org.apache.spark.sql.types.{Metadata, MetadataBuilder, StructField} * indices in the array. */ @DeveloperApi +@Since("1.4.0") class AttributeGroup private ( val name: String, val numAttributes: Option[Int], @@ -192,6 +192,7 @@ class AttributeGroup private ( * Factory methods to create attribute groups. */ @DeveloperApi +@Since("1.4.0") object AttributeGroup { import AttributeKeys._ diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeKeys.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeKeys.scala index f714f7becc7e..e2d7a66684d7 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeKeys.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeKeys.scala @@ -17,9 +17,12 @@ package org.apache.spark.ml.attribute +import org.apache.spark.annotation.Since + /** * Keys used to store attributes. */ +@Since("1.4.0") private[attribute] object AttributeKeys { val ML_ATTR: String = "ml_attr" val TYPE: String = "type" diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala index 078fecf08828..d678dda30615 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala @@ -17,7 +17,7 @@ package org.apache.spark.ml.attribute -import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.annotation.{DeveloperApi, Since} /** * :: DeveloperApi :: @@ -25,12 +25,14 @@ import org.apache.spark.annotation.DeveloperApi * and [[AttributeType$#Binary]]. */ @DeveloperApi +@Since("1.4.0") sealed abstract class AttributeType(val name: String) /** * :: DeveloperApi :: */ @DeveloperApi +@Since("1.4.0") object AttributeType { /** Numeric type. */ diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala index 1cd2b1ad8409..2aeeb78b58a3 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala @@ -19,7 +19,7 @@ package org.apache.spark.ml.attribute import scala.annotation.varargs -import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.annotation.{DeveloperApi, Since} import org.apache.spark.sql.types.{DoubleType, Metadata, MetadataBuilder, NumericType, StructField} /** @@ -27,6 +27,7 @@ import org.apache.spark.sql.types.{DoubleType, Metadata, MetadataBuilder, Numeri * Abstract class for ML attributes. */ @DeveloperApi +@Since("1.4.0") sealed abstract class Attribute extends Serializable { name.foreach { n => @@ -118,6 +119,7 @@ sealed abstract class Attribute extends Serializable { } /** Trait for ML attribute factories. */ +@Since("1.4.0") private[attribute] trait AttributeFactory { /** @@ -154,6 +156,7 @@ private[attribute] trait AttributeFactory { * :: DeveloperApi :: */ @DeveloperApi +@Since("1.4.0") object Attribute extends AttributeFactory { private[attribute] override def fromMetadata(metadata: Metadata): Attribute = { @@ -192,6 +195,7 @@ object Attribute extends AttributeFactory { * @param sparsity optional sparsity (ratio of zeros) */ @DeveloperApi +@Since("1.4.0") class NumericAttribute private[ml] ( override val name: Option[String] = None, override val index: Option[Int] = None, @@ -303,6 +307,7 @@ class NumericAttribute private[ml] ( * Factory methods for numeric attributes. */ @DeveloperApi +@Since("1.4.0") object NumericAttribute extends AttributeFactory { /** The default numeric attribute. */ @@ -331,6 +336,7 @@ object NumericAttribute extends AttributeFactory { * @param values optional values. At most one of `numValues` and `values` can be defined. */ @DeveloperApi +@Since("1.4.0") class NominalAttribute private[ml] ( override val name: Option[String] = None, override val index: Option[Int] = None, @@ -468,6 +474,7 @@ class NominalAttribute private[ml] ( * Factory methods for nominal attributes. */ @DeveloperApi +@Since("1.4.0") object NominalAttribute extends AttributeFactory { /** The default nominal attribute. */ @@ -494,6 +501,7 @@ object NominalAttribute extends AttributeFactory { * @param values optional values. If set, its size must be 2. */ @DeveloperApi +@Since("1.4.0") class BinaryAttribute private[ml] ( override val name: Option[String] = None, override val index: Option[Int] = None, @@ -570,6 +578,7 @@ class BinaryAttribute private[ml] ( * Factory methods for binary attributes. */ @DeveloperApi +@Since("1.4.0") object BinaryAttribute extends AttributeFactory { /** The default binary attribute. */ @@ -590,6 +599,7 @@ object BinaryAttribute extends AttributeFactory { * An unresolved attribute. */ @DeveloperApi +@Since("1.4.0") object UnresolvedAttribute extends Attribute { override def attrType: AttributeType = AttributeType.Unresolved diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala index d8608d885d6f..a274b075a779 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala @@ -18,7 +18,7 @@ package org.apache.spark.ml.classification import org.apache.spark.SparkException -import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.annotation.{DeveloperApi, Since} import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams} import org.apache.spark.ml.feature.LabeledPoint import org.apache.spark.ml.linalg.{Vector, VectorUDT} @@ -32,6 +32,7 @@ import org.apache.spark.sql.types.{DataType, StructType} /** * (private[spark]) Params for classification. */ +@Since("1.3.0") private[spark] trait ClassifierParams extends PredictorParams with HasRawPredictionCol { @@ -55,6 +56,7 @@ private[spark] trait ClassifierParams * @tparam M Concrete Model type */ @DeveloperApi +@Since("1.3.0") abstract class Classifier[ FeaturesType, E <: Classifier[FeaturesType, E, M], @@ -138,6 +140,7 @@ abstract class Classifier[ * @tparam M Concrete Model type */ @DeveloperApi +@Since("1.3.0") abstract class ClassificationModel[FeaturesType, M <: ClassificationModel[FeaturesType, M]] extends PredictionModel[FeaturesType, M] with ClassifierParams { diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala index ef0813480991..5d18bd863c6a 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala @@ -17,7 +17,7 @@ package org.apache.spark.ml.classification -import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.annotation.{DeveloperApi, Since} import org.apache.spark.ml.linalg.{DenseVector, Vector, VectorUDT} import org.apache.spark.ml.param.shared._ import org.apache.spark.ml.util.SchemaUtils @@ -28,6 +28,8 @@ import org.apache.spark.sql.types.{DataType, StructType} /** * (private[classification]) Params for probabilistic classification. */ + +@Since("1.3.0") private[classification] trait ProbabilisticClassifierParams extends ClassifierParams with HasProbabilityCol with HasThresholds { override protected def validateAndTransformSchema( @@ -50,6 +52,7 @@ private[classification] trait ProbabilisticClassifierParams * @tparam M Concrete Model type */ @DeveloperApi +@Since("1.3.0") abstract class ProbabilisticClassifier[ FeaturesType, E <: ProbabilisticClassifier[FeaturesType, E, M], @@ -74,6 +77,7 @@ abstract class ProbabilisticClassifier[ * @tparam M Concrete Model type */ @DeveloperApi +@Since("1.3.0") abstract class ProbabilisticClassificationModel[ FeaturesType, M <: ProbabilisticClassificationModel[FeaturesType, M]] @@ -224,6 +228,7 @@ abstract class ProbabilisticClassificationModel[ } } +@Since("1.3.0") private[ml] object ProbabilisticClassificationModel { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/ClusteringSummary.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/ClusteringSummary.scala index 44e832b058b6..f41ef0019fa8 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/clustering/ClusteringSummary.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/ClusteringSummary.scala @@ -17,7 +17,7 @@ package org.apache.spark.ml.clustering -import org.apache.spark.annotation.Experimental +import org.apache.spark.annotation.{Experimental, Since} import org.apache.spark.sql.{DataFrame, Row} /** @@ -30,6 +30,7 @@ import org.apache.spark.sql.{DataFrame, Row} * @param k Number of clusters. */ @Experimental +@Since("2.1.0") class ClusteringSummary private[clustering] ( @transient val predictions: DataFrame, val predictionCol: String, diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala index cce3ca45ccd8..ee5030aee437 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala @@ -17,6 +17,7 @@ package org.apache.spark.ml.feature +import org.apache.spark.annotation.Since import org.apache.spark.ml.linalg.Vector /** @@ -26,4 +27,5 @@ import org.apache.spark.ml.linalg.Vector * @param weight The weight of this instance. * @param features The vector of features for this data point. */ +@Since("1.6.0") private[ml] case class Instance(label: Double, weight: Double, features: Vector) diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala index 1c9f47a0b201..b16426cc9fb2 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala @@ -19,6 +19,7 @@ package org.apache.spark.ml.feature import scala.util.Random +import org.apache.spark.annotation.Since import org.apache.spark.ml.{Estimator, Model} import org.apache.spark.ml.linalg.{Vector, VectorUDT} import org.apache.spark.ml.param.{IntParam, ParamValidators} @@ -31,6 +32,7 @@ import org.apache.spark.sql.types._ /** * Params for [[LSH]]. */ +@Since("2.1.0") private[ml] trait LSHParams extends HasInputCol with HasOutputCol { /** * Param for the number of hash tables used in LSH OR-amplification. @@ -61,6 +63,7 @@ private[ml] trait LSHParams extends HasInputCol with HasOutputCol { /** * Model produced by [[LSH]]. */ +@Since("2.1.0") private[ml] abstract class LSHModel[T <: LSHModel[T]] extends Model[T] with LSHParams with MLWritable { self: T => @@ -295,6 +298,7 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]] * (2) Wang, Jingdong et al. "Hashing for similarity search: A survey." arXiv preprint * arXiv:1408.2927 (2014). */ +@Since("2.1.0") private[ml] abstract class LSH[T <: LSHModel[T]] extends Estimator[T] with LSHParams with DefaultParamsWritable { self: Estimator[T] => diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormulaParser.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormulaParser.scala index 2dd565a78271..4f888f7af55c 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormulaParser.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormulaParser.scala @@ -19,13 +19,14 @@ package org.apache.spark.ml.feature import scala.collection.mutable import scala.util.parsing.combinator.RegexParsers - +import org.apache.spark.annotation.Since import org.apache.spark.ml.linalg.VectorUDT import org.apache.spark.sql.types._ /** * Represents a parsed R formula. */ +@Since("1.5.0") private[ml] case class ParsedRFormula(label: ColumnRef, terms: Seq[Term]) { /** * Resolves formula terms into column names. A schema is necessary for inferring the meaning @@ -125,6 +126,7 @@ private[ml] case class ParsedRFormula(label: ColumnRef, terms: Seq[Term]) { * of column names; non-interaction terms as length 1 Seqs. * @param hasIntercept whether the formula specifies fitting with an intercept. */ +@Since("1.5.0") private[ml] case class ResolvedRFormula( label: String, terms: Seq[Seq[String]], hasIntercept: Boolean) { @@ -167,6 +169,7 @@ private[ml] case class Deletion(term: Term) extends Term /** * Limited implementation of R formula parsing. Currently supports: '~', '+', '-', '.', ':'. */ +@Since("1.5.0") private[ml] object RFormulaParser extends RegexParsers { private val intercept: Parser[Intercept] = "([01])".r ^^ { case a => Intercept(a == "1") } diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonVectorConverter.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonVectorConverter.scala index 781e69f8d63d..9fd17e3e0c25 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonVectorConverter.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/JsonVectorConverter.scala @@ -17,10 +17,12 @@ package org.apache.spark.ml.linalg +import org.apache.spark.annotation.Since import org.json4s.DefaultFormats import org.json4s.JsonDSL._ import org.json4s.jackson.JsonMethods.{compact, parse => parseJson, render} +@Since("2.0.0") private[ml] object JsonVectorConverter { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala index f4a8556c71f6..844047f3cce2 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala @@ -17,6 +17,7 @@ package org.apache.spark.ml.linalg +import org.apache.spark.annotation.Since import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeArrayData} import org.apache.spark.sql.types._ @@ -25,6 +26,7 @@ import org.apache.spark.sql.types._ * User-defined type for [[Matrix]] in [[mllib-local]] which allows easy interaction with SQL * via [[org.apache.spark.sql.Dataset]]. */ +@Since("2.0.0") private[spark] class MatrixUDT extends UserDefinedType[Matrix] { override def sqlType: StructType = { diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala index 917861309c57..22e519a1ef18 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala @@ -17,6 +17,7 @@ package org.apache.spark.ml.linalg +import org.apache.spark.annotation.Since import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeArrayData} import org.apache.spark.sql.types._ @@ -25,6 +26,7 @@ import org.apache.spark.sql.types._ * User-defined type for [[Vector]] in [[mllib-local]] which allows easy interaction with SQL * via [[org.apache.spark.sql.Dataset]]. */ +@Since("2.0.0") private[spark] class VectorUDT extends UserDefinedType[Vector] { override def sqlType: StructType = { diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala index 9c495512422b..49710f8bc7bd 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/optim/IterativelyReweightedLeastSquares.scala @@ -17,6 +17,7 @@ package org.apache.spark.ml.optim +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.ml.feature.Instance import org.apache.spark.ml.linalg._ @@ -29,6 +30,7 @@ import org.apache.spark.rdd.RDD * @param diagInvAtWA diagonal of matrix (A^T * W * A)^-1 in the last iteration * @param numIterations number of iterations */ +@Since("2.0.0") private[ml] class IterativelyReweightedLeastSquaresModel( val coefficients: DenseVector, val intercept: Double, @@ -55,6 +57,7 @@ private[ml] class IterativelyReweightedLeastSquaresModel( * and Resistant Alternatives, Journal of the Royal Statistical Society. * Series B, 1984. */ +@Since("2.0.0") private[ml] class IterativelyReweightedLeastSquares( val initialModel: WeightedLeastSquaresModel, val reweightFunc: (Instance, WeightedLeastSquaresModel) => (Double, Double), diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/NormalEquationSolver.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/NormalEquationSolver.scala index dc3bcc662733..4cf3e2defd73 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/optim/NormalEquationSolver.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/optim/NormalEquationSolver.scala @@ -17,7 +17,7 @@ package org.apache.spark.ml.optim import scala.collection.mutable - +import org.apache.spark.annotation.Since import breeze.linalg.{DenseVector => BDV} import breeze.optimize.{CachedDiffFunction, DiffFunction, LBFGS => BreezeLBFGS, OWLQN => BreezeOWLQN} @@ -34,6 +34,7 @@ import org.apache.spark.mllib.linalg.CholeskyDecomposition * @param objectiveHistory Option containing the objective history when an optimization program is * used to solve the normal equations. None when an analytic solver is used. */ +@Since("2.1.0") private[optim] class NormalEquationSolution( val coefficients: Array[Double], val aaInv: Option[Array[Double]], @@ -42,6 +43,7 @@ private[optim] class NormalEquationSolution( /** * Interface for classes that solve the normal equations locally. */ +@Since("2.1.0") private[optim] sealed trait NormalEquationSolver { /** Solve the normal equations from summary statistics. */ @@ -56,6 +58,7 @@ private[optim] sealed trait NormalEquationSolver { /** * A class that solves the normal equations directly, using Cholesky decomposition. */ +@Since("2.1.0") private[optim] class CholeskySolver extends NormalEquationSolver { override def solve( @@ -75,6 +78,7 @@ private[optim] class CholeskySolver extends NormalEquationSolver { /** * A class for solving the normal equations using Quasi-Newton optimization methods. */ +@Since("2.1.0") private[optim] class QuasiNewtonSolver( fitIntercept: Boolean, maxIter: Int, @@ -157,6 +161,7 @@ private[optim] class QuasiNewtonSolver( * Exception thrown when solving a linear system Ax = b for which the matrix A is non-invertible * (singular). */ +@Since("2.1.0") private[spark] class SingularMatrixException(message: String, cause: Throwable) extends IllegalArgumentException(message, cause) { diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala index 56ab9675700a..8668e6417f78 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala @@ -17,6 +17,7 @@ package org.apache.spark.ml.optim +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.ml.feature.Instance import org.apache.spark.ml.linalg._ @@ -30,6 +31,7 @@ import org.apache.spark.rdd.RDD * @param diagInvAtWA diagonal of matrix (A^T * W * A)^-1 * @param objectiveHistory objective function (scaled loss + regularization) at each iteration. */ +@Since("1.6.0") private[ml] class WeightedLeastSquaresModel( val coefficients: DenseVector, val intercept: Double, @@ -73,6 +75,7 @@ private[ml] class WeightedLeastSquaresModel( * @param maxIter maximum number of iterations. Only for QuasiNewton solverType. * @param tol the convergence tolerance of the iterations. Only for QuasiNewton solverType. */ +@Since("1.6.0") private[ml] class WeightedLeastSquares( val fitIntercept: Boolean, val regParam: Double, @@ -318,6 +321,7 @@ private[ml] class WeightedLeastSquares( } } +@Since("1.6.0") private[ml] object WeightedLeastSquares { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala index c94b8b4e9dfd..d74aadc51b2f 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.param.shared +import org.apache.spark.annotation.Since + import java.io.PrintWriter import scala.reflect.ClassTag @@ -27,6 +29,8 @@ import scala.reflect.ClassTag * build/sbt "mllib/runMain org.apache.spark.ml.param.shared.SharedParamsCodeGen" * }}} */ + +@Since("1.4.0") private[shared] object SharedParamsCodeGen { def main(args: Array[String]): Unit = { diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala index e3e03dfd43dd..21536c5be45e 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.param.shared +import org.apache.spark.annotation.Since + import org.apache.spark.ml.param._ // DO NOT MODIFY THIS FILE! It was generated by SharedParamsCodeGen. @@ -26,6 +28,7 @@ import org.apache.spark.ml.param._ /** * Trait for shared param regParam. */ +@Since("1.4.0") private[ml] trait HasRegParam extends Params { /** @@ -41,6 +44,7 @@ private[ml] trait HasRegParam extends Params { /** * Trait for shared param maxIter. */ +@Since("1.4.0") private[ml] trait HasMaxIter extends Params { /** @@ -56,6 +60,7 @@ private[ml] trait HasMaxIter extends Params { /** * Trait for shared param featuresCol (default: "features"). */ +@Since("1.4.0") private[ml] trait HasFeaturesCol extends Params { /** @@ -73,6 +78,7 @@ private[ml] trait HasFeaturesCol extends Params { /** * Trait for shared param labelCol (default: "label"). */ +@Since("1.4.0") private[ml] trait HasLabelCol extends Params { /** @@ -90,6 +96,7 @@ private[ml] trait HasLabelCol extends Params { /** * Trait for shared param predictionCol (default: "prediction"). */ +@Since("1.4.0") private[ml] trait HasPredictionCol extends Params { /** @@ -107,6 +114,7 @@ private[ml] trait HasPredictionCol extends Params { /** * Trait for shared param rawPredictionCol (default: "rawPrediction"). */ +@Since("1.4.0") private[ml] trait HasRawPredictionCol extends Params { /** @@ -124,6 +132,7 @@ private[ml] trait HasRawPredictionCol extends Params { /** * Trait for shared param probabilityCol (default: "probability"). */ +@Since("1.4.0") private[ml] trait HasProbabilityCol extends Params { /** @@ -141,6 +150,7 @@ private[ml] trait HasProbabilityCol extends Params { /** * Trait for shared param varianceCol. */ +@Since("1.4.0") private[ml] trait HasVarianceCol extends Params { /** @@ -156,6 +166,7 @@ private[ml] trait HasVarianceCol extends Params { /** * Trait for shared param threshold (default: 0.5). */ +@Since("1.4.0") private[ml] trait HasThreshold extends Params { /** @@ -173,6 +184,7 @@ private[ml] trait HasThreshold extends Params { /** * Trait for shared param thresholds. */ +@Since("1.4.0") private[ml] trait HasThresholds extends Params { /** @@ -188,6 +200,7 @@ private[ml] trait HasThresholds extends Params { /** * Trait for shared param inputCol. */ +@Since("1.4.0") private[ml] trait HasInputCol extends Params { /** @@ -203,6 +216,7 @@ private[ml] trait HasInputCol extends Params { /** * Trait for shared param inputCols. */ +@Since("1.4.0") private[ml] trait HasInputCols extends Params { /** @@ -218,6 +232,7 @@ private[ml] trait HasInputCols extends Params { /** * Trait for shared param outputCol (default: uid + "__output"). */ +@Since("1.4.0") private[ml] trait HasOutputCol extends Params { /** @@ -235,6 +250,7 @@ private[ml] trait HasOutputCol extends Params { /** * Trait for shared param checkpointInterval. */ +@Since("1.4.0") private[ml] trait HasCheckpointInterval extends Params { /** @@ -250,6 +266,7 @@ private[ml] trait HasCheckpointInterval extends Params { /** * Trait for shared param fitIntercept (default: true). */ +@Since("1.4.0") private[ml] trait HasFitIntercept extends Params { /** @@ -267,6 +284,7 @@ private[ml] trait HasFitIntercept extends Params { /** * Trait for shared param handleInvalid. */ +@Since("1.4.0") private[ml] trait HasHandleInvalid extends Params { /** @@ -282,6 +300,7 @@ private[ml] trait HasHandleInvalid extends Params { /** * Trait for shared param standardization (default: true). */ +@Since("1.4.0") private[ml] trait HasStandardization extends Params { /** @@ -299,6 +318,7 @@ private[ml] trait HasStandardization extends Params { /** * Trait for shared param seed (default: this.getClass.getName.hashCode.toLong). */ +@Since("1.4.0") private[ml] trait HasSeed extends Params { /** @@ -316,6 +336,7 @@ private[ml] trait HasSeed extends Params { /** * Trait for shared param elasticNetParam. */ +@Since("1.4.0") private[ml] trait HasElasticNetParam extends Params { /** @@ -331,6 +352,7 @@ private[ml] trait HasElasticNetParam extends Params { /** * Trait for shared param tol. */ +@Since("1.4.0") private[ml] trait HasTol extends Params { /** @@ -346,6 +368,7 @@ private[ml] trait HasTol extends Params { /** * Trait for shared param stepSize. */ +@Since("1.4.0") private[ml] trait HasStepSize extends Params { /** @@ -361,6 +384,7 @@ private[ml] trait HasStepSize extends Params { /** * Trait for shared param weightCol. */ +@Since("1.4.0") private[ml] trait HasWeightCol extends Params { /** @@ -376,6 +400,7 @@ private[ml] trait HasWeightCol extends Params { /** * Trait for shared param solver (default: "auto"). */ +@Since("1.4.0") private[ml] trait HasSolver extends Params { /** @@ -393,6 +418,7 @@ private[ml] trait HasSolver extends Params { /** * Trait for shared param aggregationDepth (default: 2). */ +@Since("1.4.0") private[ml] trait HasAggregationDepth extends Params { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/python/MLSerDe.scala b/mllib/src/main/scala/org/apache/spark/ml/python/MLSerDe.scala index da62f8518e36..14816cc068d4 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/python/MLSerDe.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/python/MLSerDe.scala @@ -19,9 +19,8 @@ package org.apache.spark.ml.python import java.io.OutputStream import java.nio.{ByteBuffer, ByteOrder} - +import org.apache.spark.annotation.Since import net.razorvine.pickle._ - import org.apache.spark.api.python.SerDeUtil import org.apache.spark.ml.linalg._ import org.apache.spark.mllib.api.python.SerDeBase @@ -29,6 +28,7 @@ import org.apache.spark.mllib.api.python.SerDeBase /** * SerDe utility functions for pyspark.ml. */ +@Since("2.0.0") private[spark] object MLSerDe extends SerDeBase with Serializable { override val PYSPARK_PACKAGE = "pyspark.ml" diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala index 0bf543d88894..6b4e21d48b50 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/AFTSurvivalRegressionWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -30,6 +32,7 @@ import org.apache.spark.ml.regression.{AFTSurvivalRegression, AFTSurvivalRegress import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.0.0") private[r] class AFTSurvivalRegressionWrapper private ( val pipeline: PipelineModel, val features: Array[String]) extends MLWritable { @@ -57,6 +60,7 @@ private[r] class AFTSurvivalRegressionWrapper private ( new AFTSurvivalRegressionWrapper.AFTSurvivalRegressionWrapperWriter(this) } +@Since("2.0.0") private[r] object AFTSurvivalRegressionWrapper extends MLReadable[AFTSurvivalRegressionWrapper] { private def formulaRewrite(formula: String): (String, String) = { diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/ALSWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/ALSWrapper.scala index ad13cced4667..b77d53b42e1c 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/ALSWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/ALSWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -26,6 +28,7 @@ import org.apache.spark.ml.recommendation.{ALS, ALSModel} import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.1.0") private[r] class ALSWrapper private ( val alsModel: ALSModel, val ratingCol: String) extends MLWritable { @@ -43,6 +46,7 @@ private[r] class ALSWrapper private ( override def write: MLWriter = new ALSWrapper.ALSWrapperWriter(this) } +@Since("2.1.0") private[r] object ALSWrapper extends MLReadable[ALSWrapper] { def fit( // scalastyle:ignore diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/BisectingKMeansWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/BisectingKMeansWrapper.scala index 71712c1c5eec..56a55465406e 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/BisectingKMeansWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/BisectingKMeansWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -29,6 +31,7 @@ import org.apache.spark.ml.feature.RFormula import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.2.0") private[r] class BisectingKMeansWrapper private ( val pipeline: PipelineModel, val features: Array[String], diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/GBTClassificationWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/GBTClassificationWrapper.scala index aacb41ee2659..3a564457ac4d 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/GBTClassificationWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/GBTClassificationWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -30,6 +32,7 @@ import org.apache.spark.ml.r.RWrapperUtils._ import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.1.0") private[r] class GBTClassifierWrapper private ( val pipeline: PipelineModel, val formula: String, @@ -58,6 +61,7 @@ private[r] class GBTClassifierWrapper private ( GBTClassifierWrapper.GBTClassifierWrapperWriter(this) } +@Since("2.1.0") private[r] object GBTClassifierWrapper extends MLReadable[GBTClassifierWrapper] { val PREDICTED_LABEL_INDEX_COL = "pred_label_idx" diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressionWrapper.scala index 585077588eb9..6a2d275e8266 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressionWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressionWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -30,6 +32,7 @@ import org.apache.spark.ml.regression.{GBTRegressionModel, GBTRegressor} import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.1.0") private[r] class GBTRegressorWrapper private ( val pipeline: PipelineModel, val formula: String, @@ -53,6 +56,7 @@ private[r] class GBTRegressorWrapper private ( GBTRegressorWrapper.GBTRegressorWrapperWriter(this) } +@Since("2.1.0") private[r] object GBTRegressorWrapper extends MLReadable[GBTRegressorWrapper] { def fit( // scalastyle:ignore data: DataFrame, diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/GeneralizedLinearRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/GeneralizedLinearRegressionWrapper.scala index cbd6cd1c7933..29e198548b8f 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/GeneralizedLinearRegressionWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/GeneralizedLinearRegressionWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -30,6 +32,7 @@ import org.apache.spark.ml.regression._ import org.apache.spark.ml.util._ import org.apache.spark.sql._ +@Since("2.0.0") private[r] class GeneralizedLinearRegressionWrapper private ( val pipeline: PipelineModel, val rFeatures: Array[String], @@ -60,6 +63,7 @@ private[r] class GeneralizedLinearRegressionWrapper private ( new GeneralizedLinearRegressionWrapper.GeneralizedLinearRegressionWrapperWriter(this) } +@Since("2.0.0") private[r] object GeneralizedLinearRegressionWrapper extends MLReadable[GeneralizedLinearRegressionWrapper] { diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/IsotonicRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/IsotonicRegressionWrapper.scala index d31ebb46afb9..6668819019a3 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/IsotonicRegressionWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/IsotonicRegressionWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -29,6 +31,7 @@ import org.apache.spark.ml.regression.{IsotonicRegression, IsotonicRegressionMod import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.1.0") private[r] class IsotonicRegressionWrapper private ( val pipeline: PipelineModel, val features: Array[String]) extends MLWritable { @@ -47,6 +50,7 @@ private[r] class IsotonicRegressionWrapper private ( override def write: MLWriter = new IsotonicRegressionWrapper.IsotonicRegressionWrapperWriter(this) } +@Since("2.1.0") private[r] object IsotonicRegressionWrapper extends MLReadable[IsotonicRegressionWrapper] { diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/KMeansWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/KMeansWrapper.scala index 8d596863b459..cf02356ba6e0 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/KMeansWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/KMeansWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -29,6 +31,7 @@ import org.apache.spark.ml.feature.RFormula import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.0.0") private[r] class KMeansWrapper private ( val pipeline: PipelineModel, val features: Array[String], @@ -63,6 +66,7 @@ private[r] class KMeansWrapper private ( override def write: MLWriter = new KMeansWrapper.KMeansWrapperWriter(this) } +@Since("2.0.0") private[r] object KMeansWrapper extends MLReadable[KMeansWrapper] { def fit( diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala index 21531eb057ad..7506519555c4 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/KSTestWrapper.scala @@ -17,10 +17,13 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.spark.mllib.stat.Statistics.kolmogorovSmirnovTest import org.apache.spark.mllib.stat.test.KolmogorovSmirnovTestResult import org.apache.spark.sql.{DataFrame, Row} +@Since("2.1.0") private[r] class KSTestWrapper private ( val testResult: KolmogorovSmirnovTestResult, val distName: String, @@ -37,6 +40,7 @@ private[r] class KSTestWrapper private ( def summary: String = testResult.toString } +@Since("2.1.0") private[r] object KSTestWrapper { def test( diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/LDAWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/LDAWrapper.scala index e096bf1f29f3..9122cbd9c49f 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/LDAWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/LDAWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import scala.collection.mutable import org.apache.hadoop.fs.Path @@ -35,7 +37,7 @@ import org.apache.spark.sql.{DataFrame, Dataset} import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StringType - +@Since("2.1.0") private[r] class LDAWrapper private ( val pipeline: PipelineModel, val logLikelihood: Double, @@ -88,6 +90,7 @@ private[r] class LDAWrapper private ( override def write: MLWriter = new LDAWrapper.LDAWrapperWriter(this) } +@Since("2.1.0") private[r] object LDAWrapper extends MLReadable[LDAWrapper] { val TOKENIZER_COL = s"${Identifiable.randomUID("rawTokens")}" diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/LinearSVCWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/LinearSVCWrapper.scala index cfd043b66ed9..f778cdc9459c 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/LinearSVCWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/LinearSVCWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -29,6 +31,7 @@ import org.apache.spark.ml.r.RWrapperUtils._ import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.2.0") private[r] class LinearSVCWrapper private ( val pipeline: PipelineModel, val features: Array[String], @@ -56,6 +59,7 @@ private[r] class LinearSVCWrapper private ( override def write: MLWriter = new LinearSVCWrapper.LinearSVCWrapperWriter(this) } +@Since("2.2.0") private[r] object LinearSVCWrapper extends MLReadable[LinearSVCWrapper] { diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/LogisticRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/LogisticRegressionWrapper.scala index c96f99cb8343..f7828ef2b731 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/LogisticRegressionWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/LogisticRegressionWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -30,6 +32,7 @@ import org.apache.spark.ml.r.RWrapperUtils._ import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.1.0") private[r] class LogisticRegressionWrapper private ( val pipeline: PipelineModel, val features: Array[String], @@ -80,6 +83,7 @@ private[r] class LogisticRegressionWrapper private ( override def write: MLWriter = new LogisticRegressionWrapper.LogisticRegressionWrapperWriter(this) } +@Since("2.1.0") private[r] object LogisticRegressionWrapper extends MLReadable[LogisticRegressionWrapper] { diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/MultilayerPerceptronClassifierWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/MultilayerPerceptronClassifierWrapper.scala index d34de3093114..4fe960ca6dda 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/MultilayerPerceptronClassifierWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/MultilayerPerceptronClassifierWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -30,6 +32,7 @@ import org.apache.spark.ml.r.RWrapperUtils._ import org.apache.spark.ml.util.{MLReadable, MLReader, MLWritable, MLWriter} import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.1.0") private[r] class MultilayerPerceptronClassifierWrapper private ( val pipeline: PipelineModel ) extends MLWritable { @@ -56,6 +59,7 @@ private[r] class MultilayerPerceptronClassifierWrapper private ( new MultilayerPerceptronClassifierWrapper.MultilayerPerceptronClassifierWrapperWriter(this) } +@Since("2.1.0") private[r] object MultilayerPerceptronClassifierWrapper extends MLReadable[MultilayerPerceptronClassifierWrapper] { diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/NaiveBayesWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/NaiveBayesWrapper.scala index 0afea4be3d1d..3a7c575ee446 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/NaiveBayesWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/NaiveBayesWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -29,6 +31,7 @@ import org.apache.spark.ml.r.RWrapperUtils._ import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.0.0") private[r] class NaiveBayesWrapper private ( val pipeline: PipelineModel, val labels: Array[String], @@ -52,6 +55,7 @@ private[r] class NaiveBayesWrapper private ( override def write: MLWriter = new NaiveBayesWrapper.NaiveBayesWrapperWriter(this) } +@Since("2.0.0") private[r] object NaiveBayesWrapper extends MLReadable[NaiveBayesWrapper] { val PREDICTED_LABEL_INDEX_COL = "pred_label_idx" diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/RWrapperUtils.scala b/mllib/src/main/scala/org/apache/spark/ml/r/RWrapperUtils.scala index 665e50af67d4..d2771210f20a 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/RWrapperUtils.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/RWrapperUtils.scala @@ -17,12 +17,15 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.spark.internal.Logging import org.apache.spark.ml.attribute.{Attribute, AttributeGroup, NominalAttribute} import org.apache.spark.ml.feature.{RFormula, RFormulaModel} import org.apache.spark.ml.util.Identifiable import org.apache.spark.sql.Dataset +@Since("2.1.0") private[r] object RWrapperUtils extends Logging { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/RWrappers.scala b/mllib/src/main/scala/org/apache/spark/ml/r/RWrappers.scala index 358e522dfe1c..762fa4376a59 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/RWrappers.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/RWrappers.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s.DefaultFormats import org.json4s.jackson.JsonMethods._ @@ -28,6 +30,7 @@ import org.apache.spark.ml.util.MLReader * This is the Scala stub of SparkR read.ml. It will dispatch the call to corresponding * model wrapper loading function according the class name extracted from rMetadata of the path. */ +@Since("2.0.0") private[r] object RWrappers extends MLReader[Object] { override def load(path: String): Object = { diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassificationWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassificationWrapper.scala index 366f375b5858..f10e1e06e914 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassificationWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassificationWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -30,6 +32,7 @@ import org.apache.spark.ml.r.RWrapperUtils._ import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.1.0") private[r] class RandomForestClassifierWrapper private ( val pipeline: PipelineModel, val formula: String, @@ -58,6 +61,7 @@ private[r] class RandomForestClassifierWrapper private ( RandomForestClassifierWrapper.RandomForestClassifierWrapperWriter(this) } +@Since("2.1.0") private[r] object RandomForestClassifierWrapper extends MLReadable[RandomForestClassifierWrapper] { val PREDICTED_LABEL_INDEX_COL = "pred_label_idx" diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressionWrapper.scala index 4b9a3a731da9..dd729e7480d4 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressionWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressionWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.r +import org.apache.spark.annotation.Since + import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.JsonDSL._ @@ -30,6 +32,7 @@ import org.apache.spark.ml.regression.{RandomForestRegressionModel, RandomForest import org.apache.spark.ml.util._ import org.apache.spark.sql.{DataFrame, Dataset} +@Since("2.1.0") private[r] class RandomForestRegressorWrapper private ( val pipeline: PipelineModel, val formula: String, @@ -53,6 +56,7 @@ private[r] class RandomForestRegressorWrapper private ( RandomForestRegressorWrapper.RandomForestRegressorWrapperWriter(this) } +@Since("2.1.0") private[r] object RandomForestRegressorWrapper extends MLReadable[RandomForestRegressorWrapper] { def fit( // scalastyle:ignore data: DataFrame, diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/TopByKeyAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/TopByKeyAggregator.scala index 517179c0eb9a..a7658de678d1 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/TopByKeyAggregator.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/TopByKeyAggregator.scala @@ -19,7 +19,7 @@ package org.apache.spark.ml.recommendation import scala.language.implicitConversions import scala.reflect.runtime.universe.TypeTag - +import org.apache.spark.annotation.Since import org.apache.spark.sql.{Encoder, Encoders} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.expressions.Aggregator @@ -30,6 +30,7 @@ import org.apache.spark.util.BoundedPriorityQueue * Works on rows of the form (K1, K2, V) where K1 & K2 are IDs and V is the score value. Finds * the top `num` K2 items based on the given Ordering. */ +@Since("2.2.0") private[recommendation] class TopByKeyAggregator[K1: TypeTag, K2: TypeTag, V: TypeTag] (num: Int, ord: Ordering[(K2, V)]) extends Aggregator[(K1, K2, V), BoundedPriorityQueue[(K2, V)], Array[(K2, V)]] { diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala index c0a1683d3cb6..2c6c6fc5239f 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala @@ -17,7 +17,7 @@ package org.apache.spark.ml.regression -import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.annotation.{DeveloperApi, Since} import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams} @@ -28,6 +28,8 @@ import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams} * @tparam Learner Concrete Estimator type * @tparam M Concrete Model type */ + +@Since("1.3.0") private[spark] abstract class Regressor[ FeaturesType, Learner <: Regressor[FeaturesType, Learner, M], @@ -46,6 +48,7 @@ private[spark] abstract class Regressor[ * @tparam M Concrete Model type. */ @DeveloperApi +@Since("1.3.0") abstract class RegressionModel[FeaturesType, M <: RegressionModel[FeaturesType, M]] extends PredictionModel[FeaturesType, M] with PredictorParams { diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMDataSource.scala b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMDataSource.scala index e4de8483cfa3..7171479c68c8 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMDataSource.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMDataSource.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.source.libsvm +import org.apache.spark.annotation.Since + /** * `libsvm` package implements Spark SQL data source API for loading LIBSVM data as `DataFrame`. * The loaded `DataFrame` has two columns: `label` containing labels stored as doubles and @@ -50,4 +52,5 @@ package org.apache.spark.ml.source.libsvm * * @see LIBSVM datasets */ +@Since("2.0.0") class LibSVMDataSource private() {} diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMOptions.scala b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMOptions.scala index 6900b4153a7e..c3299b682e94 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMOptions.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMOptions.scala @@ -17,11 +17,13 @@ package org.apache.spark.ml.source.libsvm +import org.apache.spark.annotation.Since import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap /** * Options for the LibSVM data source. */ +@Since("2.2.0") private[libsvm] class LibSVMOptions(@transient private val parameters: CaseInsensitiveMap[String]) extends Serializable { diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala index f68847a664b6..1dd387de5064 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala @@ -18,11 +18,10 @@ package org.apache.spark.ml.source.libsvm import java.io.IOException - +import org.apache.spark.annotation.Since import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext} - import org.apache.spark.TaskContext import org.apache.spark.ml.feature.LabeledPoint import org.apache.spark.ml.linalg.{Vectors, VectorUDT} @@ -37,6 +36,7 @@ import org.apache.spark.sql.sources._ import org.apache.spark.sql.types._ import org.apache.spark.util.SerializableConfiguration +@Since("1.6.0") private[libsvm] class LibSVMOutputWriter( path: String, dataSchema: StructType, @@ -66,6 +66,7 @@ private[libsvm] class LibSVMOutputWriter( /** @see [[LibSVMDataSource]] for public documentation. */ // If this is moved or renamed, please update DataSource's backwardCompatibilityMap. +@Since("1.6.0") private[libsvm] class LibSVMFileFormat extends TextBasedFileFormat with DataSourceRegister { override def shortName(): String = "libsvm" diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala index 07e98a142b10..7a32bbf9fea2 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/Node.scala @@ -17,6 +17,7 @@ package org.apache.spark.ml.tree +import org.apache.spark.annotation.Since import org.apache.spark.ml.linalg.Vector import org.apache.spark.mllib.tree.impurity.ImpurityCalculator import org.apache.spark.mllib.tree.model.{ImpurityStats, @@ -25,6 +26,7 @@ import org.apache.spark.mllib.tree.model.{ImpurityStats, /** * Decision tree node interface. */ +@Since("1.4.0") sealed abstract class Node extends Serializable { // TODO: Add aggregate stats (once available). This will happen after we move the DecisionTree @@ -110,6 +112,7 @@ private[ml] object Node { * @param prediction Prediction this node makes * @param impurity Impurity measure at this node (for training data) */ +@Since("1.4.0") class LeafNode private[ml] ( override val prediction: Double, override val impurity: Double, @@ -151,6 +154,7 @@ class LeafNode private[ml] ( * @param rightChild Right-hand child node * @param split Information about the test used to split to the left or right child. */ +@Since("1.4.0") class InternalNode private[ml] ( override val prediction: Double, override val impurity: Double, @@ -213,6 +217,7 @@ class InternalNode private[ml] ( } } +@Since("1.4.0") private object InternalNode { /** @@ -258,6 +263,7 @@ private object InternalNode { * so that we do not need to consider splitting it further. * @param stats Impurity statistics for this node. */ +@Since("1.4.0") private[tree] class LearningNode( var id: Int, var leftChild: Option[LearningNode], @@ -327,6 +333,7 @@ private[tree] class LearningNode( } +@Since("1.4.0") private[tree] object LearningNode { /** Create a node with some of its fields set. */ diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/BaggedPoint.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/BaggedPoint.scala index 4e372702f0c6..444253d3878f 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/BaggedPoint.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/BaggedPoint.scala @@ -18,7 +18,7 @@ package org.apache.spark.ml.tree.impl import org.apache.commons.math3.distribution.PoissonDistribution - +import org.apache.spark.annotation.Since import org.apache.spark.rdd.RDD import org.apache.spark.util.Utils import org.apache.spark.util.random.XORShiftRandom @@ -38,6 +38,7 @@ import org.apache.spark.util.random.XORShiftRandom * TODO: This does not currently support (Double) weighted instances. Once MLlib has weighted * dataset support, update. (We store subsampleWeights as Double for this future extension.) */ +@Since("2.0.0") private[spark] class BaggedPoint[Datum](val datum: Datum, val subsampleWeights: Array[Double]) extends Serializable diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DTStatsAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DTStatsAggregator.scala index 61091bb803e4..bd924f90af86 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DTStatsAggregator.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DTStatsAggregator.scala @@ -18,7 +18,7 @@ package org.apache.spark.ml.tree.impl import org.apache.spark.mllib.tree.impurity._ - +import org.apache.spark.annotation.Since /** @@ -27,6 +27,7 @@ import org.apache.spark.mllib.tree.impurity._ * and helps with indexing. * This class is abstract to support learning with and without feature subsampling. */ +@Since("2.0.0") private[spark] class DTStatsAggregator( val metadata: DecisionTreeMetadata, featureSubset: Option[Array[Int]]) extends Serializable { diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala index 8a9dcb486b7b..f55ac7590ffe 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala @@ -19,7 +19,7 @@ package org.apache.spark.ml.tree.impl import scala.collection.mutable import scala.util.Try - +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.ml.feature.LabeledPoint import org.apache.spark.ml.tree.RandomForestParams @@ -39,6 +39,7 @@ import org.apache.spark.rdd.RDD * I.e., the feature takes values in {0, ..., arity - 1}. * @param numBins Number of bins for each feature. */ +@Since("2.0.0") private[spark] class DecisionTreeMetadata( val numFeatures: Int, val numExamples: Long, @@ -96,6 +97,7 @@ private[spark] class DecisionTreeMetadata( } +@Since("2.0.0") private[spark] object DecisionTreeMetadata extends Logging { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala index 4c525c0714ec..7ecc2157268e 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala @@ -17,6 +17,7 @@ package org.apache.spark.ml.tree.impl +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.ml.feature.LabeledPoint import org.apache.spark.ml.linalg.Vector @@ -29,7 +30,7 @@ import org.apache.spark.mllib.tree.loss.{Loss => OldLoss} import org.apache.spark.rdd.RDD import org.apache.spark.storage.StorageLevel - +@Since("2.0.0") private[spark] object GradientBoostedTrees extends Logging { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala index a7c5f489dea8..e460139bc066 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala @@ -18,11 +18,9 @@ package org.apache.spark.ml.tree.impl import java.io.IOException - +import org.apache.spark.annotation.Since import scala.collection.mutable - import org.apache.hadoop.fs.Path - import org.apache.spark.internal.Logging import org.apache.spark.ml.tree.{LearningNode, Split} import org.apache.spark.rdd.RDD @@ -34,6 +32,7 @@ import org.apache.spark.storage.StorageLevel * @param split Split information. * @param nodeIndex The current node index of a data point that this will update. */ +@Since("1.5.0") private[tree] case class NodeIndexUpdater(split: Split, nodeIndex: Int) { /** @@ -61,6 +60,7 @@ private[tree] case class NodeIndexUpdater(split: Split, nodeIndex: Int) { * @param checkpointInterval The checkpointing interval * (how often should the cache be checkpointed.). */ +@Since("1.5.0") private[spark] class NodeIdCache( var nodeIdsForInstances: RDD[Array[Int]], val checkpointInterval: Int) extends Logging { @@ -173,6 +173,7 @@ private[spark] class NodeIdCache( } } +@Since("1.5.0") private[spark] object NodeIdCache { /** * Initialize the node Id cache with initial node Id values. diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala index 008dd19c2498..72a410566459 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala @@ -18,10 +18,9 @@ package org.apache.spark.ml.tree.impl import java.io.IOException - import scala.collection.mutable import scala.util.Random - +import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging import org.apache.spark.ml.classification.DecisionTreeClassificationModel import org.apache.spark.ml.feature.LabeledPoint @@ -77,6 +76,8 @@ import org.apache.spark.util.random.{SamplingUtils, XORShiftRandom} * the heaviest part of the computation. In general, this implementation is bound by either * the cost of statistics computation on workers or by communicating the sufficient statistics. */ + +@Since("1.5.0") private[spark] object RandomForest extends Logging { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/TimeTracker.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/TimeTracker.scala index 4cc250aa462e..1f83d4ad0b3d 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/TimeTracker.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/TimeTracker.scala @@ -17,11 +17,13 @@ package org.apache.spark.ml.tree.impl +import org.apache.spark.annotation.Since import scala.collection.mutable.{HashMap => MutableHashMap} /** * Time tracker implementation which holds labeled timers. */ +@Since("2.0.0") private[spark] class TimeTracker extends Serializable { private val starts: MutableHashMap[String, Long] = new MutableHashMap[String, Long]() diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/TreePoint.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/TreePoint.scala index a6ac64a0463c..9de9a62cae08 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/TreePoint.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/TreePoint.scala @@ -17,6 +17,7 @@ package org.apache.spark.ml.tree.impl +import org.apache.spark.annotation.Since import org.apache.spark.ml.feature.LabeledPoint import org.apache.spark.ml.tree.{ContinuousSplit, Split} import org.apache.spark.rdd.RDD @@ -37,10 +38,13 @@ import org.apache.spark.rdd.RDD * @param binnedFeatures Binned feature values. * Same length as LabeledPoint.features, but values are bin indices. */ + +@Since("1.5.0") private[spark] class TreePoint(val label: Double, val binnedFeatures: Array[Int]) extends Serializable { } +@Since("1.5.0") private[spark] object TreePoint { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala index 0d6e9034e5ce..f5ff1dca7b12 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala @@ -18,11 +18,10 @@ package org.apache.spark.ml.tree import scala.reflect.ClassTag - +import org.apache.spark.annotation.Since import org.apache.hadoop.fs.Path import org.json4s._ import org.json4s.jackson.JsonMethods._ - import org.apache.spark.ml.linalg.{Vector, Vectors} import org.apache.spark.ml.param.{Param, Params} import org.apache.spark.ml.tree.DecisionTreeModelReadWrite.NodeData @@ -39,6 +38,7 @@ import org.apache.spark.util.collection.OpenHashMap * * TODO: Add support for predicting probabilities and raw predictions SPARK-3727 */ +@Since("1.4.0") private[spark] trait DecisionTreeModel { /** Root of the decision tree */ @@ -87,6 +87,7 @@ private[spark] trait DecisionTreeModel { * * @tparam M Type of tree model in this ensemble */ +@Since("1.4.0") private[ml] trait TreeEnsembleModel[M <: DecisionTreeModel] { // Note: We use getTrees since subclasses of TreeEnsembleModel will store subclasses of @@ -120,6 +121,7 @@ private[ml] trait TreeEnsembleModel[M <: DecisionTreeModel] { lazy val totalNumNodes: Int = trees.map(_.numNodes).sum } +@Since("1.4.0") private[ml] object TreeEnsembleModel { /** @@ -240,6 +242,7 @@ private[ml] object TreeEnsembleModel { } /** Helper classes for tree model persistence */ +@Since("1.4.0") private[ml] object DecisionTreeModelReadWrite { /** @@ -376,6 +379,7 @@ private[ml] object DecisionTreeModelReadWrite { } } +@Since("1.4.0") private[ml] object EnsembleModelReadWrite { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala index 5eb707dfe7bc..a2a5e1e94dea 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala @@ -18,7 +18,7 @@ package org.apache.spark.ml.tree import scala.util.Try - +import org.apache.spark.annotation.Since import org.apache.spark.ml.PredictorParams import org.apache.spark.ml.param._ import org.apache.spark.ml.param.shared._ @@ -33,6 +33,7 @@ import org.apache.spark.sql.types.{DataType, DoubleType, StructType} * * Note: Marked as private and DeveloperApi since this may be made public in the future. */ +@Since("1.4.0") private[ml] trait DecisionTreeParams extends PredictorParams with HasCheckpointInterval with HasSeed { @@ -207,6 +208,7 @@ private[ml] trait DecisionTreeParams extends PredictorParams /** * Parameters for Decision Tree-based classification algorithms. */ +@Since("1.4.0") private[ml] trait TreeClassifierParams extends Params { /** @@ -245,17 +247,20 @@ private[ml] trait TreeClassifierParams extends Params { } } +@Since("1.4.0") private[ml] object TreeClassifierParams { // These options should be lowercase. final val supportedImpurities: Array[String] = Array("entropy", "gini").map(_.toLowerCase) } +@Since("1.4.0") private[ml] trait DecisionTreeClassifierParams extends DecisionTreeParams with TreeClassifierParams /** * Parameters for Decision Tree-based regression algorithms. */ +@Since("1.4.0") private[ml] trait TreeRegressorParams extends Params { /** @@ -293,11 +298,13 @@ private[ml] trait TreeRegressorParams extends Params { } } +@Since("1.4.0") private[ml] object TreeRegressorParams { // These options should be lowercase. final val supportedImpurities: Array[String] = Array("variance").map(_.toLowerCase) } +@Since("1.4.0") private[ml] trait DecisionTreeRegressorParams extends DecisionTreeParams with TreeRegressorParams with HasVarianceCol { @@ -319,6 +326,7 @@ private[ml] trait DecisionTreeRegressorParams extends DecisionTreeParams * * Note: Marked as private and DeveloperApi since this may be made public in the future. */ +@Since("1.4.0") private[ml] trait TreeEnsembleParams extends DecisionTreeParams { /** @@ -358,6 +366,7 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams { /** * Parameters for Random Forest algorithms. */ +@Since("1.4.0") private[ml] trait RandomForestParams extends TreeEnsembleParams { /** @@ -434,15 +443,18 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams { final def getFeatureSubsetStrategy: String = $(featureSubsetStrategy).toLowerCase } +@Since("1.4.0") private[spark] object RandomForestParams { // These options should be lowercase. final val supportedFeatureSubsetStrategies: Array[String] = Array("auto", "all", "onethird", "sqrt", "log2").map(_.toLowerCase) } +@Since("1.4.0") private[ml] trait RandomForestClassifierParams extends RandomForestParams with TreeClassifierParams +@Since("1.4.0") private[ml] trait RandomForestRegressorParams extends RandomForestParams with TreeRegressorParams @@ -451,6 +463,7 @@ private[ml] trait RandomForestRegressorParams * * Note: Marked as private and DeveloperApi since this may be made public in the future. */ +@Since("1.4.0") private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter { /* TODO: Add this doc when we add this param. SPARK-7132 @@ -506,12 +519,14 @@ private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter { private[ml] def getOldLossType: OldLoss } +@Since("1.4.0") private[ml] object GBTClassifierParams { // The losses below should be lowercase. /** Accessor for supported loss settings: logistic */ final val supportedLossTypes: Array[String] = Array("logistic").map(_.toLowerCase) } +@Since("1.4.0") private[ml] trait GBTClassifierParams extends GBTParams with TreeClassifierParams { /** @@ -541,12 +556,14 @@ private[ml] trait GBTClassifierParams extends GBTParams with TreeClassifierParam } } +@Since("1.4.0") private[ml] object GBTRegressorParams { // The losses below should be lowercase. /** Accessor for supported loss settings: squared (L2), absolute (L1) */ final val supportedLossTypes: Array[String] = Array("squared", "absolute").map(_.toLowerCase) } +@Since("1.4.0") private[ml] trait GBTRegressorParams extends GBTParams with TreeRegressorParams { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/tuning/ValidatorParams.scala b/mllib/src/main/scala/org/apache/spark/ml/tuning/ValidatorParams.scala index d55eb14d0345..23ee5698b9aa 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tuning/ValidatorParams.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tuning/ValidatorParams.scala @@ -20,7 +20,7 @@ package org.apache.spark.ml.tuning import org.apache.hadoop.fs.Path import org.json4s.{DefaultFormats, _} import org.json4s.jackson.JsonMethods._ - +import org.apache.spark.annotation.Since import org.apache.spark.SparkContext import org.apache.spark.ml.{Estimator, Model} import org.apache.spark.ml.evaluation.Evaluator @@ -33,6 +33,7 @@ import org.apache.spark.sql.types.StructType /** * Common params for [[TrainValidationSplitParams]] and [[CrossValidatorParams]]. */ +@Since("1.5.0") private[ml] trait ValidatorParams extends HasSeed with Params { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala b/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala index bd213e7362e9..ec29fa3ef482 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala @@ -19,7 +19,7 @@ package org.apache.spark.ml.util import java.util.UUID -import org.apache.spark.annotation.DeveloperApi +import org.apache.spark.annotation.{DeveloperApi, Since} /** @@ -31,6 +31,7 @@ import org.apache.spark.annotation.DeveloperApi * releases. */ @DeveloperApi +@Since("1.4.0") trait Identifiable { /** @@ -45,6 +46,7 @@ trait Identifiable { * :: DeveloperApi :: */ @DeveloperApi +@Since("1.4.0") object Identifiable { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala b/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala index 7c46f45c5971..0fdeb6a9b841 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/util/Instrumentation.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.util +import org.apache.spark.annotation.Since + import java.util.concurrent.atomic.AtomicLong import org.json4s._ @@ -39,6 +41,7 @@ import org.apache.spark.sql.Dataset * @param dataset the training dataset * @tparam E the type of the estimator */ +@Since("2.0.0") private[spark] class Instrumentation[E <: Estimator[_]] private ( estimator: E, dataset: RDD[_]) extends Logging { @@ -106,6 +109,7 @@ private[spark] class Instrumentation[E <: Estimator[_]] private ( /** * Some common methods for logging information about a training session. */ +@Since("2.0.0") private[spark] object Instrumentation { private val counter = new AtomicLong(0) diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/MetadataUtils.scala b/mllib/src/main/scala/org/apache/spark/ml/util/MetadataUtils.scala index 3e19f2718394..e5a801feb55f 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/util/MetadataUtils.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/util/MetadataUtils.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.util +import org.apache.spark.annotation.Since + import scala.collection.immutable.HashMap import org.apache.spark.ml.attribute._ @@ -27,6 +29,7 @@ import org.apache.spark.sql.types.StructField /** * Helper utilities for algorithms using ML metadata */ +@Since("1.4.0") private[spark] object MetadataUtils { /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/SchemaUtils.scala b/mllib/src/main/scala/org/apache/spark/ml/util/SchemaUtils.scala index 334410c9620d..770213ea07b4 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/util/SchemaUtils.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/util/SchemaUtils.scala @@ -17,12 +17,15 @@ package org.apache.spark.ml.util +import org.apache.spark.annotation.Since + import org.apache.spark.sql.types.{DataType, NumericType, StructField, StructType} /** * Utils for handling schemas. */ +@Since("1.4.0") private[spark] object SchemaUtils { // TODO: Move the utility methods to SQL. diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/stopwatches.scala b/mllib/src/main/scala/org/apache/spark/ml/util/stopwatches.scala index e539deca4b03..123d626decd4 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/util/stopwatches.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/util/stopwatches.scala @@ -17,6 +17,8 @@ package org.apache.spark.ml.util +import org.apache.spark.annotation.Since + import scala.collection.mutable import org.apache.spark.SparkContext @@ -25,6 +27,7 @@ import org.apache.spark.util.LongAccumulator /** * Abstract class for stopwatches. */ +@Since("1.5.0") private[spark] abstract class Stopwatch extends Serializable { @transient private var running: Boolean = false @@ -84,6 +87,7 @@ private[spark] abstract class Stopwatch extends Serializable { /** * A local [[Stopwatch]]. */ +@Since("1.5.0") private[spark] class LocalStopwatch(override val name: String) extends Stopwatch { private var elapsedTime: Long = 0L @@ -99,6 +103,7 @@ private[spark] class LocalStopwatch(override val name: String) extends Stopwatch * A distributed [[Stopwatch]] using Spark accumulator. * @param sc SparkContext */ +@Since("1.5.0") private[spark] class DistributedStopwatch( sc: SparkContext, override val name: String) extends Stopwatch { @@ -116,6 +121,7 @@ private[spark] class DistributedStopwatch( * A multiple stopwatch that contains local and distributed stopwatches. * @param sc SparkContext */ +@Since("1.5.0") private[spark] class MultiStopwatch(@transient private val sc: SparkContext) extends Serializable { private val stopwatches: mutable.Map[String, Stopwatch] = mutable.Map.empty diff --git a/mllib/src/main/scala/org/apache/spark/mllib/api/python/GaussianMixtureModelWrapper.scala b/mllib/src/main/scala/org/apache/spark/mllib/api/python/GaussianMixtureModelWrapper.scala index 364d5eea08ce..964b6237c475 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/api/python/GaussianMixtureModelWrapper.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/api/python/GaussianMixtureModelWrapper.scala @@ -17,6 +17,8 @@ package org.apache.spark.mllib.api.python +import org.apache.spark.annotation.Since + import scala.collection.JavaConverters import org.apache.spark.SparkContext @@ -26,6 +28,7 @@ import org.apache.spark.mllib.linalg.{Vector, Vectors} /** * Wrapper around GaussianMixtureModel to provide helper methods in Python */ +@Since("2.1.0") private[python] class GaussianMixtureModelWrapper(model: GaussianMixtureModel) { val weights: Vector = Vectors.dense(model.weights) val k: Int = weights.size