[SPARK-10931]

Copied parameters over from Estimator to Transformer

Estimator UID is being copied correctly to the Transformer model objects and params now, working on Doctests

Changed the way parameters are copied from the Estimator to Transformer

Checkpoint, switching back to inheritance method

Working on DocTests

Implemented Doctests for Recommendation, Clustering, Classification (except RandomForestClassifier), Evaluation, Tuning, Regression (except RandomRegression)

Ready for Code Review

Code Review changeset #1

Author: Evan Chen

python/pyspark/ml/classification.py

@@ -264,7 +264,12 @@ def getFamily(self):
         return self.getOrDefault(self.family)
 
 
-class LogisticRegressionModel(JavaModel, JavaClassificationModel, JavaMLWritable, JavaMLReadable):
+class LogisticRegressionModel(JavaModel, JavaClassificationModel, HasFeaturesCol,
+                              HasLabelCol, HasPredictionCol, HasMaxIter,
+                              HasRegParam, HasTol, HasProbabilityCol, HasRawPredictionCol,
+                              HasElasticNetParam, HasFitIntercept, HasStandardization,
+                              HasThresholds, HasWeightCol, HasAggregationDepth,
+                              JavaMLWritable, JavaMLReadable):
     """
     Model fitted by LogisticRegression.
 
@@ -669,8 +674,11 @@ def _create_model(self, java_model):
 
 
 @inherit_doc
-class DecisionTreeClassificationModel(DecisionTreeModel, JavaClassificationModel, JavaMLWritable,
-                                      JavaMLReadable):
+class DecisionTreeClassificationModel(DecisionTreeModel, JavaClassificationModel, HasFeaturesCol,
+                                      HasLabelCol, HasPredictionCol, HasProbabilityCol,
+                                      HasRawPredictionCol, DecisionTreeParams,
+                                      TreeClassifierParams, HasCheckpointInterval, HasSeed,
+                                      JavaMLWritable, JavaMLReadable):
     """
     Model fitted by DecisionTreeClassifier.
 
@@ -798,8 +806,9 @@ def _create_model(self, java_model):
         return RandomForestClassificationModel(java_model)
 
 
-class RandomForestClassificationModel(TreeEnsembleModel, JavaClassificationModel, JavaMLWritable,
-                                      JavaMLReadable):
+class RandomForestClassificationModel(TreeEnsembleModel, JavaClassificationModel, HasFeaturesCol,
+                                      HasLabelCol, HasPredictionCol, HasRawPredictionCol,
+                                      HasProbabilityCol, JavaMLWritable, JavaMLReadable):
     """
     Model fitted by RandomForestClassifier.
 
@@ -950,7 +959,8 @@ def getLossType(self):
         return self.getOrDefault(self.lossType)
 
 
-class GBTClassificationModel(TreeEnsembleModel, JavaPredictionModel, JavaMLWritable,
+class GBTClassificationModel(TreeEnsembleModel, JavaPredictionModel, HasFeaturesCol,
+                             HasLabelCol, HasPredictionCol, JavaMLWritable,
                              JavaMLReadable):
     """
     Model fitted by GBTClassifier.
@@ -1105,7 +1115,9 @@ def getModelType(self):
         return self.getOrDefault(self.modelType)
 
 
-class NaiveBayesModel(JavaModel, JavaClassificationModel, JavaMLWritable, JavaMLReadable):
+class NaiveBayesModel(JavaModel, JavaClassificationModel, HasFeaturesCol, HasLabelCol,
+                      HasPredictionCol, HasProbabilityCol, HasRawPredictionCol,
+                      JavaMLWritable, JavaMLReadable):
     """
     Model fitted by NaiveBayes.
 
@@ -1304,8 +1316,9 @@ def getInitialWeights(self):
         return self.getOrDefault(self.initialWeights)
 
 
-class MultilayerPerceptronClassificationModel(JavaModel, JavaPredictionModel, JavaMLWritable,
-                                              JavaMLReadable):
+class MultilayerPerceptronClassificationModel(JavaModel, JavaPredictionModel,
+                                              HasFeaturesCol, HasLabelCol, HasPredictionCol,
+                                              JavaMLWritable, JavaMLReadable):
     """
     .. note:: Experimental
 

python/pyspark/ml/clustering.py

@@ -27,7 +27,8 @@
            'LDA', 'LDAModel', 'LocalLDAModel', 'DistributedLDAModel']
 
 
-class GaussianMixtureModel(JavaModel, JavaMLWritable, JavaMLReadable):
+class GaussianMixtureModel(JavaModel, HasFeaturesCol, HasPredictionCol, HasMaxIter, HasTol, HasSeed,
+                           HasProbabilityCol, JavaMLWritable, JavaMLReadable):
     """
     .. note:: Experimental
 
@@ -181,7 +182,8 @@ def getK(self):
         return self.getOrDefault(self.k)
 
 
-class KMeansModel(JavaModel, JavaMLWritable, JavaMLReadable):
+class KMeansModel(JavaModel, JavaMLWritable, JavaMLReadable, HasFeaturesCol,
+                  HasPredictionCol, HasMaxIter, HasTol, HasSeed):
     """
     Model fitted by KMeans.
 
@@ -324,7 +326,8 @@ def getInitSteps(self):
         return self.getOrDefault(self.initSteps)
 
 
-class BisectingKMeansModel(JavaModel, JavaMLWritable, JavaMLReadable):
+class BisectingKMeansModel(JavaModel, HasFeaturesCol, HasPredictionCol, HasMaxIter,
+                           HasSeed, JavaMLWritable, JavaMLReadable):
     """
     .. note:: Experimental
 
@@ -461,7 +464,7 @@ def _create_model(self, java_model):
 
 
 @inherit_doc
-class LDAModel(JavaModel):
+class LDAModel(JavaModel, HasFeaturesCol, HasMaxIter, HasSeed, HasCheckpointInterval):
     """
     .. note:: Experimental
 

python/pyspark/ml/feature.py

@@ -340,7 +340,8 @@ def _create_model(self, java_model):
         return CountVectorizerModel(java_model)
 
 
-class CountVectorizerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class CountVectorizerModel(JavaModel, HasInputCol, HasOutputCol,
+                           JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`CountVectorizer`.
 
@@ -635,7 +636,7 @@ def _create_model(self, java_model):
         return IDFModel(java_model)
 
 
-class IDFModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class IDFModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`IDF`.
 
@@ -713,7 +714,7 @@ def _create_model(self, java_model):
         return MaxAbsScalerModel(java_model)
 
 
-class MaxAbsScalerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class MaxAbsScalerModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     .. note:: Experimental
 
@@ -837,7 +838,7 @@ def _create_model(self, java_model):
         return MinMaxScalerModel(java_model)
 
 
-class MinMaxScalerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class MinMaxScalerModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`MinMaxScaler`.
 
@@ -1538,7 +1539,7 @@ def _create_model(self, java_model):
         return StandardScalerModel(java_model)
 
 
-class StandardScalerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class StandardScalerModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`StandardScaler`.
 
@@ -1626,7 +1627,8 @@ def _create_model(self, java_model):
         return StringIndexerModel(java_model)
 
 
-class StringIndexerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class StringIndexerModel(JavaModel, HasInputCol, HasOutputCol, HasHandleInvalid,
+                         JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`StringIndexer`.
 
@@ -1996,7 +1998,7 @@ def _create_model(self, java_model):
         return VectorIndexerModel(java_model)
 
 
-class VectorIndexerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class VectorIndexerModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`VectorIndexer`.
 
@@ -2134,6 +2136,15 @@ class Word2Vec(JavaEstimator, HasStepSize, HasMaxIter, HasSeed, HasInputCol, Has
     >>> doc = spark.createDataFrame([(sent,), (sent,)], ["sentence"])
     >>> word2Vec = Word2Vec(vectorSize=5, seed=42, inputCol="sentence", outputCol="model")
     >>> model = word2Vec.fit(doc)
+    >>> estimator_paramMap = word2Vec.extractParamMap()
+    >>> model_paramMap = model.extractParamMap()
+    >>> all([estimator_paramMap[getattr(word2Vec, param.name)] == value
+    ...     for param, value in model_paramMap.items()])
+    True
+    >>> all([param.parent == model.uid for param in model_paramMap])
+    True
+    >>> [param.name for param in model.params]
+    ['inputCol', 'maxIter', 'outputCol', 'seed', 'stepSize']
     >>> model.getVectors().show()
     +----+--------------------+
     |word|              vector|
@@ -2292,7 +2303,8 @@ def _create_model(self, java_model):
         return Word2VecModel(java_model)
 
 
-class Word2VecModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class Word2VecModel(JavaModel, HasStepSize, HasMaxIter, HasSeed, HasInputCol,
+                    HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`Word2Vec`.
 
@@ -2333,6 +2345,15 @@ class PCA(JavaEstimator, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritab
     >>> df = spark.createDataFrame(data,["features"])
     >>> pca = PCA(k=2, inputCol="features", outputCol="pca_features")
     >>> model = pca.fit(df)
+    >>> estimator_paramMap = pca.extractParamMap()
+    >>> model_paramMap = model.extractParamMap()
+    >>> all([estimator_paramMap[getattr(pca, param.name)] == value
+    ...     for param, value in model_paramMap.items()])
+    True
+    >>> all([param.parent == model.uid for param in model_paramMap])
+    True
+    >>> [param.name for param in model.params]
+    ['inputCol', 'outputCol']
     >>> model.transform(df).collect()[0].pca_features
     DenseVector([1.648..., -4.013...])
     >>> model.explainedVariance
@@ -2394,7 +2415,7 @@ def _create_model(self, java_model):
         return PCAModel(java_model)
 
 
-class PCAModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class PCAModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`PCA`. Transforms vectors to a lower dimensional space.
 
@@ -2437,6 +2458,15 @@ class RFormula(JavaEstimator, HasFeaturesCol, HasLabelCol, JavaMLReadable, JavaM
     ... ], ["y", "x", "s"])
     >>> rf = RFormula(formula="y ~ x + s")
     >>> model = rf.fit(df)
+    >>> estimator_paramMap = rf.extractParamMap()
+    >>> model_paramMap = model.extractParamMap()
+    >>> all([estimator_paramMap[getattr(rf, param.name)] == value
+    ...     for param, value in model_paramMap.items()])
+    True
+    >>> all([param.parent == model.uid for param in model_paramMap])
+    True
+    >>> [param.name for param in model.params]
+    ['featuresCol', 'labelCol']
     >>> model.transform(df).show()
     +---+---+---+---------+-----+
     |  y|  x|  s| features|label|
@@ -2554,7 +2584,7 @@ def __str__(self):
         return "RFormula(%s) (uid=%s)" % (formulaStr, self.uid)
 
 
-class RFormulaModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class RFormulaModel(JavaModel, HasFeaturesCol, HasLabelCol, JavaMLReadable, JavaMLWritable):
     """
     .. note:: Experimental
 
@@ -2586,6 +2616,15 @@ class ChiSqSelector(JavaEstimator, HasFeaturesCol, HasOutputCol, HasLabelCol, Ja
     ...    ["features", "label"])
     >>> selector = ChiSqSelector(numTopFeatures=1, outputCol="selectedFeatures")
     >>> model = selector.fit(df)
+    >>> estimator_paramMap = selector.extractParamMap()
+    >>> model_paramMap = model.extractParamMap()
+    >>> all([estimator_paramMap[getattr(selector, param.name)] == value
+    ...     for param, value in model_paramMap.items()])
+    True
+    >>> all([param.parent == model.uid for param in model_paramMap])
+    True
+    >>> [param.name for param in model.params]
+    ['featuresCol', 'labelCol', 'outputCol']
     >>> model.transform(df).head().selectedFeatures
     DenseVector([18.0])
     >>> model.selectedFeatures
@@ -2710,7 +2749,8 @@ def _create_model(self, java_model):
         return ChiSqSelectorModel(java_model)
 
 
-class ChiSqSelectorModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class ChiSqSelectorModel(JavaModel, HasFeaturesCol, HasOutputCol, HasLabelCol,
+                         JavaMLReadable, JavaMLWritable):
     """
     .. note:: Experimental
 

python/pyspark/ml/param/__init__.py

@@ -336,6 +336,11 @@ def hasParam(self, paramName):
             return isinstance(p, Param)
         else:
             raise TypeError("hasParam(): paramName must be a string")
+        try:
+            param = self._resolveParam(paramName)
+            return param in self.params
+        except:
+            return False
 
     @since("1.4.0")
     def getOrDefault(self, param):

python/pyspark/ml/recommendation.py

@@ -26,8 +26,8 @@
 
 
 @inherit_doc
-class ALS(JavaEstimator, HasCheckpointInterval, HasMaxIter, HasPredictionCol, HasRegParam, HasSeed,
-          JavaMLWritable, JavaMLReadable):
+class ALS(JavaEstimator, HasCheckpointInterval, HasMaxIter, HasPredictionCol,
+          HasRegParam, HasSeed, JavaMLWritable, JavaMLReadable):
     """
     Alternating Least Squares (ALS) matrix factorization.
 
@@ -333,7 +333,7 @@ def getFinalStorageLevel(self):
         return self.getOrDefault(self.finalStorageLevel)
 
 
-class ALSModel(JavaModel, JavaMLWritable, JavaMLReadable):
+class ALSModel(JavaModel, HasPredictionCol, JavaMLWritable, JavaMLReadable):
     """
     Model fitted by ALS.