[SPARK-10931][PYSPARK][ML] PySpark ML Models should contain Param values

python/pyspark/ml/classification.py

@@ -264,7 +264,12 @@ def getFamily(self):
         return self.getOrDefault(self.family)
 
 
-class LogisticRegressionModel(JavaModel, JavaClassificationModel, JavaMLWritable, JavaMLReadable):
+class LogisticRegressionModel(JavaModel, JavaClassificationModel, HasFeaturesCol,
+                              HasLabelCol, HasPredictionCol, HasMaxIter,
+                              HasRegParam, HasTol, HasProbabilityCol, HasRawPredictionCol,
+                              HasElasticNetParam, HasFitIntercept, HasStandardization,
+                              HasThresholds, HasWeightCol, HasAggregationDepth,
+                              JavaMLWritable, JavaMLReadable):
     """
     Model fitted by LogisticRegression.
 
@@ -669,8 +674,11 @@ def _create_model(self, java_model):
 
 
 @inherit_doc
-class DecisionTreeClassificationModel(DecisionTreeModel, JavaClassificationModel, JavaMLWritable,
-                                      JavaMLReadable):
+class DecisionTreeClassificationModel(DecisionTreeModel, JavaClassificationModel, HasFeaturesCol,
+                                      HasLabelCol, HasPredictionCol, HasProbabilityCol,
+                                      HasRawPredictionCol, DecisionTreeParams,
+                                      TreeClassifierParams, HasCheckpointInterval, HasSeed,
+                                      JavaMLWritable, JavaMLReadable):
     """
     Model fitted by DecisionTreeClassifier.
 
@@ -798,8 +806,9 @@ def _create_model(self, java_model):
         return RandomForestClassificationModel(java_model)
 
 
-class RandomForestClassificationModel(TreeEnsembleModel, JavaClassificationModel, JavaMLWritable,
-                                      JavaMLReadable):
+class RandomForestClassificationModel(TreeEnsembleModel, JavaClassificationModel, HasFeaturesCol,
+                                      HasLabelCol, HasPredictionCol, HasRawPredictionCol,
+                                      HasProbabilityCol, JavaMLWritable, JavaMLReadable):
     """
     Model fitted by RandomForestClassifier.
 
@@ -950,7 +959,8 @@ def getLossType(self):
         return self.getOrDefault(self.lossType)
 
 
-class GBTClassificationModel(TreeEnsembleModel, JavaPredictionModel, JavaMLWritable,
+class GBTClassificationModel(TreeEnsembleModel, JavaPredictionModel, HasFeaturesCol,
+                             HasLabelCol, HasPredictionCol, JavaMLWritable,
                              JavaMLReadable):
     """
     Model fitted by GBTClassifier.
@@ -1105,7 +1115,9 @@ def getModelType(self):
         return self.getOrDefault(self.modelType)
 
 
-class NaiveBayesModel(JavaModel, JavaClassificationModel, JavaMLWritable, JavaMLReadable):
+class NaiveBayesModel(JavaModel, JavaClassificationModel, HasFeaturesCol, HasLabelCol,
+                      HasPredictionCol, HasProbabilityCol, HasRawPredictionCol,
+                      JavaMLWritable, JavaMLReadable):
     """
     Model fitted by NaiveBayes.
 
@@ -1304,8 +1316,9 @@ def getInitialWeights(self):
         return self.getOrDefault(self.initialWeights)
 
 
-class MultilayerPerceptronClassificationModel(JavaModel, JavaPredictionModel, JavaMLWritable,
-                                              JavaMLReadable):
+class MultilayerPerceptronClassificationModel(JavaModel, JavaPredictionModel,
+                                              HasFeaturesCol, HasLabelCol, HasPredictionCol,
+                                              JavaMLWritable, JavaMLReadable):
     """
     .. note:: Experimental
 

python/pyspark/ml/clustering.py

@@ -27,7 +27,8 @@
            'LDA', 'LDAModel', 'LocalLDAModel', 'DistributedLDAModel']
 
 
-class GaussianMixtureModel(JavaModel, JavaMLWritable, JavaMLReadable):
+class GaussianMixtureModel(JavaModel, HasFeaturesCol, HasPredictionCol, HasMaxIter, HasTol, HasSeed,
+                           HasProbabilityCol, JavaMLWritable, JavaMLReadable):
     """
     .. note:: Experimental
 
@@ -181,7 +182,8 @@ def getK(self):
         return self.getOrDefault(self.k)
 
 
-class KMeansModel(JavaModel, JavaMLWritable, JavaMLReadable):
+class KMeansModel(JavaModel, JavaMLWritable, JavaMLReadable, HasFeaturesCol,
+                  HasPredictionCol, HasMaxIter, HasTol, HasSeed):
     """
     Model fitted by KMeans.
 
@@ -324,7 +326,8 @@ def getInitSteps(self):
         return self.getOrDefault(self.initSteps)
 
 
-class BisectingKMeansModel(JavaModel, JavaMLWritable, JavaMLReadable):
+class BisectingKMeansModel(JavaModel, HasFeaturesCol, HasPredictionCol, HasMaxIter,
+                           HasSeed, JavaMLWritable, JavaMLReadable):
     """
     .. note:: Experimental
 
@@ -461,7 +464,7 @@ def _create_model(self, java_model):
 
 
 @inherit_doc
-class LDAModel(JavaModel):
+class LDAModel(JavaModel, HasFeaturesCol, HasMaxIter, HasSeed, HasCheckpointInterval):
     """
     .. note:: Experimental
 

python/pyspark/ml/feature.py

@@ -340,7 +340,8 @@ def _create_model(self, java_model):
         return CountVectorizerModel(java_model)
 
 
-class CountVectorizerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class CountVectorizerModel(JavaModel, HasInputCol, HasOutputCol,
+                           JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`CountVectorizer`.
 
@@ -635,7 +636,7 @@ def _create_model(self, java_model):
         return IDFModel(java_model)
 
 
-class IDFModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class IDFModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`IDF`.
 
@@ -713,7 +714,7 @@ def _create_model(self, java_model):
         return MaxAbsScalerModel(java_model)
 
 
-class MaxAbsScalerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class MaxAbsScalerModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     .. note:: Experimental
 
@@ -837,7 +838,7 @@ def _create_model(self, java_model):
         return MinMaxScalerModel(java_model)
 
 
-class MinMaxScalerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class MinMaxScalerModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`MinMaxScaler`.
 
@@ -1538,7 +1539,7 @@ def _create_model(self, java_model):
         return StandardScalerModel(java_model)
 
 
-class StandardScalerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class StandardScalerModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`StandardScaler`.
 
@@ -1626,7 +1627,8 @@ def _create_model(self, java_model):
         return StringIndexerModel(java_model)
 
 
-class StringIndexerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class StringIndexerModel(JavaModel, HasInputCol, HasOutputCol, HasHandleInvalid,
+                         JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`StringIndexer`.
 
@@ -1996,7 +1998,7 @@ def _create_model(self, java_model):
         return VectorIndexerModel(java_model)
 
 
-class VectorIndexerModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class VectorIndexerModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`VectorIndexer`.
 
@@ -2134,6 +2136,15 @@ class Word2Vec(JavaEstimator, HasStepSize, HasMaxIter, HasSeed, HasInputCol, Has
     >>> doc = spark.createDataFrame([(sent,), (sent,)], ["sentence"])
     >>> word2Vec = Word2Vec(vectorSize=5, seed=42, inputCol="sentence", outputCol="model")
     >>> model = word2Vec.fit(doc)
+    >>> estimator_paramMap = word2Vec.extractParamMap()
+    >>> model_paramMap = model.extractParamMap()
+    >>> all([estimator_paramMap[getattr(word2Vec, param.name)] == value
+    ...     for param, value in model_paramMap.items()])
+    True
+    >>> all([param.parent == model.uid for param in model_paramMap])
+    True
+    >>> [param.name for param in model.params]
+    ['inputCol', 'maxIter', 'outputCol', 'seed', 'stepSize']
     >>> model.getVectors().show()
     +----+--------------------+
     |word|              vector|
@@ -2292,7 +2303,8 @@ def _create_model(self, java_model):
         return Word2VecModel(java_model)
 
 
-class Word2VecModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class Word2VecModel(JavaModel, HasStepSize, HasMaxIter, HasSeed, HasInputCol,
+                    HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`Word2Vec`.
 
@@ -2333,6 +2345,15 @@ class PCA(JavaEstimator, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritab
     >>> df = spark.createDataFrame(data,["features"])
     >>> pca = PCA(k=2, inputCol="features", outputCol="pca_features")
     >>> model = pca.fit(df)
+    >>> estimator_paramMap = pca.extractParamMap()
+    >>> model_paramMap = model.extractParamMap()
+    >>> all([estimator_paramMap[getattr(pca, param.name)] == value
+    ...     for param, value in model_paramMap.items()])
+    True
+    >>> all([param.parent == model.uid for param in model_paramMap])
+    True
+    >>> [param.name for param in model.params]
+    ['inputCol', 'outputCol']
     >>> model.transform(df).collect()[0].pca_features
     DenseVector([1.648..., -4.013...])
     >>> model.explainedVariance
@@ -2394,7 +2415,7 @@ def _create_model(self, java_model):
         return PCAModel(java_model)
 
 
-class PCAModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class PCAModel(JavaModel, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
     """
     Model fitted by :py:class:`PCA`. Transforms vectors to a lower dimensional space.
 
@@ -2437,6 +2458,15 @@ class RFormula(JavaEstimator, HasFeaturesCol, HasLabelCol, JavaMLReadable, JavaM
     ... ], ["y", "x", "s"])
     >>> rf = RFormula(formula="y ~ x + s")
     >>> model = rf.fit(df)
+    >>> estimator_paramMap = rf.extractParamMap()
+    >>> model_paramMap = model.extractParamMap()
+    >>> all([estimator_paramMap[getattr(rf, param.name)] == value
+    ...     for param, value in model_paramMap.items()])
+    True
+    >>> all([param.parent == model.uid for param in model_paramMap])
+    True
+    >>> [param.name for param in model.params]
+    ['featuresCol', 'labelCol']
     >>> model.transform(df).show()
     +---+---+---+---------+-----+
     |  y|  x|  s| features|label|
@@ -2554,7 +2584,7 @@ def __str__(self):
         return "RFormula(%s) (uid=%s)" % (formulaStr, self.uid)
 
 
-class RFormulaModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class RFormulaModel(JavaModel, HasFeaturesCol, HasLabelCol, JavaMLReadable, JavaMLWritable):
     """
     .. note:: Experimental
 
@@ -2586,6 +2616,15 @@ class ChiSqSelector(JavaEstimator, HasFeaturesCol, HasOutputCol, HasLabelCol, Ja
     ...    ["features", "label"])
     >>> selector = ChiSqSelector(numTopFeatures=1, outputCol="selectedFeatures")
     >>> model = selector.fit(df)
+    >>> estimator_paramMap = selector.extractParamMap()
+    >>> model_paramMap = model.extractParamMap()
+    >>> all([estimator_paramMap[getattr(selector, param.name)] == value
+    ...     for param, value in model_paramMap.items()])
+    True
+    >>> all([param.parent == model.uid for param in model_paramMap])
+    True
+    >>> [param.name for param in model.params]
+    ['featuresCol', 'labelCol', 'outputCol']
     >>> model.transform(df).head().selectedFeatures
     DenseVector([18.0])
     >>> model.selectedFeatures
@@ -2710,7 +2749,8 @@ def _create_model(self, java_model):
         return ChiSqSelectorModel(java_model)
 
 
-class ChiSqSelectorModel(JavaModel, JavaMLReadable, JavaMLWritable):
+class ChiSqSelectorModel(JavaModel, HasFeaturesCol, HasOutputCol, HasLabelCol,
+                         JavaMLReadable, JavaMLWritable):
     """
     .. note:: Experimental
 

python/pyspark/ml/param/__init__.py

@@ -336,6 +336,11 @@ def hasParam(self, paramName):
             return isinstance(p, Param)
         else:
             raise TypeError("hasParam(): paramName must be a string")
+        try:
+            param = self._resolveParam(paramName)
+            return param in self.params
+        except:
+            return False
 
     @since("1.4.0")
     def getOrDefault(self, param):

python/pyspark/ml/recommendation.py

@@ -26,8 +26,8 @@
 
 
 @inherit_doc
-class ALS(JavaEstimator, HasCheckpointInterval, HasMaxIter, HasPredictionCol, HasRegParam, HasSeed,
-          JavaMLWritable, JavaMLReadable):
+class ALS(JavaEstimator, HasCheckpointInterval, HasMaxIter, HasPredictionCol,
+          HasRegParam, HasSeed, JavaMLWritable, JavaMLReadable):
     """
     Alternating Least Squares (ALS) matrix factorization.
 
@@ -333,7 +333,7 @@ def getFinalStorageLevel(self):
         return self.getOrDefault(self.finalStorageLevel)
 
 
-class ALSModel(JavaModel, JavaMLWritable, JavaMLReadable):
+class ALSModel(JavaModel, HasPredictionCol, JavaMLWritable, JavaMLReadable):
     """
     Model fitted by ALS.