address feedback

Author: mengxr

mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala

@@ -47,7 +47,7 @@ class StopWordsRemover(override val uid: String)
   /**
    * The words to be filtered out.
    * Default: English stop words
-   * @see [[StopWordsRemover.loadStopWords()]]
+   * @see [[StopWordsRemover.loadDefaultStopWords()]]
    * @group param
    */
   val stopWords: StringArrayParam =
@@ -65,15 +65,15 @@ class StopWordsRemover(override val uid: String)
    * @group param
    */
   val caseSensitive: BooleanParam = new BooleanParam(this, "caseSensitive",
-    "whether to do a case-sensitive comparison over the stop stop words")
+    "whether to do a case-sensitive comparison over the stop words")
 
   /** @group setParam */
   def setCaseSensitive(value: Boolean): this.type = set(caseSensitive, value)
 
   /** @group getParam */
   def getCaseSensitive: Boolean = $(caseSensitive)
 
-  setDefault(stopWords -> StopWordsRemover.loadStopWords("english"), caseSensitive -> false)
+  setDefault(stopWords -> StopWordsRemover.loadDefaultStopWords("english"), caseSensitive -> false)
 
   @Since("2.0.0")
   override def transform(dataset: Dataset[_]): DataFrame = {
@@ -108,20 +108,21 @@ class StopWordsRemover(override val uid: String)
 @Since("1.6.0")
 object StopWordsRemover extends DefaultParamsReadable[StopWordsRemover] {
 
-  private val supportedLanguages = Set("danish", "dutch", "english", "finnish", "french", "german",
+  private[feature]
+  val supportedLanguages = Set("danish", "dutch", "english", "finnish", "french", "german",
     "hungarian", "italian", "norwegian", "portuguese", "russian", "spanish", "swedish", "turkish")
 
   @Since("1.6.0")
   override def load(path: String): StopWordsRemover = super.load(path)
 
   /**
-   * Load stop words for the language
+   * Loads the default stop words for the given language.
    * Supported languages: danish, dutch, english, finnish, french, german, hungarian,
    * italian, norwegian, portuguese, russian, spanish, swedish, turkish
    * @see [[http://anoncvs.postgresql.org/cvsweb.cgi/pgsql/src/backend/snowball/stopwords/]]
    */
   @Since("2.0.0")
-  def loadStopWords(language: String): Array[String] = {
+  def loadDefaultStopWords(language: String): Array[String] = {
     require(supportedLanguages.contains(language),
       s"$language is not in the supported language list: ${supportedLanguages.mkString(", ")}.")
     val is = getClass.getResourceAsStream(s"/org/apache/spark/ml/feature/stopwords/$language.txt")

mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala

@@ -85,8 +85,15 @@ class StopWordsRemoverSuite
     testStopWordsRemover(remover, dataSet)
   }
 
+  test("default stop words of supported languages are not empty") {
+    StopWordsRemover.supportedLanguages.foreach { lang =>
+      assert(StopWordsRemover.loadDefaultStopWords(lang).nonEmpty,
+        s"The default stop words of $lang cannot be empty.")
+    }
+  }
+
   test("StopWordsRemover with language selection") {
-    val stopWords = StopWordsRemover.loadStopWords("turkish")
+    val stopWords = StopWordsRemover.loadDefaultStopWords("turkish")
     val remover = new StopWordsRemover()
       .setInputCol("raw")
       .setOutputCol("filtered")
@@ -100,7 +107,7 @@ class StopWordsRemoverSuite
   }
 
   test("StopWordsRemover with ignored words") {
-    val stopWords = StopWordsRemover.loadStopWords("english").toSet -- Set("a")
+    val stopWords = StopWordsRemover.loadDefaultStopWords("english").toSet -- Set("a")
     val remover = new StopWordsRemover()
       .setInputCol("raw")
       .setOutputCol("filtered")
@@ -114,7 +121,7 @@ class StopWordsRemoverSuite
   }
 
   test("StopWordsRemover with additional words") {
-    val stopWords = StopWordsRemover.loadStopWords("english").toSet ++ Set("python", "scala")
+    val stopWords = StopWordsRemover.loadDefaultStopWords("english").toSet ++ Set("python", "scala")
     val remover = new StopWordsRemover()
       .setInputCol("raw")
       .setOutputCol("filtered")

python/pyspark/ml/feature.py

@@ -1778,7 +1778,7 @@ def __init__(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive=
     @since("1.6.0")
     def setParams(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive=False):
         """
-        setParams(self, inputCol="input", outputCol="output", stopWords=None, caseSensitive=false)
+        setParams(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive=false)
         Sets params for this StopWordRemover.
         """
         kwargs = self.setParams._input_kwargs
@@ -1787,43 +1787,43 @@ def setParams(self, inputCol=None, outputCol=None, stopWords=None, caseSensitive
     @since("1.6.0")
     def setStopWords(self, value):
         """
-        Specify the stopwords to be filtered.
+        Sets the value of :py:attr:`stopWords`.
         """
         self._set(stopWords=value)
         return self
 
     @since("1.6.0")
     def getStopWords(self):
         """
-        Get the stopwords.
+        Gets the value of :py:attr:`stopWords` or its default value.
         """
         return self.getOrDefault(self.stopWords)
 
     @since("1.6.0")
     def setCaseSensitive(self, value):
         """
-        Set whether to do a case sensitive comparison over the stop words
+        Sets the value of :py:attr:`caseSensitive`.
         """
         self._set(caseSensitive=value)
         return self
 
     @since("1.6.0")
     def getCaseSensitive(self):
         """
-        Get whether to do a case sensitive comparison over the stop words.
+        Gets the value of :py:attr:`caseSensitive` or its default value.
         """
         return self.getOrDefault(self.caseSensitive)
 
     @staticmethod
     @since("2.0.0")
-    def loadStopWords(language):
+    def loadDefaultStopWords(language):
         """
-        Load stop words for the language
+        Loads the default stop words for the given language.
         Supported languages: danish, dutch, english, finnish, french, german, hungarian,
         italian, norwegian, portuguese, russian, spanish, swedish, turkish
         """
         stopWordsObj = _jvm().org.apache.spark.ml.feature.StopWordsRemover
-        return list(stopWordsObj.loadStopWords(language))
+        return list(stopWordsObj.loadDefaultStopWords(language))
 
 
 @inherit_doc
@@ -1875,7 +1875,7 @@ def __init__(self, inputCol=None, outputCol=None):
     @since("1.3.0")
     def setParams(self, inputCol=None, outputCol=None):
         """
-        setParams(self, inputCol="input", outputCol="output")
+        setParams(self, inputCol=None, outputCol=None)
         Sets params for this Tokenizer.
         """
         kwargs = self.setParams._input_kwargs

python/pyspark/ml/tests.py

@@ -418,7 +418,7 @@ def test_stopwordsremover(self):
         transformedDF = stopWordRemover.transform(dataset)
         self.assertEqual(transformedDF.head().output, ["a"])
         # with language selection
-        stopwords = StopWordsRemover.loadStopWords("turkish")
+        stopwords = StopWordsRemover.loadDefaultStopWords("turkish")
         dataset = sqlContext.createDataFrame([Row(input=["acaba", "ama", "biri"])])
         stopWordRemover.setStopWords(stopwords)
         self.assertEqual(stopWordRemover.getStopWords(), stopwords)