address feedback

Author: burakkose

mllib/src/main/scala/org/apache/spark/ml/feature/StopWordsRemover.scala

@@ -49,7 +49,7 @@ class StopWordsRemover(override val uid: String)
   /**
    * The words to be filtered out.
    * Default: English stop words
-   * @see [[StopWordsRemover.loadStopWords()]]
+   * @see [[StopWordsRemover.loadDefaultStopWords()]]
    * @group param
    */
   val stopWords: StringArrayParam =
@@ -89,7 +89,7 @@ class StopWordsRemover(override val uid: String)
   /** @group getParam */
   def getLocale: String = $(locale)
 
-  setDefault(stopWords -> StopWordsRemover.loadStopWords("english"),
+  setDefault(stopWords -> StopWordsRemover.loadDefaultStopWords("english"),
     caseSensitive -> false, locale -> "en")
 
   @Since("2.0.0")
@@ -123,20 +123,21 @@ object StopWordsRemover extends DefaultParamsReadable[StopWordsRemover] {
 
   private def loadLocale(value : String) = new Locale(value)
 
-  private val supportedLanguages = Set("danish", "dutch", "english", "finnish", "french", "german",
+  private[feature]
+  val supportedLanguages = Set("danish", "dutch", "english", "finnish", "french", "german",
     "hungarian", "italian", "norwegian", "portuguese", "russian", "spanish", "swedish", "turkish")
 
   @Since("1.6.0")
   override def load(path: String): StopWordsRemover = super.load(path)
 
   /**
-   * Load stop words for the language
+   * Loads the default stop words for the given language.
    * Supported languages: danish, dutch, english, finnish, french, german, hungarian,
    * italian, norwegian, portuguese, russian, spanish, swedish, turkish
    * @see [[http://anoncvs.postgresql.org/cvsweb.cgi/pgsql/src/backend/snowball/stopwords/]]
    */
   @Since("2.0.0")
-  def loadStopWords(language: String): Array[String] = {
+  def loadDefaultStopWords(language: String): Array[String] = {
     require(supportedLanguages.contains(language),
       s"$language is not in the supported language list: ${supportedLanguages.mkString(", ")}.")
     val is = getClass.getResourceAsStream(s"/org/apache/spark/ml/feature/stopwords/$language.txt")

mllib/src/test/scala/org/apache/spark/ml/feature/StopWordsRemoverSuite.scala

@@ -85,8 +85,15 @@ class StopWordsRemoverSuite
     testStopWordsRemover(remover, dataSet)
   }
 
+  test("default stop words of supported languages are not empty") {
+    StopWordsRemover.supportedLanguages.foreach { lang =>
+      assert(StopWordsRemover.loadDefaultStopWords(lang).nonEmpty,
+        s"The default stop words of $lang cannot be empty.")
+    }
+  }
+
   test("StopWordsRemover with language selection") {
-    val stopWords = StopWordsRemover.loadStopWords("turkish")
+    val stopWords = StopWordsRemover.loadDefaultStopWords("turkish")
     val remover = new StopWordsRemover()
       .setInputCol("raw")
       .setOutputCol("filtered")
@@ -100,7 +107,7 @@ class StopWordsRemoverSuite
   }
 
   test("StopWordsRemover with ignored words") {
-    val stopWords = StopWordsRemover.loadStopWords("english").toSet -- Set("a")
+    val stopWords = StopWordsRemover.loadDefaultStopWords("english").toSet -- Set("a")
     val remover = new StopWordsRemover()
       .setInputCol("raw")
       .setOutputCol("filtered")
@@ -114,7 +121,7 @@ class StopWordsRemoverSuite
   }
 
   test("StopWordsRemover with additional words") {
-    val stopWords = StopWordsRemover.loadStopWords("english").toSet ++ Set("python", "scala")
+    val stopWords = StopWordsRemover.loadDefaultStopWords("english").toSet ++ Set("python", "scala")
     val remover = new StopWordsRemover()
       .setInputCol("raw")
       .setOutputCol("filtered")

python/pyspark/ml/feature.py

@@ -1743,13 +1743,13 @@ class StopWordsRemover(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadabl
     def __init__(self, inputCol=None, outputCol=None, stopWords=None,
                  caseSensitive=False, locale="en"):
         """
-        __init__(self, inputCol=None, outputCol=None, stopWords=None,\
-                 caseSensitive=false, locale="en")
+        __init__(self, inputCol=None, outputCol=None, stopWords=None, \
+                caseSensitive=false, locale="en")
         """
         super(StopWordsRemover, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StopWordsRemover",
                                             self.uid)
-        self._setDefault(stopWords=StopWordsRemover.loadStopWords("english"),
+        self._setDefault(stopWords=StopWordsRemover.loadDefaultStopWords("english"),
                          caseSensitive=False, locale="en")
         kwargs = self.__init__._input_kwargs
         self.setParams(**kwargs)
@@ -1759,8 +1759,8 @@ def __init__(self, inputCol=None, outputCol=None, stopWords=None,
     def setParams(self, inputCol=None, outputCol=None, stopWords=None,
                   caseSensitive=False, locale="en"):
         """
-        setParams(self, inputCol="input", outputCol="output", stopWords=None,
-                  caseSensitive=false, locale="en")
+        setParams(self, inputCol="input", outputCol="output", stopWords=None, \
+                 caseSensitive=false, locale="en")
         Sets params for this StopWordRemover.
         """
         kwargs = self.setParams._input_kwargs
@@ -1769,56 +1769,56 @@ def setParams(self, inputCol=None, outputCol=None, stopWords=None,
     @since("1.6.0")
     def setStopWords(self, value):
         """
-        Specify the stopwords to be filtered.
+        Sets the value of :py:attr:`stopWords`.
         """
         return self._set(stopWords=value)
 
     @since("1.6.0")
     def getStopWords(self):
         """
-        Get the stopwords.
+        Gets the value of :py:attr:`stopWords` or its default value.
         """
         return self.getOrDefault(self.stopWords)
 
     @since("1.6.0")
     def setCaseSensitive(self, value):
         """
-        Set whether to do a case sensitive comparison over the stop words
+        Sets the value of :py:attr:`caseSensitive`.
         """
         return self._set(caseSensitive=value)
 
     @since("1.6.0")
     def getCaseSensitive(self):
         """
-        Get whether to do a case sensitive comparison over the stop words.
+        Gets the value of :py:attr:`caseSensitive` or its default value.
         """
         return self.getOrDefault(self.caseSensitive)
 
     @since("2.0.0")
     def setLocale(self, value):
         """
-        Set locale for doing a case sensitive comparison
+        Sets the value of :py:attr:`locale`.
         """
         self._set(caseSensitive=value)
         return self
 
     @since("2.0.0")
     def getLocale(self):
         """
-        Get locale for doing a case sensitive comparison
+        Gets the value of :py:attr:`locale`.
         """
         return self.getOrDefault(self.caseSensitive)
 
     @staticmethod
     @since("2.0.0")
-    def loadStopWords(language):
+    def loadDefaultStopWords(language):
         """
-        Load stop words for the language
+        Loads the default stop words for the given language.
         Supported languages: danish, dutch, english, finnish, french, german, hungarian,
         italian, norwegian, portuguese, russian, spanish, swedish, turkish
         """
         stopWordsObj = _jvm().org.apache.spark.ml.feature.StopWordsRemover
-        return list(stopWordsObj.loadStopWords(language))
+        return list(stopWordsObj.loadDefaultStopWords(language))
 
 
 @inherit_doc

python/pyspark/ml/tests.py

@@ -410,15 +410,15 @@ def test_stopwordsremover(self):
         self.assertEqual(transformedDF.head().output, ["panda"])
         self.assertEqual(type(stopWordRemover.getStopWords()), list)
         self.assertTrue(isinstance(stopWordRemover.getStopWords()[0], basestring))
-        # with particular stop words list
+        # Custom
         stopwords = ["panda"]
         stopWordRemover.setStopWords(stopwords)
         self.assertEqual(stopWordRemover.getInputCol(), "input")
         self.assertEqual(stopWordRemover.getStopWords(), stopwords)
         transformedDF = stopWordRemover.transform(dataset)
         self.assertEqual(transformedDF.head().output, ["a"])
         # with language selection
-        stopwords = StopWordsRemover.loadStopWords("turkish")
+        stopwords = StopWordsRemover.loadDefaultStopWords("turkish")
         dataset = sqlContext.createDataFrame([Row(input=["acaba", "ama", "biri"])])
         stopWordRemover.setStopWords(stopwords)
         self.assertEqual(stopWordRemover.getStopWords(), stopwords)