To match Tokenizer and for compatibility with Word2Vec, output a nullable string array type in NGram

srowen · srowen · commit 0c7e552bae0f · 2016-09-21T09:46:58.000+01:00
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/NGram.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/NGram.scala
@@ -68,7 +68,7 @@ class NGram @Since("1.5.0") (@Since("1.5.0") override val uid: String)
       s"Input type must be ArrayType(StringType) but got $inputType.")
   }
 
-  override protected def outputDataType: DataType = new ArrayType(StringType, false)
+  override protected def outputDataType: DataType = new ArrayType(StringType, true)
 }
 
 @Since("1.6.0")

Original file line number	Diff line number	Diff line change
`@@ -68,7 +68,7 @@ class NGram @Since("1.5.0") (@Since("1.5.0") override val uid: String)`
`68`	`68`	`s"Input type must be ArrayType(StringType) but got $inputType.")`
`69`	`69`	`}`
`70`	`70`
`71`		`- override protected def outputDataType: DataType = new ArrayType(StringType, false)`
	`71`	`+ override protected def outputDataType: DataType = new ArrayType(StringType, true)`
`72`	`72`	`}`
`73`	`73`
`74`	`74`	`@Since("1.6.0")`