Support encoding for lineSep

MaxGekk · MaxGekk · commit 1f5399f32a45 · 2018-11-21T12:09:16.000+01:00
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
@@ -202,7 +202,7 @@ class CSVOptions(
   }
 
   val lineSeparatorInRead: Option[Array[Byte]] = lineSeparator.map { lineSep =>
-    lineSep.getBytes("UTF-8")
+    lineSep.getBytes(charset)
   }
   val lineSeparatorInWrite: Option[String] = lineSeparator
 
@@ -232,9 +232,9 @@ class CSVOptions(
     format.setDelimiter(delimiter)
     format.setQuote(quote)
     format.setQuoteEscape(escape)
-    lineSeparatorInRead.foreach {sep =>
-      format.setLineSeparator(sep.map(_.toChar))
-      format.setNormalizedNewline(sep.head.toChar)
+    lineSeparator.foreach {sep =>
+      format.setLineSeparator(sep)
+      format.setNormalizedNewline(0x00.toChar)
     }
     charToEscapeQuoteEscaping.foreach(format.setCharToEscapeQuoteEscaping)
     format.setComment(comment)

Original file line number	Diff line number	Diff line change
`@@ -202,7 +202,7 @@ class CSVOptions(`
`202`	`202`	`}`
`203`	`203`
`204`	`204`	`val lineSeparatorInRead: Option[Array[Byte]] = lineSeparator.map { lineSep =>`
`205`		`- lineSep.getBytes("UTF-8")`
	`205`	`+ lineSep.getBytes(charset)`
`206`	`206`	`}`
`207`	`207`	`val lineSeparatorInWrite: Option[String] = lineSeparator`
`208`	`208`
`@@ -232,9 +232,9 @@ class CSVOptions(`
`232`	`232`	`format.setDelimiter(delimiter)`
`233`	`233`	`format.setQuote(quote)`
`234`	`234`	`format.setQuoteEscape(escape)`
`235`		`- lineSeparatorInRead.foreach {sep =>`
`236`		`- format.setLineSeparator(sep.map(_.toChar))`
`237`		`- format.setNormalizedNewline(sep.head.toChar)`
	`235`	`+ lineSeparator.foreach {sep =>`
	`236`	`+ format.setLineSeparator(sep)`
	`237`	`+ format.setNormalizedNewline(0x00.toChar)`
`238`	`238`	`}`
`239`	`239`	`charToEscapeQuoteEscaping.foreach(format.setCharToEscapeQuoteEscaping)`
`240`	`240`	`format.setComment(comment)`