Skip to content

Commit f99c1e1

Browse files
committed
Revert lineSepInWrite back to lineSep
1 parent b2020fa commit f99c1e1

File tree

2 files changed

+5
-3
lines changed
  • sql

2 files changed

+5
-3
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,9 @@ private[sql] class JSONOptions(
113113
val lineSeparatorInRead: Option[Array[Byte]] = lineSeparator
114114

115115
// Note that JSON uses writer with UTF-8 charset. This string will be written out as UTF-8.
116-
val lineSeparatorInWrite: String = parameters.get("lineSepInWrite").getOrElse("\n")
116+
val lineSeparatorInWrite: String = {
117+
lineSeparator.map(new String(_, charset.getOrElse("UTF-8"))).getOrElse("\n")
118+
}
117119

118120
/** Sets config options on a Jackson [[JsonFactory]]. */
119121
def setJacksonOptions(factory: JsonFactory): Unit = {

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2131,7 +2131,7 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData {
21312131
// Write
21322132
withTempPath { path =>
21332133
Seq("a", "b", "c").toDF("value").coalesce(1)
2134-
.write.option("lineSepInWrite", lineSep).json(path.getAbsolutePath)
2134+
.write.option("lineSep", lineSep).json(path.getAbsolutePath)
21352135
val partFile = TestUtils.recursiveList(path).filter(f => f.getName.startsWith("part-")).head
21362136
val readBack = new String(Files.readAllBytes(partFile.toPath), StandardCharsets.UTF_8)
21372137
assert(
@@ -2141,7 +2141,7 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData {
21412141
// Roundtrip
21422142
withTempPath { path =>
21432143
val df = Seq("a", "b", "c").toDF()
2144-
df.write.option("lineSepInWrite", lineSep).json(path.getAbsolutePath)
2144+
df.write.option("lineSep", lineSep).json(path.getAbsolutePath)
21452145
val readBack = spark.read.option("lineSep", lineSep).json(path.getAbsolutePath)
21462146
checkAnswer(df, readBack)
21472147
}

0 commit comments

Comments
 (0)