Skip to content

Commit 6d90b24

Browse files
committed
For comments.
1 parent dc9abc3 commit 6d90b24

File tree

2 files changed

+27
-2
lines changed

2 files changed

+27
-2
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JacksonParser.scala

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,22 @@ object JacksonParser {
101101
parser.getFloatValue
102102

103103
case (VALUE_STRING, FloatType) =>
104-
parser.getFloatValue
104+
// Special case handling for quoted non-numeric numbers.
105+
if (configOptions.allowNonNumericNumbers) {
106+
val value = parser.getText
107+
val lowerCaseValue = value.toLowerCase()
108+
if (lowerCaseValue.equals("nan") ||
109+
lowerCaseValue.equals("infinity") ||
110+
lowerCaseValue.equals("-infinity") ||
111+
lowerCaseValue.equals("inf") ||
112+
lowerCaseValue.equals("-inf")) {
113+
value.toFloat
114+
} else {
115+
sys.error(s"Cannot parse $value as FloatType.")
116+
}
117+
} else {
118+
parser.getFloatValue
119+
}
105120

106121
case (VALUE_NUMBER_INT | VALUE_NUMBER_FLOAT, DoubleType) =>
107122
parser.getDoubleValue

sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ class JsonParsingOptionsSuite extends QueryTest with SharedSQLContext {
9494
}
9595

9696
test("allowNonNumericNumbers off") {
97-
val testCases: Seq[String] = Seq("""{"age": NaN}""", """{"age": Infinity}""",
97+
var testCases: Seq[String] = Seq("""{"age": NaN}""", """{"age": Infinity}""",
9898
"""{"age": -Infinity}""")
9999

100100
testCases.foreach { str =>
@@ -103,6 +103,16 @@ class JsonParsingOptionsSuite extends QueryTest with SharedSQLContext {
103103

104104
assert(df.schema.head.name == "_corrupt_record")
105105
}
106+
107+
testCases = Seq("""{"age": "NaN"}""", """{"age": "Infinity"}""",
108+
"""{"age": "-Infinity"}""")
109+
110+
testCases.foreach { str =>
111+
val rdd = sqlContext.sparkContext.parallelize(Seq(str))
112+
val df = sqlContext.read.option("allowNonNumericNumbers", "false").json(rdd)
113+
114+
assert(df.schema.head.name == "age")
115+
}
106116
}
107117

108118
test("allowNonNumericNumbers on") {

0 commit comments

Comments
 (0)