File tree Expand file tree Collapse file tree 2 files changed +27
-2
lines changed
main/scala/org/apache/spark/sql/execution/datasources/json
test/scala/org/apache/spark/sql/execution/datasources/json Expand file tree Collapse file tree 2 files changed +27
-2
lines changed Original file line number Diff line number Diff line change @@ -101,7 +101,22 @@ object JacksonParser {
101101 parser.getFloatValue
102102
103103 case (VALUE_STRING , FloatType ) =>
104- parser.getFloatValue
104+ // Special case handling for quoted non-numeric numbers.
105+ if (configOptions.allowNonNumericNumbers) {
106+ val value = parser.getText
107+ val lowerCaseValue = value.toLowerCase()
108+ if (lowerCaseValue.equals(" nan" ) ||
109+ lowerCaseValue.equals(" infinity" ) ||
110+ lowerCaseValue.equals(" -infinity" ) ||
111+ lowerCaseValue.equals(" inf" ) ||
112+ lowerCaseValue.equals(" -inf" )) {
113+ value.toFloat
114+ } else {
115+ sys.error(s " Cannot parse $value as FloatType. " )
116+ }
117+ } else {
118+ parser.getFloatValue
119+ }
105120
106121 case (VALUE_NUMBER_INT | VALUE_NUMBER_FLOAT , DoubleType ) =>
107122 parser.getDoubleValue
Original file line number Diff line number Diff line change @@ -94,7 +94,7 @@ class JsonParsingOptionsSuite extends QueryTest with SharedSQLContext {
9494 }
9595
9696 test(" allowNonNumericNumbers off" ) {
97- val testCases : Seq [String ] = Seq (""" {"age": NaN}""" , """ {"age": Infinity}""" ,
97+ var testCases : Seq [String ] = Seq (""" {"age": NaN}""" , """ {"age": Infinity}""" ,
9898 """ {"age": -Infinity}""" )
9999
100100 testCases.foreach { str =>
@@ -103,6 +103,16 @@ class JsonParsingOptionsSuite extends QueryTest with SharedSQLContext {
103103
104104 assert(df.schema.head.name == " _corrupt_record" )
105105 }
106+
107+ testCases = Seq (""" {"age": "NaN"}""" , """ {"age": "Infinity"}""" ,
108+ """ {"age": "-Infinity"}""" )
109+
110+ testCases.foreach { str =>
111+ val rdd = sqlContext.sparkContext.parallelize(Seq (str))
112+ val df = sqlContext.read.option(" allowNonNumericNumbers" , " false" ).json(rdd)
113+
114+ assert(df.schema.head.name == " age" )
115+ }
106116 }
107117
108118 test(" allowNonNumericNumbers on" ) {
You can’t perform that action at this time.
0 commit comments