From b8672084b69a550e13f2cac2941a02a60fac8482 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 29 Apr 2019 21:50:51 -0700 Subject: [PATCH] Revert "[SPARK-24601][SPARK-27051][BACKPORT][CORE] Update to Jackson 2.9.8" This reverts commit 6f394a20bf49f67b4d6329a1c25171c8024a2fae. --- core/pom.xml | 4 --- .../rest/SubmitRestProtocolMessage.scala | 2 +- .../apache/spark/rdd/RDDOperationScope.scala | 2 +- .../org/apache/spark/status/KVUtils.scala | 2 +- .../status/api/v1/JacksonMessageWriter.scala | 2 +- dev/deps/spark-deps-hadoop-2.6 | 16 ++++----- dev/deps/spark-deps-hadoop-2.7 | 16 ++++----- dev/deps/spark-deps-hadoop-3.1 | 16 ++++----- pom.xml | 7 ++-- .../expressions/JsonExpressionsSuite.scala | 7 ---- .../datasources/json/JsonBenchmarks.scala | 33 +++++++++---------- 11 files changed, 47 insertions(+), 60 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 521b9529dda89..59caf2e1ee6ef 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -224,10 +224,6 @@ org.scala-lang scala-library - - org.scala-lang - scala-reflect - org.json4s json4s-jackson_${scala.binary.version} diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala index 97b689cdadd5f..ef5a7e35ad562 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala @@ -36,7 +36,7 @@ import org.apache.spark.util.Utils * (2) the Spark version of the client / server * (3) an optional message */ -@JsonInclude(Include.NON_ABSENT) +@JsonInclude(Include.NON_NULL) @JsonAutoDetect(getterVisibility = Visibility.ANY, setterVisibility = Visibility.ANY) @JsonPropertyOrder(alphabetic = true) private[rest] abstract class SubmitRestProtocolMessage { diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala index 3abb2d8a11f35..53d69ba26811f 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala @@ -41,7 +41,7 @@ import org.apache.spark.internal.Logging * There is no particular relationship between an operation scope and a stage or a job. * A scope may live inside one stage (e.g. map) or span across multiple jobs (e.g. take). */ -@JsonInclude(Include.NON_ABSENT) +@JsonInclude(Include.NON_NULL) @JsonPropertyOrder(Array("id", "name", "parent")) private[spark] class RDDOperationScope( val name: String, diff --git a/core/src/main/scala/org/apache/spark/status/KVUtils.scala b/core/src/main/scala/org/apache/spark/status/KVUtils.scala index 45348be5c98b9..99b1843d8e1c0 100644 --- a/core/src/main/scala/org/apache/spark/status/KVUtils.scala +++ b/core/src/main/scala/org/apache/spark/status/KVUtils.scala @@ -42,7 +42,7 @@ private[spark] object KVUtils extends Logging { private[spark] class KVStoreScalaSerializer extends KVStoreSerializer { mapper.registerModule(DefaultScalaModule) - mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT) + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) } diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala index 50a286d0d3b0f..4560d300cb0c8 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala @@ -49,7 +49,7 @@ private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{ } mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule) mapper.enable(SerializationFeature.INDENT_OUTPUT) - mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT) + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat) override def isWriteable( diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index e7fdbf998ab45..ca13c9691b341 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -86,16 +86,16 @@ htrace-core-3.0.4.jar httpclient-4.5.6.jar httpcore-4.4.10.jar ivy-2.4.0.jar -jackson-annotations-2.9.8.jar -jackson-core-2.9.8.jar +jackson-annotations-2.6.7.jar +jackson-core-2.6.7.jar jackson-core-asl-1.9.13.jar -jackson-databind-2.9.8.jar -jackson-dataformat-yaml-2.9.8.jar +jackson-databind-2.6.7.1.jar +jackson-dataformat-yaml-2.6.7.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations-2.9.8.jar -jackson-module-paranamer-2.9.8.jar -jackson-module-scala_2.11-2.9.8.jar +jackson-module-jaxb-annotations-2.6.7.jar +jackson-module-paranamer-2.7.9.jar +jackson-module-scala_2.11-2.6.7.1.jar jackson-xc-1.9.13.jar janino-3.0.9.jar javassist-3.18.1-GA.jar @@ -178,7 +178,7 @@ shapeless_2.11-2.3.2.jar shims-0.7.45.jar slf4j-api-1.7.16.jar slf4j-log4j12-1.7.16.jar -snakeyaml-1.23.jar +snakeyaml-1.15.jar snappy-0.2.jar snappy-java-1.1.7.3.jar spire-macros_2.11-0.13.0.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 91fe6e8e6846b..95b778f711760 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -86,16 +86,16 @@ htrace-core-3.1.0-incubating.jar httpclient-4.5.6.jar httpcore-4.4.10.jar ivy-2.4.0.jar -jackson-annotations-2.9.8.jar -jackson-core-2.9.8.jar +jackson-annotations-2.6.7.jar +jackson-core-2.6.7.jar jackson-core-asl-1.9.13.jar -jackson-databind-2.9.8.jar -jackson-dataformat-yaml-2.9.8.jar +jackson-databind-2.6.7.1.jar +jackson-dataformat-yaml-2.6.7.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations-2.9.8.jar -jackson-module-paranamer-2.9.8.jar -jackson-module-scala_2.11-2.9.8.jar +jackson-module-jaxb-annotations-2.6.7.jar +jackson-module-paranamer-2.7.9.jar +jackson-module-scala_2.11-2.6.7.1.jar jackson-xc-1.9.13.jar janino-3.0.9.jar javassist-3.18.1-GA.jar @@ -179,7 +179,7 @@ shapeless_2.11-2.3.2.jar shims-0.7.45.jar slf4j-api-1.7.16.jar slf4j-log4j12-1.7.16.jar -snakeyaml-1.23.jar +snakeyaml-1.15.jar snappy-0.2.jar snappy-java-1.1.7.3.jar spire-macros_2.11-0.13.0.jar diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index 3c0137254820d..9b2cb8d93499c 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -86,17 +86,17 @@ htrace-core4-4.1.0-incubating.jar httpclient-4.5.6.jar httpcore-4.4.10.jar ivy-2.4.0.jar -jackson-annotations-2.9.8.jar -jackson-core-2.9.8.jar +jackson-annotations-2.6.7.jar +jackson-core-2.6.7.jar jackson-core-asl-1.9.13.jar -jackson-databind-2.9.8.jar -jackson-dataformat-yaml-2.9.8.jar +jackson-databind-2.6.7.1.jar +jackson-dataformat-yaml-2.6.7.jar jackson-jaxrs-base-2.7.8.jar jackson-jaxrs-json-provider-2.7.8.jar jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations-2.9.8.jar -jackson-module-paranamer-2.9.8.jar -jackson-module-scala_2.11-2.9.8.jar +jackson-module-jaxb-annotations-2.6.7.jar +jackson-module-paranamer-2.7.9.jar +jackson-module-scala_2.11-2.6.7.1.jar janino-3.0.9.jar javassist-3.18.1-GA.jar javax.annotation-api-1.2.jar @@ -199,7 +199,7 @@ shapeless_2.11-2.3.2.jar shims-0.7.45.jar slf4j-api-1.7.16.jar slf4j-log4j12-1.7.16.jar -snakeyaml-1.23.jar +snakeyaml-1.15.jar snappy-0.2.jar snappy-java-1.1.7.3.jar spire-macros_2.11-0.13.0.jar diff --git a/pom.xml b/pom.xml index 0c2666fa343fa..704e258482aba 100644 --- a/pom.xml +++ b/pom.xml @@ -158,7 +158,8 @@ 2.11.12 2.11 1.9.13 - 2.9.8 + 2.6.7 + 2.6.7.1 1.1.7.3 1.1.2 1.2.0-incubating @@ -633,7 +634,7 @@ com.fasterxml.jackson.core jackson-databind - ${fasterxml.jackson.version} + ${fasterxml.jackson.databind.version} com.fasterxml.jackson.core @@ -645,7 +646,7 @@ com.fasterxml.jackson.module jackson-module-scala_${scala.binary.version} - ${fasterxml.jackson.version} + ${fasterxml.jackson.databind.version} com.google.guava diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index c3ea57a4523cf..0e9c8abec33e4 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -244,13 +244,6 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with "1234") } - test("some big value") { - val value = "x" * 3000 - checkEvaluation( - GetJsonObject(NonFoldableLiteral((s"""{"big": "$value"}""")), - NonFoldableLiteral("$.big")), value) - } - val jsonTupleQuery = Literal("f1") :: Literal("f2") :: Literal("f3") :: diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala index 50fe7a38480ac..5592aa6e3346b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala @@ -72,13 +72,12 @@ object JSONBenchmarks { } /* - Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5 - Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz + Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz - JSON schema inferring: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative - --------------------------------------------------------------------------------------------- - No encoding 45908 / 46480 2.2 459.1 1.0X - UTF-8 is set 68469 / 69762 1.5 684.7 0.7X + JSON schema inferring: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative + -------------------------------------------------------------------------------------------- + No encoding 38902 / 39282 2.6 389.0 1.0X + UTF-8 is set 56959 / 57261 1.8 569.6 0.7X */ benchmark.run() } @@ -114,13 +113,12 @@ object JSONBenchmarks { } /* - Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5 - Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz + Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz - JSON per-line parsing: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative - --------------------------------------------------------------------------------------------- - No encoding 9982 / 10237 10.0 99.8 1.0X - UTF-8 is set 16373 / 16806 6.1 163.7 0.6X + JSON per-line parsing: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative + -------------------------------------------------------------------------------------------- + No encoding 25947 / 26188 3.9 259.5 1.0X + UTF-8 is set 46319 / 46417 2.2 463.2 0.6X */ benchmark.run() } @@ -163,13 +161,12 @@ object JSONBenchmarks { } /* - Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5 - Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz + Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz - JSON parsing of wide lines: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative - --------------------------------------------------------------------------------------------- - No encoding 26038 / 26386 0.4 2603.8 1.0X - UTF-8 is set 28343 / 28557 0.4 2834.3 0.9X + JSON parsing of wide lines: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative + -------------------------------------------------------------------------------------------- + No encoding 45543 / 45660 0.2 4554.3 1.0X + UTF-8 is set 65737 / 65957 0.2 6573.7 0.7X */ benchmark.run() }