From 4693801be78418f1939e07a0d8dc482e3e1efda2 Mon Sep 17 00:00:00 2001 From: alexmnyc Date: Tue, 12 Sep 2017 14:49:40 -0400 Subject: [PATCH 1/5] Upgrade codahale metrics library so that Graphite constructor can re-resolve hosts behind a CNAME with re-tried DNS lookups Upgrade codahale metrics library so that Graphite constructor can re-resolve hosts behind a CNAME with re-tried DNS lookups. When Graphite is deployed behind an ELB, ELB may change IP addresses based on auto-scaling needs. Using current approach yields Graphite usage impossible, fixing for that use case --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a051fea77b6b8..26d1c174f8b31 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 0.8.4 2.4.0 2.0.8 - 3.1.2 + 3.1.5 1.7.7 hadoop2 0.9.3 From 5e2daadaa65d4e5f1988937fed050ee7905a42ea Mon Sep 17 00:00:00 2001 From: alexmnyc Date: Tue, 12 Sep 2017 14:52:41 -0400 Subject: [PATCH 2/5] Force coda-hale Graphite re-try DNS resolution for Graphite instances behind auto-scaled load balancers This fix is introduced in codahale 3.1.5 - https://github.com/dropwizard/metrics/compare/v3.1.2...v3.1.5#diff-6916c85d2dd08d89fe771c952e3b8512R120 --- .../main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala index 23e31823f4930..fe85c37f60b8f 100644 --- a/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala +++ b/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala @@ -69,7 +69,7 @@ private[spark] class GraphiteSink(val property: Properties, val registry: Metric val graphite = propertyToOption(GRAPHITE_KEY_PROTOCOL).map(_.toLowerCase(Locale.ROOT)) match { case Some("udp") => new GraphiteUDP(new InetSocketAddress(host, port)) - case Some("tcp") | None => new Graphite(new InetSocketAddress(host, port)) + case Some("tcp") | None => new Graphite(host, port) case Some(p) => throw new Exception(s"Invalid Graphite protocol: $p") } From cf4157789e7c49e930b5f7dfa46ed258a3f644e7 Mon Sep 17 00:00:00 2001 From: alexmnyc Date: Wed, 13 Sep 2017 10:32:11 -0400 Subject: [PATCH 3/5] Updating metrics 3.1.5 dev dependencies --- dev/deps/spark-deps-hadoop-2.7 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index d39747e9ee058..02c5a19d173be 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -140,10 +140,10 @@ machinist_2.11-0.6.1.jar macro-compat_2.11-1.1.1.jar mail-1.4.7.jar mesos-1.3.0-shaded-protobuf.jar -metrics-core-3.1.2.jar -metrics-graphite-3.1.2.jar -metrics-json-3.1.2.jar -metrics-jvm-3.1.2.jar +metrics-core-3.1.5.jar +metrics-graphite-3.1.5.jar +metrics-json-3.1.5.jar +metrics-jvm-3.1.5.jar minlog-1.3.0.jar mx4j-3.0.2.jar netty-3.9.9.Final.jar From 8e982c7d450498580ab857baeed2650488ea1837 Mon Sep 17 00:00:00 2001 From: alexmnyc Date: Wed, 13 Sep 2017 10:33:00 -0400 Subject: [PATCH 4/5] Updating metrics 3.1.5 dev dependencies --- dev/deps/spark-deps-hadoop-2.6 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 9ac753861dd84..e534e38213fb1 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -139,10 +139,10 @@ machinist_2.11-0.6.1.jar macro-compat_2.11-1.1.1.jar mail-1.4.7.jar mesos-1.3.0-shaded-protobuf.jar -metrics-core-3.1.2.jar -metrics-graphite-3.1.2.jar -metrics-json-3.1.2.jar -metrics-jvm-3.1.2.jar +metrics-core-3.1.5.jar +metrics-graphite-3.1.5.jar +metrics-json-3.1.5.jar +metrics-jvm-3.1.5.jar minlog-1.3.0.jar mx4j-3.0.2.jar netty-3.9.9.Final.jar From 0458123c1b3827f8b4b55eeb8bd5f7dbc749a4aa Mon Sep 17 00:00:00 2001 From: alexmnyc Date: Thu, 14 Sep 2017 21:18:51 -0400 Subject: [PATCH 5/5] Adding dynamic dns resolution for UDP configurations in GraphiteSink --- .../main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala index fe85c37f60b8f..ac33e68abb490 100644 --- a/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala +++ b/core/src/main/scala/org/apache/spark/metrics/sink/GraphiteSink.scala @@ -68,7 +68,7 @@ private[spark] class GraphiteSink(val property: Properties, val registry: Metric MetricsSystem.checkMinimalPollingPeriod(pollUnit, pollPeriod) val graphite = propertyToOption(GRAPHITE_KEY_PROTOCOL).map(_.toLowerCase(Locale.ROOT)) match { - case Some("udp") => new GraphiteUDP(new InetSocketAddress(host, port)) + case Some("udp") => new GraphiteUDP(host, port) case Some("tcp") | None => new Graphite(host, port) case Some(p) => throw new Exception(s"Invalid Graphite protocol: $p") }