From 0ae9cc52737c6864d8db95ac2cbfe7b2334c5e5c Mon Sep 17 00:00:00 2001 From: "iurii.ant" Date: Mon, 24 Jul 2017 21:00:52 -0500 Subject: [PATCH] [SPARK-21491][GraphX] Enhance GraphX performance: eliminate intermediate collections creation with breakOut --- .../scala/org/apache/spark/graphx/lib/LabelPropagation.scala | 2 +- .../scala/org/apache/spark/graphx/lib/ShortestPaths.scala | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala index fc7547a2c7c2..cb3025f8bef5 100644 --- a/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala +++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala @@ -55,7 +55,7 @@ object LabelPropagation { val count1Val = count1.getOrElse(i, 0L) val count2Val = count2.getOrElse(i, 0L) i -> (count1Val + count2Val) - }.toMap + }(collection.breakOut) // more efficient alternative to [[collection.Traversable.toMap]] } def vertexProgram(vid: VertexId, attr: Long, message: Map[VertexId, Long]): VertexId = { if (message.isEmpty) attr else message.maxBy(_._2)._1 diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala index f0c6bcb93445..4cac633aed00 100644 --- a/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala +++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala @@ -33,10 +33,11 @@ object ShortestPaths { private def incrementMap(spmap: SPMap): SPMap = spmap.map { case (v, d) => v -> (d + 1) } - private def addMaps(spmap1: SPMap, spmap2: SPMap): SPMap = + private def addMaps(spmap1: SPMap, spmap2: SPMap): SPMap = { (spmap1.keySet ++ spmap2.keySet).map { k => k -> math.min(spmap1.getOrElse(k, Int.MaxValue), spmap2.getOrElse(k, Int.MaxValue)) - }.toMap + }(collection.breakOut) // more efficient alternative to [[collection.Traversable.toMap]] + } /** * Computes shortest paths to the given set of landmark vertices.