From 81f9fd247a4ce8c69636495ade2f130f6fa6aa6f Mon Sep 17 00:00:00 2001 From: xutingjun Date: Tue, 16 Jun 2015 17:09:14 +0800 Subject: [PATCH 1/4] put the filter inside --- core/src/main/scala/org/apache/spark/ui/UIUtils.scala | 2 +- .../org/apache/spark/ui/scope/RDDOperationGraph.scala | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 65162f4fdcd6..baadbbc42a7a 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -362,7 +362,7 @@ private[spark] object UIUtils extends Logging { { g.incomingEdges.map { e =>
{e.fromId},{e.toId}
} } { g.outgoingEdges.map { e =>
{e.fromId},{e.toId}
} } { - g.rootCluster.getAllNodes.filter(_.cached).map { n => + g.rootCluster.getCachedNode.map { n =>
{n.id}
} } diff --git a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala index d6a5085db1ef..b33bf9603e12 100644 --- a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala +++ b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala @@ -70,6 +70,16 @@ private[ui] class RDDOperationCluster(val id: String, private var _name: String) def getAllNodes: Seq[RDDOperationNode] = { _childNodes ++ _childClusters.flatMap(_.childNodes) } + + /** Return all the node which are cached. */ + def getCachedNode: Seq[RDDOperationNode] = { + var cachedNodes = new ListBuffer[RDDOperationNode] + cachedNodes ++= (_childNodes.filter(_.cached)) + for(cluster <- _childClusters) { + cachedNodes ++= (cluster._childNodes.filter(_.cached)) + } + cachedNodes + } } private[ui] object RDDOperationGraph extends Logging { From f87c663816bfada91c422bdc6618a25768a1d66b Mon Sep 17 00:00:00 2001 From: xutingjun Date: Tue, 16 Jun 2015 20:53:00 +0800 Subject: [PATCH 2/4] put the filter inside --- core/src/main/scala/org/apache/spark/ui/UIUtils.scala | 2 +- .../org/apache/spark/ui/scope/RDDOperationGraph.scala | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index baadbbc42a7a..789803951920 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -362,7 +362,7 @@ private[spark] object UIUtils extends Logging { { g.incomingEdges.map { e =>
{e.fromId},{e.toId}
} } { g.outgoingEdges.map { e =>
{e.fromId},{e.toId}
} } { - g.rootCluster.getCachedNode.map { n => + g.rootCluster.getCachedNodes.map { n =>
{n.id}
} } diff --git a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala index b33bf9603e12..554d2ecdfe48 100644 --- a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala +++ b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala @@ -72,12 +72,9 @@ private[ui] class RDDOperationCluster(val id: String, private var _name: String) } /** Return all the node which are cached. */ - def getCachedNode: Seq[RDDOperationNode] = { - var cachedNodes = new ListBuffer[RDDOperationNode] - cachedNodes ++= (_childNodes.filter(_.cached)) - for(cluster <- _childClusters) { - cachedNodes ++= (cluster._childNodes.filter(_.cached)) - } + def getCachedNodes: Seq[RDDOperationNode] = { + val cachedNodes = _childNodes.filter(_.cached) + _childClusters.foreach(cluster => cachedNodes ++= cluster._childNodes.filter(_.cached)) cachedNodes } } From f98728bdbef0d3388f36928dccd573fa15bc6536 Mon Sep 17 00:00:00 2001 From: xutingjun Date: Wed, 17 Jun 2015 09:38:44 +0800 Subject: [PATCH 3/4] fix words: node -> nodes --- .../scala/org/apache/spark/ui/scope/RDDOperationGraph.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala index 554d2ecdfe48..5705aa2f030e 100644 --- a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala +++ b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala @@ -71,7 +71,7 @@ private[ui] class RDDOperationCluster(val id: String, private var _name: String) _childNodes ++ _childClusters.flatMap(_.childNodes) } - /** Return all the node which are cached. */ + /** Return all the nodes which are cached. */ def getCachedNodes: Seq[RDDOperationNode] = { val cachedNodes = _childNodes.filter(_.cached) _childClusters.foreach(cluster => cachedNodes ++= cluster._childNodes.filter(_.cached)) From 53b03ea3f36c0002f5af8424509fef3391f5e003 Mon Sep 17 00:00:00 2001 From: xutingjun Date: Thu, 18 Jun 2015 09:47:00 +0800 Subject: [PATCH 4/4] change code to more concise and easier to read --- .../org/apache/spark/ui/scope/RDDOperationGraph.scala | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala index 5705aa2f030e..ffea9817c0b0 100644 --- a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala +++ b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala @@ -66,16 +66,9 @@ private[ui] class RDDOperationCluster(val id: String, private var _name: String) _childClusters += childCluster } - /** Return all the nodes container in this cluster, including ones nested in other clusters. */ - def getAllNodes: Seq[RDDOperationNode] = { - _childNodes ++ _childClusters.flatMap(_.childNodes) - } - /** Return all the nodes which are cached. */ def getCachedNodes: Seq[RDDOperationNode] = { - val cachedNodes = _childNodes.filter(_.cached) - _childClusters.foreach(cluster => cachedNodes ++= cluster._childNodes.filter(_.cached)) - cachedNodes + _childNodes.filter(_.cached) ++ _childClusters.flatMap(_.getCachedNodes) } }