Skip to content

Commit ccc69e2

Browse files
Marcelo Vanzinandrewor14
Marcelo Vanzin
authored and
andrewor14
committed
[SPARK-2845] Add timestamps to block manager events.
These are not used by the UI but are useful when analysing the logs from a spark job. Author: Marcelo Vanzin <[email protected]> Closes #654 from vanzin/bm-event-tstamp and squashes the following commits: d5d6e66 [Marcelo Vanzin] Fix tests. ec06218 [Marcelo Vanzin] Review feedback. f134dbc [Marcelo Vanzin] Merge branch 'master' into bm-event-tstamp b495b7c [Marcelo Vanzin] Merge branch 'master' into bm-event-tstamp 7d2fe9e [Marcelo Vanzin] Review feedback. d6f381c [Marcelo Vanzin] Update tests added after patch was created. 45e3bf8 [Marcelo Vanzin] Fix unit test after merge. b37a10f [Marcelo Vanzin] Use === in test assertions. ef72824 [Marcelo Vanzin] Handle backwards compatibility with 1.0.0. aca1151 [Marcelo Vanzin] Fix unit test to check new fields. efdda8e [Marcelo Vanzin] Add timestamps to block manager events.
1 parent e5d3768 commit ccc69e2

File tree

6 files changed

+58
-24
lines changed

6 files changed

+58
-24
lines changed

‎core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,11 @@ case class SparkListenerEnvironmentUpdate(environmentDetails: Map[String, Seq[(S
6767
extends SparkListenerEvent
6868

6969
@DeveloperApi
70-
case class SparkListenerBlockManagerAdded(blockManagerId: BlockManagerId, maxMem: Long)
70+
case class SparkListenerBlockManagerAdded(time: Long, blockManagerId: BlockManagerId, maxMem: Long)
7171
extends SparkListenerEvent
7272

7373
@DeveloperApi
74-
case class SparkListenerBlockManagerRemoved(blockManagerId: BlockManagerId)
74+
case class SparkListenerBlockManagerRemoved(time: Long, blockManagerId: BlockManagerId)
7575
extends SparkListenerEvent
7676

7777
@DeveloperApi

‎core/src/main/scala/org/apache/spark/storage/BlockManagerMasterActor.scala‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ class BlockManagerMasterActor(val isLocal: Boolean, conf: SparkConf, listenerBus
203203
blockLocations.remove(blockId)
204204
}
205205
}
206-
listenerBus.post(SparkListenerBlockManagerRemoved(blockManagerId))
206+
listenerBus.post(SparkListenerBlockManagerRemoved(System.currentTimeMillis(), blockManagerId))
207207
}
208208

209209
private def expireDeadHosts() {
@@ -325,6 +325,7 @@ class BlockManagerMasterActor(val isLocal: Boolean, conf: SparkConf, listenerBus
325325
}
326326

327327
private def register(id: BlockManagerId, maxMemSize: Long, slaveActor: ActorRef) {
328+
val time = System.currentTimeMillis()
328329
if (!blockManagerInfo.contains(id)) {
329330
blockManagerIdByExecutor.get(id.executorId) match {
330331
case Some(manager) =>
@@ -340,9 +341,9 @@ class BlockManagerMasterActor(val isLocal: Boolean, conf: SparkConf, listenerBus
340341
id.hostPort, Utils.bytesToString(maxMemSize)))
341342

342343
blockManagerInfo(id) =
343-
new BlockManagerInfo(id, System.currentTimeMillis(), maxMemSize, slaveActor)
344+
new BlockManagerInfo(id, time, maxMemSize, slaveActor)
344345
}
345-
listenerBus.post(SparkListenerBlockManagerAdded(id, maxMemSize))
346+
listenerBus.post(SparkListenerBlockManagerAdded(time, id, maxMemSize))
346347
}
347348

348349
private def updateBlockInfo(

‎core/src/main/scala/org/apache/spark/util/JsonProtocol.scala‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -152,13 +152,15 @@ private[spark] object JsonProtocol {
152152
val blockManagerId = blockManagerIdToJson(blockManagerAdded.blockManagerId)
153153
("Event" -> Utils.getFormattedClassName(blockManagerAdded)) ~
154154
("Block Manager ID" -> blockManagerId) ~
155-
("Maximum Memory" -> blockManagerAdded.maxMem)
155+
("Maximum Memory" -> blockManagerAdded.maxMem) ~
156+
("Timestamp" -> blockManagerAdded.time)
156157
}
157158

158159
def blockManagerRemovedToJson(blockManagerRemoved: SparkListenerBlockManagerRemoved): JValue = {
159160
val blockManagerId = blockManagerIdToJson(blockManagerRemoved.blockManagerId)
160161
("Event" -> Utils.getFormattedClassName(blockManagerRemoved)) ~
161-
("Block Manager ID" -> blockManagerId)
162+
("Block Manager ID" -> blockManagerId) ~
163+
("Timestamp" -> blockManagerRemoved.time)
162164
}
163165

164166
def unpersistRDDToJson(unpersistRDD: SparkListenerUnpersistRDD): JValue = {
@@ -466,12 +468,14 @@ private[spark] object JsonProtocol {
466468
def blockManagerAddedFromJson(json: JValue): SparkListenerBlockManagerAdded = {
467469
val blockManagerId = blockManagerIdFromJson(json \ "Block Manager ID")
468470
val maxMem = (json \ "Maximum Memory").extract[Long]
469-
SparkListenerBlockManagerAdded(blockManagerId, maxMem)
471+
val time = Utils.jsonOption(json \ "Timestamp").map(_.extract[Long]).getOrElse(-1L)
472+
SparkListenerBlockManagerAdded(time, blockManagerId, maxMem)
470473
}
471474

472475
def blockManagerRemovedFromJson(json: JValue): SparkListenerBlockManagerRemoved = {
473476
val blockManagerId = blockManagerIdFromJson(json \ "Block Manager ID")
474-
SparkListenerBlockManagerRemoved(blockManagerId)
477+
val time = Utils.jsonOption(json \ "Timestamp").map(_.extract[Long]).getOrElse(-1L)
478+
SparkListenerBlockManagerRemoved(time, blockManagerId)
475479
}
476480

477481
def unpersistRDDFromJson(json: JValue): SparkListenerUnpersistRDD = {

‎core/src/test/scala/org/apache/spark/storage/StorageStatusListenerSuite.scala‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,34 +36,34 @@ class StorageStatusListenerSuite extends FunSuite {
3636

3737
// Block manager add
3838
assert(listener.executorIdToStorageStatus.size === 0)
39-
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(bm1, 1000L))
39+
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
4040
assert(listener.executorIdToStorageStatus.size === 1)
4141
assert(listener.executorIdToStorageStatus.get("big").isDefined)
4242
assert(listener.executorIdToStorageStatus("big").blockManagerId === bm1)
4343
assert(listener.executorIdToStorageStatus("big").maxMem === 1000L)
4444
assert(listener.executorIdToStorageStatus("big").numBlocks === 0)
45-
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(bm2, 2000L))
45+
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm2, 2000L))
4646
assert(listener.executorIdToStorageStatus.size === 2)
4747
assert(listener.executorIdToStorageStatus.get("fat").isDefined)
4848
assert(listener.executorIdToStorageStatus("fat").blockManagerId === bm2)
4949
assert(listener.executorIdToStorageStatus("fat").maxMem === 2000L)
5050
assert(listener.executorIdToStorageStatus("fat").numBlocks === 0)
5151

5252
// Block manager remove
53-
listener.onBlockManagerRemoved(SparkListenerBlockManagerRemoved(bm1))
53+
listener.onBlockManagerRemoved(SparkListenerBlockManagerRemoved(1L, bm1))
5454
assert(listener.executorIdToStorageStatus.size === 1)
5555
assert(!listener.executorIdToStorageStatus.get("big").isDefined)
5656
assert(listener.executorIdToStorageStatus.get("fat").isDefined)
57-
listener.onBlockManagerRemoved(SparkListenerBlockManagerRemoved(bm2))
57+
listener.onBlockManagerRemoved(SparkListenerBlockManagerRemoved(1L, bm2))
5858
assert(listener.executorIdToStorageStatus.size === 0)
5959
assert(!listener.executorIdToStorageStatus.get("big").isDefined)
6060
assert(!listener.executorIdToStorageStatus.get("fat").isDefined)
6161
}
6262

6363
test("task end without updated blocks") {
6464
val listener = new StorageStatusListener
65-
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(bm1, 1000L))
66-
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(bm2, 2000L))
65+
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
66+
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm2, 2000L))
6767
val taskMetrics = new TaskMetrics
6868

6969
// Task end with no updated blocks
@@ -79,8 +79,8 @@ class StorageStatusListenerSuite extends FunSuite {
7979

8080
test("task end with updated blocks") {
8181
val listener = new StorageStatusListener
82-
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(bm1, 1000L))
83-
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(bm2, 2000L))
82+
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
83+
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm2, 2000L))
8484
val taskMetrics1 = new TaskMetrics
8585
val taskMetrics2 = new TaskMetrics
8686
val block1 = (RDDBlockId(1, 1), BlockStatus(StorageLevel.DISK_ONLY, 0L, 100L, 0L))
@@ -128,7 +128,7 @@ class StorageStatusListenerSuite extends FunSuite {
128128

129129
test("unpersist RDD") {
130130
val listener = new StorageStatusListener
131-
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(bm1, 1000L))
131+
listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
132132
val taskMetrics1 = new TaskMetrics
133133
val taskMetrics2 = new TaskMetrics
134134
val block1 = (RDDBlockId(1, 1), BlockStatus(StorageLevel.DISK_ONLY, 0L, 100L, 0L))

‎core/src/test/scala/org/apache/spark/ui/storage/StorageTabSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ class StorageTabSuite extends FunSuite with BeforeAndAfter {
108108
val myRddInfo1 = rddInfo1
109109
val myRddInfo2 = rddInfo2
110110
val stageInfo0 = new StageInfo(0, 0, "0", 100, Seq(myRddInfo0, myRddInfo1, myRddInfo2), "details")
111-
bus.postToAll(SparkListenerBlockManagerAdded(bm1, 1000L))
111+
bus.postToAll(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
112112
bus.postToAll(SparkListenerStageSubmitted(stageInfo0))
113113
assert(storageListener._rddInfoMap.size === 3)
114114
assert(storageListener.rddInfoList.size === 0) // not cached
@@ -175,7 +175,7 @@ class StorageTabSuite extends FunSuite with BeforeAndAfter {
175175
val block1 = (RDDBlockId(1, 1), BlockStatus(memOnly, 200L, 0L, 0L))
176176
taskMetrics0.updatedBlocks = Some(Seq(block0))
177177
taskMetrics1.updatedBlocks = Some(Seq(block1))
178-
bus.postToAll(SparkListenerBlockManagerAdded(bm1, 1000L))
178+
bus.postToAll(SparkListenerBlockManagerAdded(1L, bm1, 1000L))
179179
bus.postToAll(SparkListenerStageSubmitted(stageInfo0))
180180
assert(storageListener.rddInfoList.size === 0)
181181
bus.postToAll(SparkListenerTaskEnd(0, 0, "big", Success, taskInfo, taskMetrics0))

‎core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ import java.util.Properties
2121

2222
import scala.collection.Map
2323

24+
import org.json4s.DefaultFormats
25+
import org.json4s.JsonDSL._
26+
import org.json4s.JsonAST._
2427
import org.json4s.jackson.JsonMethods._
2528
import org.scalatest.FunSuite
2629

@@ -52,9 +55,9 @@ class JsonProtocolSuite extends FunSuite {
5255
"System Properties" -> Seq(("Username", "guest"), ("Password", "guest")),
5356
"Classpath Entries" -> Seq(("Super library", "/tmp/super_library"))
5457
))
55-
val blockManagerAdded = SparkListenerBlockManagerAdded(
58+
val blockManagerAdded = SparkListenerBlockManagerAdded(1L,
5659
BlockManagerId("Stars", "In your multitude...", 300), 500)
57-
val blockManagerRemoved = SparkListenerBlockManagerRemoved(
60+
val blockManagerRemoved = SparkListenerBlockManagerRemoved(2L,
5861
BlockManagerId("Scarce", "to be counted...", 100))
5962
val unpersistRdd = SparkListenerUnpersistRDD(12345)
6063
val applicationStart = SparkListenerApplicationStart("The winner of all", 42L, "Garfield")
@@ -151,6 +154,28 @@ class JsonProtocolSuite extends FunSuite {
151154
assert(newMetrics.inputMetrics.isEmpty)
152155
}
153156

157+
test("BlockManager events backward compatibility") {
158+
// SparkListenerBlockManagerAdded/Removed in Spark 1.0.0 do not have a "time" property.
159+
val blockManagerAdded = SparkListenerBlockManagerAdded(1L,
160+
BlockManagerId("Stars", "In your multitude...", 300), 500)
161+
val blockManagerRemoved = SparkListenerBlockManagerRemoved(2L,
162+
BlockManagerId("Scarce", "to be counted...", 100))
163+
164+
val oldBmAdded = JsonProtocol.blockManagerAddedToJson(blockManagerAdded)
165+
.removeField({ _._1 == "Timestamp" })
166+
167+
val deserializedBmAdded = JsonProtocol.blockManagerAddedFromJson(oldBmAdded)
168+
assert(SparkListenerBlockManagerAdded(-1L, blockManagerAdded.blockManagerId,
169+
blockManagerAdded.maxMem) === deserializedBmAdded)
170+
171+
val oldBmRemoved = JsonProtocol.blockManagerRemovedToJson(blockManagerRemoved)
172+
.removeField({ _._1 == "Timestamp" })
173+
174+
val deserializedBmRemoved = JsonProtocol.blockManagerRemovedFromJson(oldBmRemoved)
175+
assert(SparkListenerBlockManagerRemoved(-1L, blockManagerRemoved.blockManagerId) ===
176+
deserializedBmRemoved)
177+
}
178+
154179

155180
/** -------------------------- *
156181
| Helper test running methods |
@@ -242,8 +267,10 @@ class JsonProtocolSuite extends FunSuite {
242267
assertEquals(e1.environmentDetails, e2.environmentDetails)
243268
case (e1: SparkListenerBlockManagerAdded, e2: SparkListenerBlockManagerAdded) =>
244269
assert(e1.maxMem === e2.maxMem)
270+
assert(e1.time === e2.time)
245271
assertEquals(e1.blockManagerId, e2.blockManagerId)
246272
case (e1: SparkListenerBlockManagerRemoved, e2: SparkListenerBlockManagerRemoved) =>
273+
assert(e1.time === e2.time)
247274
assertEquals(e1.blockManagerId, e2.blockManagerId)
248275
case (e1: SparkListenerUnpersistRDD, e2: SparkListenerUnpersistRDD) =>
249276
assert(e1.rddId == e2.rddId)
@@ -945,7 +972,8 @@ class JsonProtocolSuite extends FunSuite {
945972
| "Host": "In your multitude...",
946973
| "Port": 300
947974
| },
948-
| "Maximum Memory": 500
975+
| "Maximum Memory": 500,
976+
| "Timestamp": 1
949977
|}
950978
"""
951979

@@ -957,7 +985,8 @@ class JsonProtocolSuite extends FunSuite {
957985
| "Executor ID": "Scarce",
958986
| "Host": "to be counted...",
959987
| "Port": 100
960-
| }
988+
| },
989+
| "Timestamp": 2
961990
|}
962991
"""
963992

0 commit comments

Comments
 (0)