Skip to content

Commit 590e93f

Browse files
author
pgandhi
committed
[SPARK-21809] : Fixing Tests and task metric table
1 parent 649e95b commit 590e93f

16 files changed

+824
-437
lines changed

‎core/src/main/resources/org/apache/spark/ui/static/stagepage.js‎

Lines changed: 211 additions & 217 deletions
Large diffs are not rendered by default.

‎core/src/main/resources/org/apache/spark/ui/static/utils.js‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,13 @@ function getStandAloneAppId(cb) {
110110
});
111111
}
112112

113+
function getStageAttemptId() {
114+
var words = document.baseURI.split('?');
115+
var attemptIdStr = words[1].split('&')[1];
116+
var stgAttemptId = attemptIdStr.substr(attemptIdStr.indexOf('=') + 1, attemptIdStr.length - 1);
117+
return stgAttemptId;
118+
}
119+
113120
// This function is a helper function for sorting in datatable.
114121
// When the data is in duration (e.g. 12ms 2s 2min 2h )
115122
// It will convert the string into integer for correct ordering

‎core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -122,28 +122,24 @@ private[v1] class StagesResource extends BaseAppResource {
122122
}
123123

124124
@GET
125-
@Path("{stageId: \\d+}/taskTable")
125+
@Path("{stageId: \\d+}/{stageAttemptId: \\d+}/taskTable")
126126
def taskTable(
127127
@PathParam("stageId") stageId: Int,
128+
@PathParam("stageAttemptId") stageAttemptId: Int,
128129
@QueryParam("details") @DefaultValue("true") details: Boolean, @Context uriInfo: UriInfo): util.HashMap[String, Object] = {
129130
withUI { ui =>
130131
val abc = uriInfo.getQueryParameters(true)
131132
val totalRecords = abc.getFirst("numTasks")
132133
var isSearch = false
133134
var searchValue: String = null
134-
val iter = abc.keySet().iterator()
135-
while(iter.hasNext) {
136-
var ac = iter.next()
137-
System.err.println("hereeeeeeeeeeee 1 " + ac+" : "+abc.get(ac))
138-
}
139135
var _tasksToShow: Seq[TaskData] = null
140136
if (abc.getFirst("search[value]") != null && abc.getFirst("search[value]").length > 0) {
141-
_tasksToShow = ui.store.taskList(stageId, 0, 0, totalRecords.toInt,
137+
_tasksToShow = ui.store.taskList(stageId, stageAttemptId, 0, totalRecords.toInt,
142138
indexName("Index"), true)
143139
isSearch = true
144140
searchValue = abc.getFirst("search[value]")
145141
} else {
146-
_tasksToShow = doPagination(abc, stageId)
142+
_tasksToShow = doPagination(abc, stageId, stageAttemptId)
147143
}
148144
if (_tasksToShow.nonEmpty) {
149145
val iterator = _tasksToShow.iterator
@@ -161,7 +157,7 @@ private[v1] class StagesResource extends BaseAppResource {
161157
if (filteredTaskList.length > 0) {
162158
ret5.put("aaData", filteredTaskList)
163159
} else {
164-
_tasksToShow = doPagination(abc, stageId)
160+
_tasksToShow = doPagination(abc, stageId, stageAttemptId)
165161
val iterator = _tasksToShow.iterator
166162
while(iterator.hasNext) {
167163
val t1: TaskData = iterator.next()
@@ -185,7 +181,7 @@ private[v1] class StagesResource extends BaseAppResource {
185181
}
186182
}
187183

188-
def doPagination(queryParameters: MultivaluedMap[String, String], stageId: Int): Seq[TaskData] = {
184+
def doPagination(queryParameters: MultivaluedMap[String, String], stageId: Int, stageAttemptId: Int): Seq[TaskData] = {
189185
val queryParams = queryParameters.keySet()
190186
var columnToSort = 0
191187
if (queryParams.contains("order[0][column]")) {
@@ -199,7 +195,7 @@ private[v1] class StagesResource extends BaseAppResource {
199195
val isAscendingStr = queryParameters.getFirst("order[0][dir]")
200196
val pageStartIndex = queryParameters.getFirst("start").toInt
201197
val pageLength = queryParameters.getFirst("length").toInt
202-
return withUI(_.store.taskList(stageId, 0, pageStartIndex, pageLength,
198+
return withUI(_.store.taskList(stageId, stageAttemptId, pageStartIndex, pageLength,
203199
indexName(columnNameToSort), isAscendingStr.equalsIgnoreCase("asc")))
204200
}
205201

‎core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json‎

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,9 @@
7878
"writeTime": 3873006,
7979
"recordsWritten": 0
8080
}
81-
}
81+
},
82+
"schedulerDelay" : 0,
83+
"gettingResultTime" : 0
8284
},
8385
"5": {
8486
"taskId": 5,
@@ -125,7 +127,9 @@
125127
"writeTime": 262919,
126128
"recordsWritten": 1
127129
}
128-
}
130+
},
131+
"schedulerDelay" : 0,
132+
"gettingResultTime" : 0
129133
},
130134
"10": {
131135
"taskId": 10,
@@ -172,7 +176,9 @@
172176
"writeTime": 243647,
173177
"recordsWritten": 1
174178
}
175-
}
179+
},
180+
"schedulerDelay" : 0,
181+
"gettingResultTime" : 0
176182
},
177183
"1": {
178184
"taskId": 1,
@@ -219,7 +225,9 @@
219225
"writeTime": 2409488,
220226
"recordsWritten": 1
221227
}
222-
}
228+
},
229+
"schedulerDelay" : 0,
230+
"gettingResultTime" : 0
223231
},
224232
"6": {
225233
"taskId": 6,
@@ -266,7 +274,9 @@
266274
"writeTime": 385110,
267275
"recordsWritten": 1
268276
}
269-
}
277+
},
278+
"schedulerDelay" : 0,
279+
"gettingResultTime" : 0
270280
},
271281
"9": {
272282
"taskId": 9,
@@ -313,7 +323,9 @@
313323
"writeTime": 259354,
314324
"recordsWritten": 1
315325
}
316-
}
326+
},
327+
"schedulerDelay" : 0,
328+
"gettingResultTime" : 0
317329
},
318330
"2": {
319331
"taskId": 2,
@@ -361,7 +373,9 @@
361373
"writeTime": 126128,
362374
"recordsWritten": 0
363375
}
364-
}
376+
},
377+
"schedulerDelay" : 0,
378+
"gettingResultTime" : 0
365379
},
366380
"7": {
367381
"taskId": 7,
@@ -408,7 +422,9 @@
408422
"writeTime": 205520,
409423
"recordsWritten": 1
410424
}
411-
}
425+
},
426+
"schedulerDelay" : 0,
427+
"gettingResultTime" : 0
412428
},
413429
"3": {
414430
"taskId": 3,
@@ -455,7 +471,9 @@
455471
"writeTime": 207014,
456472
"recordsWritten": 1
457473
}
458-
}
474+
},
475+
"schedulerDelay" : 0,
476+
"gettingResultTime" : 0
459477
},
460478
"11": {
461479
"taskId": 11,
@@ -502,7 +520,9 @@
502520
"writeTime": 233652,
503521
"recordsWritten": 1
504522
}
505-
}
523+
},
524+
"schedulerDelay" : 0,
525+
"gettingResultTime" : 0
506526
},
507527
"8": {
508528
"taskId": 8,
@@ -549,7 +569,9 @@
549569
"writeTime": 213296,
550570
"recordsWritten": 1
551571
}
552-
}
572+
},
573+
"schedulerDelay" : 0,
574+
"gettingResultTime" : 0
553575
},
554576
"4": {
555577
"taskId": 4,
@@ -596,7 +618,9 @@
596618
"writeTime": 292381,
597619
"recordsWritten": 1
598620
}
599-
}
621+
},
622+
"schedulerDelay" : 0,
623+
"gettingResultTime" : 0
600624
}
601625
},
602626
"executorSummary": {

‎core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json‎

Lines changed: 42 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,9 @@
7474
"writeTime" : 3662221,
7575
"recordsWritten" : 3
7676
}
77-
}
77+
},
78+
"schedulerDelay" : 0,
79+
"gettingResultTime" : 0
7880
},
7981
"5" : {
8082
"taskId" : 5,
@@ -122,7 +124,9 @@
122124
"writeTime" : 191901,
123125
"recordsWritten" : 0
124126
}
125-
}
127+
},
128+
"schedulerDelay" : 0,
129+
"gettingResultTime" : 0
126130
},
127131
"10" : {
128132
"taskId" : 10,
@@ -169,7 +173,9 @@
169173
"writeTime" : 301705,
170174
"recordsWritten" : 3
171175
}
172-
}
176+
},
177+
"schedulerDelay" : 0,
178+
"gettingResultTime" : 0
173179
},
174180
"1" : {
175181
"taskId" : 1,
@@ -217,7 +223,9 @@
217223
"writeTime" : 3075188,
218224
"recordsWritten" : 0
219225
}
220-
}
226+
},
227+
"schedulerDelay" : 0,
228+
"gettingResultTime" : 0
221229
},
222230
"6" : {
223231
"taskId" : 6,
@@ -265,7 +273,9 @@
265273
"writeTime" : 183718,
266274
"recordsWritten" : 0
267275
}
268-
}
276+
},
277+
"schedulerDelay" : 0,
278+
"gettingResultTime" : 0
269279
},
270280
"9" : {
271281
"taskId" : 9,
@@ -312,7 +322,9 @@
312322
"writeTime" : 366050,
313323
"recordsWritten" : 3
314324
}
315-
}
325+
},
326+
"schedulerDelay" : 0,
327+
"gettingResultTime" : 0
316328
},
317329
"13" : {
318330
"taskId" : 13,
@@ -359,7 +371,9 @@
359371
"writeTime" : 369513,
360372
"recordsWritten" : 3
361373
}
362-
}
374+
},
375+
"schedulerDelay" : 0,
376+
"gettingResultTime" : 0
363377
},
364378
"2" : {
365379
"taskId" : 2,
@@ -406,7 +420,9 @@
406420
"writeTime" : 3322956,
407421
"recordsWritten" : 3
408422
}
409-
}
423+
},
424+
"schedulerDelay" : 0,
425+
"gettingResultTime" : 0
410426
},
411427
"12" : {
412428
"taskId" : 12,
@@ -453,7 +469,9 @@
453469
"writeTime" : 319101,
454470
"recordsWritten" : 3
455471
}
456-
}
472+
},
473+
"schedulerDelay" : 0,
474+
"gettingResultTime" : 0
457475
},
458476
"7" : {
459477
"taskId" : 7,
@@ -500,7 +518,9 @@
500518
"writeTime" : 377601,
501519
"recordsWritten" : 3
502520
}
503-
}
521+
},
522+
"schedulerDelay" : 0,
523+
"gettingResultTime" : 0
504524
},
505525
"3" : {
506526
"taskId" : 3,
@@ -547,7 +567,9 @@
547567
"writeTime" : 3587839,
548568
"recordsWritten" : 3
549569
}
550-
}
570+
},
571+
"schedulerDelay" : 0,
572+
"gettingResultTime" : 0
551573
},
552574
"11" : {
553575
"taskId" : 11,
@@ -594,7 +616,9 @@
594616
"writeTime" : 323898,
595617
"recordsWritten" : 3
596618
}
597-
}
619+
},
620+
"schedulerDelay" : 0,
621+
"gettingResultTime" : 0
598622
},
599623
"8" : {
600624
"taskId" : 8,
@@ -641,7 +665,9 @@
641665
"writeTime" : 311940,
642666
"recordsWritten" : 3
643667
}
644-
}
668+
},
669+
"schedulerDelay" : 0,
670+
"gettingResultTime" : 0
645671
},
646672
"4" : {
647673
"taskId" : 4,
@@ -689,7 +715,9 @@
689715
"writeTime" : 16858066,
690716
"recordsWritten" : 0
691717
}
692-
}
718+
},
719+
"schedulerDelay" : 0,
720+
"gettingResultTime" : 0
693721
}
694722
},
695723
"executorSummary" : {

0 commit comments

Comments
 (0)