Skip to content

Commit ec0afac

Browse files
windpigerwindpiger
committed
merge with master
2 parents a3ac29b + a350bc1 commit ec0afac

File tree

44 files changed

+499
-428
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+499
-428
lines changed

‎common/network-common/src/test/java/org/apache/spark/network/TransportResponseHandlerSuite.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import static org.junit.Assert.assertEquals;
2727
import static org.mockito.Mockito.*;
2828

29-
import org.apache.spark.network.buffer.ManagedBuffer;
3029
import org.apache.spark.network.buffer.NioManagedBuffer;
3130
import org.apache.spark.network.client.ChunkReceivedCallback;
3231
import org.apache.spark.network.client.RpcResponseCallback;

‎common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,8 @@ public void writeTo(ByteBuffer buffer) {
153153
*
154154
* Unlike getBytes this will not create a copy the array if this is a slice.
155155
*/
156-
public @Nonnull ByteBuffer getByteBuffer() {
156+
@Nonnull
157+
public ByteBuffer getByteBuffer() {
157158
if (base instanceof byte[] && offset >= BYTE_ARRAY_OFFSET) {
158159
final byte[] bytes = (byte[]) base;
159160

‎core/src/main/java/org/apache/spark/SparkFirehoseListener.java‎

Lines changed: 113 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -30,116 +30,117 @@
3030
*/
3131
public class SparkFirehoseListener implements SparkListenerInterface {
3232

33-
public void onEvent(SparkListenerEvent event) { }
34-
35-
@Override
36-
public final void onStageCompleted(SparkListenerStageCompleted stageCompleted) {
37-
onEvent(stageCompleted);
38-
}
39-
40-
@Override
41-
public final void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) {
42-
onEvent(stageSubmitted);
43-
}
44-
45-
@Override
46-
public final void onTaskStart(SparkListenerTaskStart taskStart) {
47-
onEvent(taskStart);
48-
}
49-
50-
@Override
51-
public final void onTaskGettingResult(SparkListenerTaskGettingResult taskGettingResult) {
52-
onEvent(taskGettingResult);
53-
}
54-
55-
@Override
56-
public final void onTaskEnd(SparkListenerTaskEnd taskEnd) {
57-
onEvent(taskEnd);
58-
}
59-
60-
@Override
61-
public final void onJobStart(SparkListenerJobStart jobStart) {
62-
onEvent(jobStart);
63-
}
64-
65-
@Override
66-
public final void onJobEnd(SparkListenerJobEnd jobEnd) {
67-
onEvent(jobEnd);
68-
}
69-
70-
@Override
71-
public final void onEnvironmentUpdate(SparkListenerEnvironmentUpdate environmentUpdate) {
72-
onEvent(environmentUpdate);
73-
}
74-
75-
@Override
76-
public final void onBlockManagerAdded(SparkListenerBlockManagerAdded blockManagerAdded) {
77-
onEvent(blockManagerAdded);
78-
}
79-
80-
@Override
81-
public final void onBlockManagerRemoved(SparkListenerBlockManagerRemoved blockManagerRemoved) {
82-
onEvent(blockManagerRemoved);
83-
}
84-
85-
@Override
86-
public final void onUnpersistRDD(SparkListenerUnpersistRDD unpersistRDD) {
87-
onEvent(unpersistRDD);
88-
}
89-
90-
@Override
91-
public final void onApplicationStart(SparkListenerApplicationStart applicationStart) {
92-
onEvent(applicationStart);
93-
}
94-
95-
@Override
96-
public final void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) {
97-
onEvent(applicationEnd);
98-
}
99-
100-
@Override
101-
public final void onExecutorMetricsUpdate(
102-
SparkListenerExecutorMetricsUpdate executorMetricsUpdate) {
103-
onEvent(executorMetricsUpdate);
104-
}
105-
106-
@Override
107-
public final void onExecutorAdded(SparkListenerExecutorAdded executorAdded) {
108-
onEvent(executorAdded);
109-
}
110-
111-
@Override
112-
public final void onExecutorRemoved(SparkListenerExecutorRemoved executorRemoved) {
113-
onEvent(executorRemoved);
114-
}
115-
116-
@Override
117-
public final void onExecutorBlacklisted(SparkListenerExecutorBlacklisted executorBlacklisted) {
118-
onEvent(executorBlacklisted);
119-
}
120-
121-
@Override
122-
public final void onExecutorUnblacklisted(SparkListenerExecutorUnblacklisted executorUnblacklisted) {
123-
onEvent(executorUnblacklisted);
124-
}
125-
126-
@Override
127-
public final void onNodeBlacklisted(SparkListenerNodeBlacklisted nodeBlacklisted) {
128-
onEvent(nodeBlacklisted);
129-
}
130-
131-
@Override
132-
public final void onNodeUnblacklisted(SparkListenerNodeUnblacklisted nodeUnblacklisted) {
133-
onEvent(nodeUnblacklisted);
134-
}
135-
136-
@Override
137-
public void onBlockUpdated(SparkListenerBlockUpdated blockUpdated) {
138-
onEvent(blockUpdated);
139-
}
140-
141-
@Override
142-
public void onOtherEvent(SparkListenerEvent event) {
143-
onEvent(event);
144-
}
33+
public void onEvent(SparkListenerEvent event) { }
34+
35+
@Override
36+
public final void onStageCompleted(SparkListenerStageCompleted stageCompleted) {
37+
onEvent(stageCompleted);
38+
}
39+
40+
@Override
41+
public final void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) {
42+
onEvent(stageSubmitted);
43+
}
44+
45+
@Override
46+
public final void onTaskStart(SparkListenerTaskStart taskStart) {
47+
onEvent(taskStart);
48+
}
49+
50+
@Override
51+
public final void onTaskGettingResult(SparkListenerTaskGettingResult taskGettingResult) {
52+
onEvent(taskGettingResult);
53+
}
54+
55+
@Override
56+
public final void onTaskEnd(SparkListenerTaskEnd taskEnd) {
57+
onEvent(taskEnd);
58+
}
59+
60+
@Override
61+
public final void onJobStart(SparkListenerJobStart jobStart) {
62+
onEvent(jobStart);
63+
}
64+
65+
@Override
66+
public final void onJobEnd(SparkListenerJobEnd jobEnd) {
67+
onEvent(jobEnd);
68+
}
69+
70+
@Override
71+
public final void onEnvironmentUpdate(SparkListenerEnvironmentUpdate environmentUpdate) {
72+
onEvent(environmentUpdate);
73+
}
74+
75+
@Override
76+
public final void onBlockManagerAdded(SparkListenerBlockManagerAdded blockManagerAdded) {
77+
onEvent(blockManagerAdded);
78+
}
79+
80+
@Override
81+
public final void onBlockManagerRemoved(SparkListenerBlockManagerRemoved blockManagerRemoved) {
82+
onEvent(blockManagerRemoved);
83+
}
84+
85+
@Override
86+
public final void onUnpersistRDD(SparkListenerUnpersistRDD unpersistRDD) {
87+
onEvent(unpersistRDD);
88+
}
89+
90+
@Override
91+
public final void onApplicationStart(SparkListenerApplicationStart applicationStart) {
92+
onEvent(applicationStart);
93+
}
94+
95+
@Override
96+
public final void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) {
97+
onEvent(applicationEnd);
98+
}
99+
100+
@Override
101+
public final void onExecutorMetricsUpdate(
102+
SparkListenerExecutorMetricsUpdate executorMetricsUpdate) {
103+
onEvent(executorMetricsUpdate);
104+
}
105+
106+
@Override
107+
public final void onExecutorAdded(SparkListenerExecutorAdded executorAdded) {
108+
onEvent(executorAdded);
109+
}
110+
111+
@Override
112+
public final void onExecutorRemoved(SparkListenerExecutorRemoved executorRemoved) {
113+
onEvent(executorRemoved);
114+
}
115+
116+
@Override
117+
public final void onExecutorBlacklisted(SparkListenerExecutorBlacklisted executorBlacklisted) {
118+
onEvent(executorBlacklisted);
119+
}
120+
121+
@Override
122+
public final void onExecutorUnblacklisted(
123+
SparkListenerExecutorUnblacklisted executorUnblacklisted) {
124+
onEvent(executorUnblacklisted);
125+
}
126+
127+
@Override
128+
public final void onNodeBlacklisted(SparkListenerNodeBlacklisted nodeBlacklisted) {
129+
onEvent(nodeBlacklisted);
130+
}
131+
132+
@Override
133+
public final void onNodeUnblacklisted(SparkListenerNodeUnblacklisted nodeUnblacklisted) {
134+
onEvent(nodeUnblacklisted);
135+
}
136+
137+
@Override
138+
public void onBlockUpdated(SparkListenerBlockUpdated blockUpdated) {
139+
onEvent(blockUpdated);
140+
}
141+
142+
@Override
143+
public void onOtherEvent(SparkListenerEvent event) {
144+
onEvent(event);
145+
}
145146
}

‎core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,9 @@ private UnsafeExternalSorter(
161161
// Register a cleanup task with TaskContext to ensure that memory is guaranteed to be freed at
162162
// the end of the task. This is necessary to avoid memory leaks in when the downstream operator
163163
// does not fully consume the sorter's output (e.g. sort followed by limit).
164-
taskContext.addTaskCompletionListener(context -> { cleanupResources(); });
164+
taskContext.addTaskCompletionListener(context -> {
165+
cleanupResources();
166+
});
165167
}
166168

167169
/**

0 commit comments

Comments
 (0)