File tree Expand file tree Collapse file tree 4 files changed +9
-10
lines changed
common/network-shuffle/src/test/java/org/apache/spark/network/shuffle
main/scala/org/apache/spark/deploy/worker
test/scala/org/apache/spark/deploy/worker Expand file tree Collapse file tree 4 files changed +9
-10
lines changed Original file line number Diff line number Diff line change 2727
2828import com .google .common .util .concurrent .MoreExecutors ;
2929import org .junit .Test ;
30- import static org .junit .Assert .assertFalse ;
3130import static org .junit .Assert .assertTrue ;
3231
3332import org .apache .spark .network .util .MapConfigProvider ;
Original file line number Diff line number Diff line change @@ -99,9 +99,9 @@ private[deploy] class Worker(
9999 private val APP_DATA_RETENTION_SECONDS =
100100 conf.getLong(" spark.worker.cleanup.appDataTtl" , 7 * 24 * 3600 )
101101
102- // Whether or not cleanup the non-shuffle files on executor death .
102+ // Whether or not cleanup the non-shuffle files on executor exits .
103103 private val CLEANUP_NON_SHUFFLE_FILES_ENABLED =
104- conf.getBoolean(" spark.storage.cleanupFilesAfterExecutorDeath " , true )
104+ conf.getBoolean(" spark.storage.cleanupFilesAfterExecutorExit " , true )
105105
106106 private val testing : Boolean = sys.props.contains(" spark.testing" )
107107 private var master : Option [RpcEndpointRef ] = None
Original file line number Diff line number Diff line change @@ -212,18 +212,18 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
212212 }
213213 }
214214
215- test(" cleanup non-shuffle files after executor death when config " +
216- " spark.storage.cleanupFilesAfterExecutorDeath =true" ) {
215+ test(" cleanup non-shuffle files after executor exits when config " +
216+ " spark.storage.cleanupFilesAfterExecutorExit =true" ) {
217217 testCleanupFilesWithConfig(true )
218218 }
219219
220- test(" don't cleanup non-shuffle files after executor death when config " +
221- " spark.storage.cleanupFilesAfterExecutorDeath =false" ) {
220+ test(" don't cleanup non-shuffle files after executor exits when config " +
221+ " spark.storage.cleanupFilesAfterExecutorExit =false" ) {
222222 testCleanupFilesWithConfig(false )
223223 }
224224
225225 private def testCleanupFilesWithConfig (value : Boolean ) = {
226- val conf = new SparkConf ().set(" spark.storage.cleanupFilesAfterExecutorDeath " , value.toString)
226+ val conf = new SparkConf ().set(" spark.storage.cleanupFilesAfterExecutorExit " , value.toString)
227227
228228 val cleanupCalled = new AtomicBoolean (false )
229229 when(shuffleService.executorRemoved(any[String ], any[String ])).thenAnswer(new Answer [Unit ] {
Original file line number Diff line number Diff line change @@ -255,11 +255,11 @@ SPARK_WORKER_OPTS supports the following system properties:
255255 </td >
256256</tr >
257257<tr >
258- <td ><code >spark.storage.cleanupFilesAfterExecutorDeath </code ></td >
258+ <td ><code >spark.storage.cleanupFilesAfterExecutorExit </code ></td >
259259 <td >true</td >
260260 <td >
261261 Enable cleanup non-shuffle files(such as temp. shuffle blocks, cached RDD/broadcast blocks,
262- spill files, etc) of worker directories following executor death . Note that this doesn't
262+ spill files, etc) of worker directories following executor exits . Note that this doesn't
263263 overlap with `spark.worker.cleanup.enabled`, as this enables cleanup of non-shuffle files in
264264 local directories of a dead executor, while `spark.worker.cleanup.enabled` enables cleanup of
265265 all files/subdirectories of a stopped and timeout application.
You can’t perform that action at this time.
0 commit comments