leave expired apps alone to be deleted

Author: WangTaoTheTonic

core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala

@@ -278,23 +278,29 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis
 
       val now = System.currentTimeMillis()
       val appsToRetain = new mutable.LinkedHashMap[String, FsApplicationHistoryInfo]()
+      val appsToClean = new mutable.LinkedHashMap[String, FsApplicationHistoryInfo]()
 
       // Scan all logs from the log directory.
       // Only completed applications older than the specified max age will be deleted.
       applications.values.foreach { info =>
         if (now - info.lastUpdated <= maxAge || !info.completed) {
           appsToRetain += (info.id -> info)
         } else {
-          try {
-            fs.delete(new Path(logDir + "/" + info.logPath), true)
-          } catch {
-            case t: IOException => logError(s"IOException in cleaning logs of ${info.logPath}", t)
-            appsToRetain += (info.id -> info)
-          }
+          appsToClean += (info.id -> info)
         }
       }
 
       applications = appsToRetain
+
+      appsToClean.values.foreach { info =>
+        try {
+          fs.delete(new Path(logDir + "/" + info.logPath), true)
+        } catch {
+          case t: IOException =>
+            logError(s"IOException in cleaning logs of ${info.logPath}", t)
+            applications += (info.id -> info)
+        }
+      }
     } catch {
       case t: Exception => logError("Exception in cleaning logs", t)
     }