Fix IndexAuditTrail rolling restart on rollover edge. (#35988)

This fixes two independent bugs , both tripping integ test failures.
They are both facilitated by the rolling nature of the audit index.
Moreover, they will both manifest only during a rolling upgrade executed
while the audit index rolls over.

Author: albertzaharovits

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java

@@ -277,7 +277,7 @@ private boolean canStart(ClusterState clusterState) {
         }
 
         if (TemplateUtils.checkTemplateExistsAndVersionMatches(INDEX_TEMPLATE_NAME, SECURITY_VERSION_STRING,
-                clusterState, logger, Version.CURRENT::onOrAfter) == false) {
+                clusterState, logger, Version.CURRENT::onOrBefore) == false) {
             logger.debug("security audit index template [{}] is not up to date", INDEX_TEMPLATE_NAME);
             return false;
         }
@@ -308,6 +308,15 @@ private String getIndexName() {
         return index;
     }
 
+    private boolean hasStaleMessage() {
+        final Message first = peek();
+        if (first == null) {
+            return false;
+        }
+        return false == IndexNameResolver.resolve(first.timestamp, rollover)
+                .equals(IndexNameResolver.resolve(DateTime.now(DateTimeZone.UTC), rollover));
+    }
+
     /**
      * Starts the service. The state is moved to {@link org.elasticsearch.xpack.security.audit.index.IndexAuditTrail.State#STARTING}
      * at the beginning of the method. The service's components are initialized and if the current node is the master, the index
@@ -381,7 +390,7 @@ void updateCurrentIndexMappingsIfNecessary(ClusterState state) {
             IndexMetaData indexMetaData = indices.get(0);
             MappingMetaData docMapping = indexMetaData.mapping("doc");
             if (docMapping == null) {
-                if (indexToRemoteCluster || state.nodes().isLocalNodeElectedMaster()) {
+                if (indexToRemoteCluster || state.nodes().isLocalNodeElectedMaster() || hasStaleMessage()) {
                     putAuditIndexMappingsAndStart(index);
                 } else {
                     logger.trace("audit index [{}] is missing mapping for type [{}]", index, DOC_TYPE);
@@ -399,7 +408,7 @@ void updateCurrentIndexMappingsIfNecessary(ClusterState state) {
                 if (versionString != null && Version.fromString(versionString).onOrAfter(Version.CURRENT)) {
                     innerStart();
                 } else {
-                    if (indexToRemoteCluster || state.nodes().isLocalNodeElectedMaster()) {
+                    if (indexToRemoteCluster || state.nodes().isLocalNodeElectedMaster() || hasStaleMessage()) {
                         putAuditIndexMappingsAndStart(index);
                     } else if (versionString == null) {
                         logger.trace("audit index [{}] mapping is missing meta field [{}]", index, SECURITY_VERSION_STRING);

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexAuditUpgradeIT.java

@@ -62,7 +62,6 @@ public void findMinVersionInCluster() throws IOException {
         }
     }
 
-    @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/35867")
     public void testAuditLogs() throws Exception {
         assertBusy(() -> {
             assertAuditDocsExist();