Tests: Improve stability and logging of TemplateUpgradeServiceIT tests (#25386)

Relates to #25382

Author: imotov

core/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java

@@ -120,10 +120,13 @@ public void clusterChanged(ClusterChangedEvent event) {
             return;
         }
 
-
         lastTemplateMetaData = templates;
         Optional<Tuple<Map<String, BytesReference>, Set<String>>> changes = calculateTemplateChanges(templates);
         if (changes.isPresent()) {
+            logger.info("Starting template upgrade to version {}, {} templates will be updated and {} will be removed",
+                Version.CURRENT,
+                changes.get().v1().size(),
+                changes.get().v2().size());
             if (updatesInProgress.compareAndSet(0, changes.get().v1().size() + changes.get().v2().size())) {
                 threadPool.generic().execute(() -> updateTemplates(changes.get().v1(), changes.get().v2()));
             }
@@ -140,8 +143,12 @@ boolean shouldLocalNodeUpdateTemplates(DiscoveryNodes nodes) {
         DiscoveryNode localNode = nodes.getLocalNode();
         // Only data and master nodes should update the template
         if (localNode.isDataNode() || localNode.isMasterNode()) {
+            DiscoveryNode masterNode = nodes.getMasterNode();
+            if (masterNode == null) {
+                return false;
+            }
             Version maxVersion = nodes.getLargestNonClientNodeVersion();
-            if (maxVersion.equals(nodes.getMasterNode().getVersion())) {
+            if (maxVersion.equals(masterNode.getVersion())) {
                 // If the master has the latest version - we will allow it to handle the update
                 return nodes.isLocalNodeElectedMaster();
             } else {
@@ -171,15 +178,19 @@ void updateTemplates(Map<String, BytesReference> changes, Set<String> deletions)
             client.admin().indices().putTemplate(request, new ActionListener<PutIndexTemplateResponse>() {
                 @Override
                 public void onResponse(PutIndexTemplateResponse response) {
-                    updatesInProgress.decrementAndGet();
+                    if(updatesInProgress.decrementAndGet() == 0) {
+                        logger.info("Finished upgrading templates to version {}", Version.CURRENT);
+                    }
                     if (response.isAcknowledged() == false) {
                         logger.warn("Error updating template [{}], request was not acknowledged", change.getKey());
                     }
                 }
 
                 @Override
                 public void onFailure(Exception e) {
-                    updatesInProgress.decrementAndGet();
+                    if(updatesInProgress.decrementAndGet() == 0) {
+                        logger.info("Templates were upgraded to version {}", Version.CURRENT);
+                    }
                     logger.warn(new ParameterizedMessage("Error updating template [{}]", change.getKey()), e);
                 }
             });

core/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceIT.java

@@ -32,7 +32,6 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -41,8 +40,8 @@
 import java.util.function.UnaryOperator;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
 public class TemplateUpgradeServiceIT extends ESIntegTestCase {
@@ -105,10 +104,15 @@ public void testTemplateUpdate() throws Exception {
         assertAcked(client().admin().indices().preparePutTemplate("test_removed_template").setOrder(1)
             .setPatterns(Collections.singletonList("*")).get());
 
+        AtomicInteger updateCount = new AtomicInteger();
         // Wait for the templates to be updated back to normal
         assertBusy(() -> {
+            // the updates only happen on cluster state updates, so we need to make sure that the cluster state updates are happening
+            // so we need to simulate updates to make sure the template upgrade kicks in
+            assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(
+                Settings.builder().put(TestPlugin.UPDATE_TEMPLATE_DUMMY_SETTING.getKey(), updateCount.incrementAndGet())
+            ).get());
             List<IndexTemplateMetaData> templates = client().admin().indices().prepareGetTemplates("test_*").get().getIndexTemplates();
-            assertThat(templates.size(), equalTo(3));
             boolean addedFound = false;
             boolean changedFound = false;
             boolean dummyFound = false;
@@ -133,10 +137,10 @@ public void testTemplateUpdate() throws Exception {
                         break;
                 }
             }
-
             assertTrue(addedFound);
             assertTrue(changedFound);
             assertTrue(dummyFound);
+            assertThat(templates.size(), equalTo(3));
         });
 
         // Wipe out all templates
@@ -157,7 +161,6 @@ private void assertTemplates() throws Exception {
             ).get());
 
             List<IndexTemplateMetaData> templates = client().admin().indices().prepareGetTemplates("test_*").get().getIndexTemplates();
-            assertThat(templates.size(), equalTo(2));
             boolean addedFound = false;
             boolean changedFound = false;
             for (int i = 0; i < 2; i++) {
@@ -180,6 +183,7 @@ private void assertTemplates() throws Exception {
 
             assertTrue(addedFound);
             assertTrue(changedFound);
+            assertThat(templates, hasSize(2));
         });
     }