Allow plugins to upgrade templates and index metadata on startup (#24379)

The UpgraderPlugin adds two additional extension points called during cluster upgrade and when old indices are introduced into the cluster state during initial recovery, restore from a snapshot or as a dangling index. One extension point allows plugin to update old templates and another extension points allows to update/check index metadata.

Author: imotov

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java

@@ -35,11 +35,14 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.plugins.Plugin;
 
 import java.util.AbstractMap;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.UnaryOperator;
 
 /**
  * This service is responsible for upgrading legacy index metadata to the current version
@@ -54,14 +57,23 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
     private final NamedXContentRegistry xContentRegistry;
     private final MapperRegistry mapperRegistry;
     private final IndexScopedSettings indexScopedSettings;
+    private final UnaryOperator<IndexMetaData> upgraders;
 
     @Inject
     public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry,
-            IndexScopedSettings indexScopedSettings) {
+                                       IndexScopedSettings indexScopedSettings,
+                                       Collection<UnaryOperator<IndexMetaData>> indexMetaDataUpgraders) {
         super(settings);
         this.xContentRegistry = xContentRegistry;
         this.mapperRegistry = mapperRegistry;
         this.indexScopedSettings = indexScopedSettings;
+        this.upgraders = indexMetaData -> {
+            IndexMetaData newIndexMetaData = indexMetaData;
+            for (UnaryOperator<IndexMetaData> upgrader : indexMetaDataUpgraders) {
+                newIndexMetaData = upgrader.apply(newIndexMetaData);
+            }
+            return newIndexMetaData;
+        };
     }
 
     /**
@@ -84,6 +96,8 @@ public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version m
         newMetaData = archiveBrokenIndexSettings(newMetaData);
         // only run the check with the upgraded settings!!
         checkMappingsCompatibility(newMetaData);
+        // apply plugin checks
+        newMetaData = upgraders.apply(newMetaData);
         return markAsUpgraded(newMetaData);
     }
 

core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -50,6 +51,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.UnaryOperator;
 
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableSet;
@@ -219,8 +223,8 @@ private void ensureNoPre019State() throws Exception {
                     final String name = stateFile.getFileName().toString();
                     if (name.startsWith("metadata-")) {
                         throw new IllegalStateException("Detected pre 0.19 metadata file please upgrade to a version before "
-                                + Version.CURRENT.minimumCompatibilityVersion()
-                                + " first to upgrade state structures - metadata found: [" + stateFile.getParent().toAbsolutePath());
+                            + Version.CURRENT.minimumCompatibilityVersion()
+                            + " first to upgrade state structures - metadata found: [" + stateFile.getParent().toAbsolutePath());
                     }
                 }
             }
@@ -247,23 +251,41 @@ static MetaData upgradeMetaData(MetaData metaData,
             changed |= indexMetaData != newMetaData;
             upgradedMetaData.put(newMetaData, false);
         }
-        // collect current customs
-        Map<String, MetaData.Custom> existingCustoms = new HashMap<>();
-        for (ObjectObjectCursor<String, MetaData.Custom> customCursor : metaData.customs()) {
-            existingCustoms.put(customCursor.key, customCursor.value);
-        }
         // upgrade global custom meta data
-        Map<String, MetaData.Custom> upgradedCustoms = metaDataUpgrader.customMetaDataUpgraders.apply(existingCustoms);
-        if (upgradedCustoms.equals(existingCustoms) == false) {
-            existingCustoms.keySet().forEach(upgradedMetaData::removeCustom);
-            for (Map.Entry<String, MetaData.Custom> upgradedCustomEntry : upgradedCustoms.entrySet()) {
-                upgradedMetaData.putCustom(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
-            }
+        if (applyPluginUpgraders(metaData.getCustoms(), metaDataUpgrader.customMetaDataUpgraders,
+            upgradedMetaData::removeCustom,upgradedMetaData::putCustom)) {
+            changed = true;
+        }
+        // upgrade current templates
+        if (applyPluginUpgraders(metaData.getTemplates(), metaDataUpgrader.indexTemplateMetaDataUpgraders,
+            upgradedMetaData::removeTemplate, (s, indexTemplateMetaData) -> upgradedMetaData.put(indexTemplateMetaData))) {
             changed = true;
         }
         return changed ? upgradedMetaData.build() : metaData;
     }
 
+    private static <Data> boolean applyPluginUpgraders(ImmutableOpenMap<String, Data> existingData,
+                                                       UnaryOperator<Map<String, Data>> upgrader,
+                                                       Consumer<String> removeData,
+                                                       BiConsumer<String, Data> putData) {
+        // collect current data
+        Map<String, Data> existingMap = new HashMap<>();
+        for (ObjectObjectCursor<String, Data> customCursor : existingData) {
+            existingMap.put(customCursor.key, customCursor.value);
+        }
+        // upgrade global custom meta data
+        Map<String, Data> upgradedCustoms = upgrader.apply(existingMap);
+        if (upgradedCustoms.equals(existingMap) == false) {
+            // remove all data first so a plugin can remove custom metadata or templates if needed
+            existingMap.keySet().forEach(removeData);
+            for (Map.Entry<String, Data> upgradedCustomEntry : upgradedCustoms.entrySet()) {
+                putData.accept(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
+            }
+            return true;
+        }
+        return false;
+    }
+
     // shard state BWC
     private void ensureNoPre019ShardState(NodeEnvironment nodeEnv) throws Exception {
         for (Path dataLocation : nodeEnv.nodeDataPaths()) {

core/src/main/java/org/elasticsearch/node/Node.java

@@ -43,6 +43,8 @@
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -400,14 +402,20 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
                 .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService,
                                                  scriptModule.getScriptService(), xContentRegistry).stream())
                 .collect(Collectors.toList());
-            Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders =
-                pluginsService.filterPlugins(Plugin.class).stream()
-                .map(Plugin::getCustomMetaDataUpgrader)
-                .collect(Collectors.toList());
             final RestController restController = actionModule.getRestController();
             final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class),
                     threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, restController);
-            final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders);
+            Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders =
+                pluginsService.filterPlugins(Plugin.class).stream()
+                    .map(Plugin::getCustomMetaDataUpgrader)
+                    .collect(Collectors.toList());
+            Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders =
+                pluginsService.filterPlugins(Plugin.class).stream()
+                    .map(Plugin::getIndexTemplateMetaDataUpgrader)
+                    .collect(Collectors.toList());
+            Collection<UnaryOperator<IndexMetaData>> indexMetaDataUpgraders = pluginsService.filterPlugins(Plugin.class).stream()
+                    .map(Plugin::getIndexMetaDataUpgrader).collect(Collectors.toList());
+            final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders, indexTemplateMetaDataUpgraders);
             final Transport transport = networkModule.getTransportSupplier().get();
             final TransportService transportService = newTransportService(settings, transport, threadPool,
                 networkModule.getTransportInterceptor(), localNodeFactory, settingsModule.getClusterSettings());
@@ -462,8 +470,8 @@ protected Node(final Environment environment, Collection<Class<? extends Plugin>
                     b.bind(TransportService.class).toInstance(transportService);
                     b.bind(NetworkService.class).toInstance(networkService);
                     b.bind(UpdateHelper.class).toInstance(new UpdateHelper(settings, scriptModule.getScriptService()));
-                    b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings,
-                        xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings()));
+                    b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings, xContentRegistry,
+                        indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), indexMetaDataUpgraders));
                     b.bind(ClusterInfoService.class).toInstance(clusterInfoService);
                     b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery());
                     {

core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.plugins;
 
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 
 import java.util.Collection;
@@ -32,13 +33,24 @@
 public class MetaDataUpgrader {
     public final UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgraders;
 
-    public MetaDataUpgrader(Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders) {
+    public final UnaryOperator<Map<String, IndexTemplateMetaData>> indexTemplateMetaDataUpgraders;
+
+    public MetaDataUpgrader(Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders,
+                            Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders) {
         this.customMetaDataUpgraders = customs -> {
             Map<String, MetaData.Custom> upgradedCustoms = new HashMap<>(customs);
             for (UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgrader : customMetaDataUpgraders) {
                 upgradedCustoms = customMetaDataUpgrader.apply(upgradedCustoms);
             }
             return upgradedCustoms;
         };
+
+        this.indexTemplateMetaDataUpgraders = templates -> {
+            Map<String, IndexTemplateMetaData> upgradedTemplates = new HashMap<>(templates);
+            for (UnaryOperator<Map<String, IndexTemplateMetaData>> upgrader : indexTemplateMetaDataUpgraders) {
+                upgradedTemplates = upgrader.apply(upgradedTemplates);
+            }
+            return upgradedTemplates;
+        };
     }
 }

core/src/main/java/org/elasticsearch/plugins/Plugin.java

@@ -23,6 +23,8 @@
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.LifecycleComponent;
@@ -153,14 +155,49 @@ public void onIndexModule(IndexModule indexModule) {}
      * Provides a function to modify global custom meta data on startup.
      * <p>
      * Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required.
+     * <p>
+     * The order of custom meta data upgraders calls is undefined and can change between runs so, it is expected that
+     * plugins will modify only data owned by them to avoid conflicts.
+     * <p>
      * @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map.
      * @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom}
-     *         is unsupported
+     *                               is unsupported
      */
     public UnaryOperator<Map<String, MetaData.Custom>> getCustomMetaDataUpgrader() {
         return UnaryOperator.identity();
     }
 
+    /**
+     * Provides a function to modify index template meta data on startup.
+     * <p>
+     * Plugins should return the input template map via {@link UnaryOperator#identity()} if no upgrade is required.
+     * <p>
+     * The order of the template upgrader calls is undefined and can change between runs so, it is expected that
+     * plugins will modify only templates owned by them to avoid conflicts.
+     * <p>
+     * @return Never {@code null}. The same or upgraded {@code IndexTemplateMetaData} map.
+     * @throws IllegalStateException if the node should not start because at least one {@code IndexTemplateMetaData}
+     *                               cannot be upgraded
+     */
+    public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
+        return UnaryOperator.identity();
+    }
+
+    /**
+     * Provides a function to modify index meta data when an index is introduced into the cluster state for the first time.
+     * <p>
+     * Plugins should return the input index metadata via {@link UnaryOperator#identity()} if no upgrade is required.
+     * <p>
+     * The order of the index upgrader calls for the same index is undefined and can change between runs so, it is expected that
+     * plugins will modify only indices owned by them to avoid conflicts.
+     * <p>
+     * @return Never {@code null}. The same or upgraded {@code IndexMetaData}.
+     * @throws IllegalStateException if the node should not start because the index is unsupported
+     */
+    public UnaryOperator<IndexMetaData> getIndexMetaDataUpgrader() {
+        return UnaryOperator.identity();
+    }
+
     /**
      * Provides the list of this plugin's custom thread pools, empty if
      * none.