Prevent nodes from joining if newer indices exist in the cluster (#23843)

Today we prevent nodes from joining when indices exists that are too old.
Yet, the opposite can happen too since lucene / elasticsearch is not forward
compatible when it gets to indices we won't let nodes join the cluster once
there are indices in the clusterstate that are newer than the nodes version.
This prevents forward compatibility issues which we never test against. Yet,
this will not prevent rolling restarts or anything like this since indices
are always created with the minimum node version in the cluster such that an index
can only get the version of the higher nodes once all nodes are upgraded to this version.

Author: s1monw

core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java

@@ -178,20 +178,28 @@ static class ValidateJoinRequestRequestHandler implements TransportRequestHandle
 
         @Override
         public void messageReceived(ValidateJoinRequest request, TransportChannel channel) throws Exception {
-            ensureIndexCompatibility(Version.CURRENT.minimumIndexCompatibilityVersion(), request.state.getMetaData());
+            ensureIndexCompatibility(Version.CURRENT, request.state.getMetaData());
             // for now, the mere fact that we can serialize the cluster state acts as validation....
             channel.sendResponse(TransportResponse.Empty.INSTANCE);
         }
     }
 
     /**
-     * Ensures that all indices are compatible with the supported index version.
+     * Ensures that all indices are compatible with the given node version. This will ensure that all indices in the given metadata
+     * will not be created with a newer version of elasticsearch as well as that all indices are newer or equal to the minimum index
+     * compatibility version.
+     * @see Version#minimumIndexCompatibilityVersion()
      * @throws IllegalStateException if any index is incompatible with the given version
      */
-    static void ensureIndexCompatibility(final Version supportedIndexVersion, MetaData metaData) {
+    static void ensureIndexCompatibility(final Version nodeVersion, MetaData metaData) {
+        Version supportedIndexVersion = nodeVersion.minimumIndexCompatibilityVersion();
         // we ensure that all indices in the cluster we join are compatible with us no matter if they are
         // closed or not we can't read mappings of these indices so we need to reject the join...
         for (IndexMetaData idxMetaData : metaData) {
+            if (idxMetaData.getCreationVersion().after(nodeVersion)) {
+                throw new IllegalStateException("index " + idxMetaData.getIndex() + " version not supported: "
+                    + idxMetaData.getCreationVersion() + " the node version is: " + nodeVersion);
+            }
             if (idxMetaData.getCreationVersion().before(supportedIndexVersion)) {
                 throw new IllegalStateException("index " + idxMetaData.getIndex() + " version not supported: "
                     + idxMetaData.getCreationVersion() + " minimum compatible index version is: " + supportedIndexVersion);

core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java

@@ -453,7 +453,7 @@ public ClusterTasksResult<DiscoveryNode> execute(ClusterState currentState, List
             }
             // we do this validation quite late to prevent race conditions between nodes joining and importing dangling indices
             // we have to reject nodes that don't support all indices we have in this cluster
-            MembershipAction.ensureIndexCompatibility(minNodeVersion.minimumIndexCompatibilityVersion(), currentState.getMetaData());
+            MembershipAction.ensureIndexCompatibility(minNodeVersion, currentState.getMetaData());
             if (nodesChanged) {
                 newState.nodes(nodesBuilder);
                 return results.build(allocationService.reroute(newState.build(), "node_join"));

core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java

@@ -839,7 +839,7 @@ void handleJoinRequest(final DiscoveryNode node, final ClusterState state, final
         } else {
             // we do this in a couple of places including the cluster update thread. This one here is really just best effort
             // to ensure we fail as fast as possible.
-            MembershipAction.ensureIndexCompatibility(node.getVersion().minimumIndexCompatibilityVersion(), state.getMetaData());
+            MembershipAction.ensureIndexCompatibility(node.getVersion(), state.getMetaData());
             // try and connect to the node, if it fails, we can raise an exception back to the client...
             transportService.connectToNode(node);
 

core/src/test/java/org/elasticsearch/discovery/zen/MembershipActionTests.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.discovery.zen;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+
+public class MembershipActionTests extends ESTestCase {
+
+    public void testPreventJoinClusterWithNewerIndices() {
+        Settings.builder().build();
+        MetaData.Builder metaBuilder = MetaData.builder();
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(settings(Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(1).build();
+        metaBuilder.put(indexMetaData, false);
+        MetaData metaData = metaBuilder.build();
+        MembershipAction.ensureIndexCompatibility(Version.CURRENT, metaData);
+
+        expectThrows(IllegalStateException.class, () ->
+        MembershipAction.ensureIndexCompatibility(VersionUtils.getPreviousVersion(Version.CURRENT),
+            metaData));
+    }
+
+    public void testPreventJoinClusterWithUnsupportedIndices() {
+        Settings.builder().build();
+        MetaData.Builder metaBuilder = MetaData.builder();
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(settings(VersionUtils.getPreviousVersion(Version.CURRENT
+                .minimumIndexCompatibilityVersion())))
+            .numberOfShards(1)
+            .numberOfReplicas(1).build();
+        metaBuilder.put(indexMetaData, false);
+        MetaData metaData = metaBuilder.build();
+        expectThrows(IllegalStateException.class, () ->
+            MembershipAction.ensureIndexCompatibility(Version.CURRENT,
+                metaData));
+    }
+
+    public void testSuccess() {
+        Settings.builder().build();
+        MetaData.Builder metaBuilder = MetaData.builder();
+        IndexMetaData indexMetaData = IndexMetaData.builder("test")
+            .settings(settings(VersionUtils.randomVersionBetween(random(),
+                Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT)))
+            .numberOfShards(1)
+            .numberOfReplicas(1).build();
+        metaBuilder.put(indexMetaData, false);
+        indexMetaData = IndexMetaData.builder("test1")
+            .settings(settings(VersionUtils.randomVersionBetween(random(),
+                Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT)))
+            .numberOfShards(1)
+            .numberOfReplicas(1).build();
+        metaBuilder.put(indexMetaData, false);
+        MetaData metaData = metaBuilder.build();
+            MembershipAction.ensureIndexCompatibility(Version.CURRENT,
+                metaData);
+    }
+}