Isolate Zen1 (#39470)

Cherry-picks a few commits from #39466 to align 7.x with master branch.

Author: ywelsch

docs/plugins/integrations.asciidoc

@@ -194,7 +194,7 @@ releases 2.0 and later do not support rivers.
   A pluggable elastic JavaScript query DSL builder for Elasticsearch
 
 * https://www.wireshark.org/[Wireshark]:
-  Protocol dissection for Zen discovery, HTTP and the binary protocol
+  Protocol dissection for HTTP and the transport protocol
 
 * https://www.itemsapi.com/[ItemsAPI]:
   Search backend for mobile and web

docs/reference/getting-started.asciidoc

@@ -248,8 +248,8 @@ If everything goes well with installation, you should see a bunch of messages th
 [2018-09-13T12:20:05,202][INFO ][o.e.t.TransportService   ] [localhost.localdomain] publish_address {127.0.0.1:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300}
 [2018-09-13T12:20:05,221][WARN ][o.e.b.BootstrapChecks    ] [localhost.localdomain] max file descriptors [4096] for elasticsearch process is too low, increase to at least [65535]
 [2018-09-13T12:20:05,221][WARN ][o.e.b.BootstrapChecks    ] [localhost.localdomain] max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
-[2018-09-13T12:20:08,355][INFO ][o.e.c.s.MasterService    ] [localhost.localdomain] zen-disco-elected-as-master ([0] nodes joined)[, ], reason: master node changed {previous [], current [{localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test}]}
-[2018-09-13T12:20:08,360][INFO ][o.e.c.s.ClusterApplierService] [localhost.localdomain] master node changed {previous [], current [{localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test}]}, reason: apply cluster state (from master [master {localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test} committed version [1] source [zen-disco-elected-as-master ([0] nodes joined)[, ]]])
+[2018-09-13T12:20:08,355][INFO ][o.e.c.s.MasterService    ] [localhost.localdomain] elected-as-master ([0] nodes joined)[, ], reason: master node changed {previous [], current [{localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test}]}
+[2018-09-13T12:20:08,360][INFO ][o.e.c.s.ClusterApplierService] [localhost.localdomain] master node changed {previous [], current [{localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test}]}, reason: apply cluster state (from master [master {localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test} committed version [1] source [elected-as-master ([0] nodes joined)[, ]]])
 [2018-09-13T12:20:08,384][INFO ][o.e.h.n.Netty4HttpServerTransport] [localhost.localdomain] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}
 [2018-09-13T12:20:08,384][INFO ][o.e.n.Node               ] [localhost.localdomain] started
 

docs/reference/modules/transport.asciidoc

@@ -156,7 +156,7 @@ PUT _cluster/settings
 {
    "transient" : {
       "transport.tracer.include" : "*",
-      "transport.tracer.exclude" : "internal:discovery/zen/fd*"
+      "transport.tracer.exclude" : "internal:coordination/fault_detection/*"
    }
 }
 --------------------------------------------------

server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.coordination.PublicationTransportHandler;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -42,8 +43,6 @@
 import java.io.IOException;
 import java.util.function.Predicate;
 
-import static org.elasticsearch.discovery.zen.PublishClusterStateAction.serializeFullClusterState;
-
 public class TransportClusterStateAction extends TransportMasterNodeReadAction<ClusterStateRequest, ClusterStateResponse> {
 
 
@@ -185,7 +184,7 @@ private void buildResponse(final ClusterStateRequest request,
             }
         }
         listener.onResponse(new ClusterStateResponse(currentState.getClusterName(), builder.build(),
-            serializeFullClusterState(currentState, Version.CURRENT).length(), false));
+            PublicationTransportHandler.serializeFullClusterState(currentState, Version.CURRENT).length(), false));
     }
 
 

server/src/main/java/org/elasticsearch/cluster/ClusterState.java

@@ -60,7 +60,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.discovery.Discovery;
-import org.elasticsearch.discovery.zen.PublishClusterStateAction;
 
 import java.io.IOException;
 import java.util.EnumSet;
@@ -80,9 +79,8 @@
  * The cluster state can be updated only on the master node. All updates are performed by on a
  * single thread and controlled by the {@link ClusterService}. After every update the
  * {@link Discovery#publish} method publishes a new version of the cluster state to all other nodes in the
- * cluster.  The actual publishing mechanism is delegated to the {@link Discovery#publish} method and depends on
- * the type of discovery. In the Zen Discovery it is handled in the {@link PublishClusterStateAction#publish} method. The
- * publishing mechanism can be overridden by other discovery.
+ * cluster. The actual publishing mechanism is delegated to the {@link Discovery#publish} method and depends on
+ * the type of discovery.
  * <p>
  * The cluster state implements the {@link Diffable} interface in order to support publishing of cluster state
  * differences instead of the entire state on each change. The publishing mechanism should only send differences

server/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java

@@ -21,6 +21,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.cluster.coordination.FollowersChecker;
+import org.elasticsearch.cluster.coordination.LeaderChecker;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -32,8 +34,6 @@
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.KeyedLock;
-import org.elasticsearch.discovery.zen.MasterFaultDetection;
-import org.elasticsearch.discovery.zen.NodesFaultDetection;
 import org.elasticsearch.threadpool.Scheduler;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -51,8 +51,7 @@
  * This component is responsible for connecting to nodes once they are added to the cluster state, and disconnect when they are
  * removed. Also, it periodically checks that all connections are still open and if needed restores them.
  * Note that this component is *not* responsible for removing nodes from the cluster if they disconnect / do not respond
- * to pings. This is done by {@link NodesFaultDetection}. Master fault detection
- * is done by {@link MasterFaultDetection}.
+ * to pings. This is done by {@link FollowersChecker}. Master fault detection is done by {@link LeaderChecker}.
  */
 public class NodeConnectionsService extends AbstractLifecycleComponent {
     private static final Logger logger = LogManager.getLogger(NodeConnectionsService.class);

server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java

@@ -1071,6 +1071,10 @@ private void cancelActivePublication(String reason) {
         }
     }
 
+    public Collection<BiConsumer<DiscoveryNode, ClusterState>> getOnJoinValidators() {
+        return onJoinValidators;
+    }
+
     public enum Mode {
         CANDIDATE, LEADER, FOLLOWER
     }

server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java

@@ -131,7 +131,7 @@ public ClusterTasksResult<JoinTaskExecutor.Task> execute(ClusterState currentSta
             });
 
         transportService.registerRequestHandler(VALIDATE_JOIN_ACTION_NAME,
-            MembershipAction.ValidateJoinRequest::new, ThreadPool.Names.GENERIC,
+            ValidateJoinRequest::new, ThreadPool.Names.GENERIC,
             (request, channel, task) -> {
                 final ClusterState localState = currentStateSupplier.get();
                 if (localState.metaData().clusterUUIDCommitted() &&
@@ -145,7 +145,7 @@ public ClusterTasksResult<JoinTaskExecutor.Task> execute(ClusterState currentSta
             });
 
         transportService.registerRequestHandler(MembershipAction.DISCOVERY_JOIN_VALIDATE_ACTION_NAME,
-            MembershipAction.ValidateJoinRequest::new, ThreadPool.Names.GENERIC,
+            ValidateJoinRequest::new, ThreadPool.Names.GENERIC,
             (request, channel, task) -> {
                 joinValidators.forEach(action -> action.accept(transportService.getLocalNode(), request.getState()));
                 channel.sendResponse(Empty.INSTANCE);
@@ -276,7 +276,7 @@ public void sendValidateJoinRequest(DiscoveryNode node, ClusterState state, Acti
             actionName = VALIDATE_JOIN_ACTION_NAME;
         }
         transportService.sendRequest(node, actionName,
-            new MembershipAction.ValidateJoinRequest(state),
+            new ValidateJoinRequest(state),
             TransportRequestOptions.builder().withTimeout(joinTimeout).build(),
             new EmptyTransportResponseHandler(ThreadPool.Names.GENERIC) {
                 @Override

server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.transport.TransportRequest;
+
+import java.io.IOException;
+
+public class ValidateJoinRequest extends TransportRequest {
+    private ClusterState state;
+
+    public ValidateJoinRequest() {}
+
+    public ValidateJoinRequest(ClusterState state) {
+        this.state = state;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        this.state = ClusterState.readFrom(in, null);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        this.state.writeTo(out);
+    }
+
+    public ClusterState getState() {
+        return state;
+    }
+}

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -67,10 +67,10 @@
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.PeerFinder;
+import org.elasticsearch.discovery.SeedHostsResolver;
+import org.elasticsearch.discovery.SettingsBasedSeedHostsProvider;
 import org.elasticsearch.discovery.zen.ElectMasterService;
 import org.elasticsearch.discovery.zen.FaultDetection;
-import org.elasticsearch.discovery.SettingsBasedSeedHostsProvider;
-import org.elasticsearch.discovery.zen.UnicastZenPing;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
@@ -407,10 +407,10 @@ public void apply(Settings value, Settings current, Settings previous) {
                     ZenDiscovery.MAX_PENDING_CLUSTER_STATES_SETTING,
                     SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING,
                     SettingsBasedSeedHostsProvider.LEGACY_DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING,
-                    UnicastZenPing.DISCOVERY_SEED_RESOLVER_MAX_CONCURRENT_RESOLVERS_SETTING,
-                    UnicastZenPing.DISCOVERY_SEED_RESOLVER_TIMEOUT_SETTING,
-                    UnicastZenPing.LEGACY_DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING,
-                    UnicastZenPing.LEGACY_DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT,
+                    SeedHostsResolver.DISCOVERY_SEED_RESOLVER_MAX_CONCURRENT_RESOLVERS_SETTING,
+                    SeedHostsResolver.DISCOVERY_SEED_RESOLVER_TIMEOUT_SETTING,
+                    SeedHostsResolver.LEGACY_DISCOVERY_ZEN_PING_UNICAST_CONCURRENT_CONNECTS_SETTING,
+                    SeedHostsResolver.LEGACY_DISCOVERY_ZEN_PING_UNICAST_HOSTS_RESOLVE_TIMEOUT,
                     SearchService.DEFAULT_KEEPALIVE_SETTING,
                     SearchService.KEEPALIVE_INTERVAL_SETTING,
                     SearchService.MAX_KEEPALIVE_SETTING,