From c69f13c991d994aeedc238c926506da2ceb25e5e Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 22 Jan 2019 09:09:13 +0000 Subject: [PATCH 1/9] Expose minimum_master_nodes in cluster state To safely support rolling upgrades from 6.x to 7.x we need the 7.x nodes to have access to the `minimum_master_nodes` setting, but this setting is otherwise unnecessary in 7.x and we would like to remove it. Since a rolling upgrade from 6.x to 7.x involves the 7.x nodes joining a 6.x master, we can avoid the need for setting `minimum_master_nodes` on the 7.x nodes by copying the value set on the 6.x master. This change exposes the master's node-level value for `minimum_master_nodes` via a field in the cluster state. --- .../state/TransportClusterStateAction.java | 1 + .../elasticsearch/cluster/ClusterState.java | 35 ++++++++-- .../cluster/coordination/JoinHelper.java | 2 +- .../coordination/JoinTaskExecutor.java | 11 +++- .../discovery/zen/NodeJoinController.java | 6 +- .../discovery/zen/ZenDiscovery.java | 2 +- .../ClusterSerializationTests.java | 6 +- .../MinimumMasterNodesInClusterStateIT.java | 66 +++++++++++++++++++ .../zen/NodeJoinControllerTests.java | 2 +- .../indices/cluster/ClusterStateChanges.java | 2 +- 10 files changed, 119 insertions(+), 14 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/discovery/zen/MinimumMasterNodesInClusterStateIT.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java index f4fa0176cb9e5..4e28dccad5fe7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -127,6 +127,7 @@ private void buildResponse(final ClusterStateRequest request, ClusterState.Builder builder = ClusterState.builder(currentState.getClusterName()); builder.version(currentState.version()); builder.stateUUID(currentState.stateUUID()); + builder.minimumMasterNodesOnPublishingMaster(currentState.getMinimumMasterNodesOnPublishingMaster()); if (request.nodes()) { builder.nodes(currentState.nodes()); diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 8dd7291410ecc..163ec1a808caf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -23,6 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.elasticsearch.Version; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -178,17 +179,19 @@ default boolean isPrivate() { private final boolean wasReadFromDiff; + private final int minimumMasterNodesOnPublishingMaster; + // built on demand private volatile RoutingNodes routingNodes; public ClusterState(long version, String stateUUID, ClusterState state) { this(state.clusterName, version, stateUUID, state.metaData(), state.routingTable(), state.nodes(), state.blocks(), - state.customs(), false); + state.customs(), -1, false); } public ClusterState(ClusterName clusterName, long version, String stateUUID, MetaData metaData, RoutingTable routingTable, DiscoveryNodes nodes, ClusterBlocks blocks, ImmutableOpenMap customs, - boolean wasReadFromDiff) { + int minimumMasterNodesOnPublishingMaster, boolean wasReadFromDiff) { this.version = version; this.stateUUID = stateUUID; this.clusterName = clusterName; @@ -197,6 +200,7 @@ public ClusterState(ClusterName clusterName, long version, String stateUUID, Met this.nodes = nodes; this.blocks = blocks; this.customs = customs; + this.minimumMasterNodesOnPublishingMaster = minimumMasterNodesOnPublishingMaster; this.wasReadFromDiff = wasReadFromDiff; } @@ -290,6 +294,8 @@ public Set getVotingConfigExclusions() { return coordinationMetaData().getVotingConfigExclusions(); } + public int getMinimumMasterNodesOnPublishingMaster() { return minimumMasterNodesOnPublishingMaster; } + // Used for testing and logging to determine how this cluster state was send over the wire public boolean wasReadFromDiff() { return wasReadFromDiff; @@ -436,6 +442,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (metrics.contains(Metric.MASTER_NODE)) { builder.field("master_node", nodes().getMasterNodeId()); + builder.field("minimum_master_nodes", minimumMasterNodesOnPublishingMaster); } if (metrics.contains(Metric.BLOCKS)) { @@ -644,7 +651,7 @@ public static class Builder { private ClusterBlocks blocks = ClusterBlocks.EMPTY_CLUSTER_BLOCK; private final ImmutableOpenMap.Builder customs; private boolean fromDiff; - + private int minimumMasterNodesOnPublishingMaster = -1; public Builder(ClusterState state) { this.clusterName = state.clusterName; @@ -655,6 +662,7 @@ public Builder(ClusterState state) { this.metaData = state.metaData(); this.blocks = state.blocks(); this.customs = ImmutableOpenMap.builder(state.customs()); + this.minimumMasterNodesOnPublishingMaster = state.minimumMasterNodesOnPublishingMaster; this.fromDiff = false; } @@ -715,6 +723,11 @@ public Builder stateUUID(String uuid) { return this; } + public Builder minimumMasterNodesOnPublishingMaster(int minimumMasterNodesOnPublishingMaster) { + this.minimumMasterNodesOnPublishingMaster = minimumMasterNodesOnPublishingMaster; + return this; + } + public Builder putCustom(String type, Custom custom) { customs.put(type, custom); return this; @@ -739,7 +752,8 @@ public ClusterState build() { if (UNKNOWN_UUID.equals(uuid)) { uuid = UUIDs.randomBase64UUID(); } - return new ClusterState(clusterName, version, uuid, metaData, routingTable, nodes, blocks, customs.build(), fromDiff); + return new ClusterState(clusterName, version, uuid, metaData, routingTable, nodes, blocks, customs.build(), + minimumMasterNodesOnPublishingMaster, fromDiff); } public static byte[] toBytes(ClusterState state) throws IOException { @@ -782,6 +796,7 @@ public static ClusterState readFrom(StreamInput in, DiscoveryNode localNode) thr Custom customIndexMetaData = in.readNamedWriteable(Custom.class); builder.putCustom(customIndexMetaData.getWriteableName(), customIndexMetaData); } + builder.minimumMasterNodesOnPublishingMaster = in.getVersion().onOrAfter(Version.V_7_0_0) ? in.readVInt() : -1; return builder.build(); } @@ -807,6 +822,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeNamedWriteable(cursor.value); } } + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeVInt(minimumMasterNodesOnPublishingMaster); + } } private static class ClusterStateDiff implements Diff { @@ -829,6 +847,8 @@ private static class ClusterStateDiff implements Diff { private final Diff> customs; + private final int minimumMasterNodesOnPublishingMaster; + ClusterStateDiff(ClusterState before, ClusterState after) { fromUuid = before.stateUUID; toUuid = after.stateUUID; @@ -839,6 +859,7 @@ private static class ClusterStateDiff implements Diff { metaData = after.metaData.diff(before.metaData); blocks = after.blocks.diff(before.blocks); customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); + minimumMasterNodesOnPublishingMaster = after.minimumMasterNodesOnPublishingMaster; } ClusterStateDiff(StreamInput in, DiscoveryNode localNode) throws IOException { @@ -851,6 +872,7 @@ private static class ClusterStateDiff implements Diff { metaData = MetaData.readDiffFrom(in); blocks = ClusterBlocks.readDiffFrom(in); customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); + minimumMasterNodesOnPublishingMaster = in.getVersion().onOrAfter(Version.V_7_0_0) ? in.readVInt() : -1; } @Override @@ -864,6 +886,9 @@ public void writeTo(StreamOutput out) throws IOException { metaData.writeTo(out); blocks.writeTo(out); customs.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeVInt(minimumMasterNodesOnPublishingMaster); + } } @Override @@ -883,9 +908,9 @@ public ClusterState apply(ClusterState state) { builder.metaData(metaData.apply(state.metaData)); builder.blocks(blocks.apply(state.blocks)); builder.customs(customs.apply(state.customs)); + builder.minimumMasterNodesOnPublishingMaster(minimumMasterNodesOnPublishingMaster); builder.fromDiff(true); return builder.build(); } - } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java index 8c41d7b2eaa52..53fada396fcef 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java @@ -90,7 +90,7 @@ public JoinHelper(Settings settings, AllocationService allocationService, Master this.masterService = masterService; this.transportService = transportService; this.joinTimeout = JOIN_TIMEOUT_SETTING.get(settings); - this.joinTaskExecutor = new JoinTaskExecutor(allocationService, logger) { + this.joinTaskExecutor = new JoinTaskExecutor(settings, allocationService, logger) { @Override public ClusterTasksResult execute(ClusterState currentState, List joiningTasks) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java index c4c76d8a8fe74..c52ab963324b3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinTaskExecutor.java @@ -29,7 +29,9 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; +import org.elasticsearch.discovery.zen.ElectMasterService; import java.util.ArrayList; import java.util.Collection; @@ -45,6 +47,8 @@ public class JoinTaskExecutor implements ClusterStateTaskExecutor secondThirdNodes = internalCluster().startNodes(2); + assertThat(internalCluster().getMasterName(), equalTo(firstNode)); + + final List allNodes = Stream.concat(Stream.of(firstNode), secondThirdNodes.stream()).collect(Collectors.toList()); + for (final String node : allNodes) { + final ClusterState localState = client(node).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState(); + assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(1)); + assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(localState.metaData().settings()), equalTo(2)); + } + + internalCluster().stopRandomNode(nameFilter(firstNode)); + assertThat(internalCluster().getMasterName(), isIn(secondThirdNodes)); + + for (final String node : secondThirdNodes) { + final ClusterState localState = client(node).admin().cluster().state(new ClusterStateRequest().local(true)).get().getState(); + assertThat(localState.getMinimumMasterNodesOnPublishingMaster(), equalTo(2)); + assertThat(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(localState.metaData().settings()), equalTo(2)); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/server/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java index a3ae6b07b19c9..35a2173e0aea0 100644 --- a/server/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java @@ -141,7 +141,7 @@ private void setupMasterServiceAndNodeJoinController(ClusterState initialState) throw new IllegalStateException("method setupMasterServiceAndNodeJoinController can only be called once"); } masterService = ClusterServiceUtils.createMasterService(threadPool, initialState); - nodeJoinController = new NodeJoinController(masterService, createAllocationService(Settings.EMPTY), + nodeJoinController = new NodeJoinController(Settings.EMPTY, masterService, createAllocationService(Settings.EMPTY), new ElectMasterService(Settings.EMPTY)); } diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 387ba1c3d9653..8a00be28f5eb2 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -213,7 +213,7 @@ allocationService, new AliasValidator(), environment, transportService, clusterService, threadPool, createIndexService, actionFilters, indexNameExpressionResolver); nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, logger); - joinTaskExecutor = new JoinTaskExecutor(allocationService, logger); + joinTaskExecutor = new JoinTaskExecutor(Settings.EMPTY, allocationService, logger); } public ClusterState createIndex(ClusterState state, CreateIndexRequest request) { From 1ebb0886df9a4bfc3f37d87ed25059b723fbe1a7 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 22 Jan 2019 10:17:39 +0000 Subject: [PATCH 2/9] Use m_m_nodes from Zen1 master for Zen2 bootstrap Today we support a smooth rolling upgrade from Zen1 to Zen2 by automatically bootstrapping the cluster once all the Zen1 nodes have left, as long as the `minimum_master_nodes` count is satisfied. However this means that Zen2 nodes also require the `minimum_master_nodes` setting for this one specific and transient situation. Since nodes only perform this automatic bootstrapping if they previously belonged to a Zen1 cluster, they can keep track of the `minimum_master_nodes` setting from the previous master instead of requiring it to be set on the Zen2 node. --- .../cluster/coordination/Coordinator.java | 2 +- .../coordination/DiscoveryUpgradeService.java | 65 ++++++++++--------- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 72fe2e081de74..3e3174ac6cf9c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -463,7 +463,7 @@ void becomeCandidate(String method) { clusterFormationFailureHelper.start(); if (getCurrentTerm() == ZEN1_BWC_TERM) { - discoveryUpgradeService.activate(lastKnownLeader); + discoveryUpgradeService.activate(lastKnownLeader, coordinationState.get().getLastAcceptedState()); } leaderChecker.setCurrentNodes(DiscoveryNodes.EMPTY_NODES); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java index 496adb65bb6f0..015585bb779da 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java @@ -24,6 +24,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; @@ -60,6 +61,7 @@ import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING; import static org.elasticsearch.cluster.ClusterState.UNKNOWN_VERSION; import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentSet; +import static org.elasticsearch.discovery.zen.ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING; import static org.elasticsearch.discovery.zen.ZenDiscovery.PING_TIMEOUT_SETTING; /** @@ -80,7 +82,8 @@ public class DiscoveryUpgradeService { public static final Setting ENABLE_UNSAFE_BOOTSTRAPPING_ON_UPGRADE_SETTING = Setting.boolSetting("discovery.zen.unsafe_rolling_upgrades_enabled", true, Setting.Property.NodeScope); - private final ElectMasterService electMasterService; + private static final ElectMasterService permissiveElectMasterService = new ElectMasterService(Settings.EMPTY); + private final TransportService transportService; private final BooleanSupplier isBootstrappedSupplier; private final JoinHelper joinHelper; @@ -98,7 +101,6 @@ public DiscoveryUpgradeService(Settings settings, ClusterSettings clusterSetting Supplier> peersSupplier, Consumer initialConfigurationConsumer) { assert Version.CURRENT.major == Version.V_6_6_0.major + 1 : "remove this service once unsafe upgrades are no longer needed"; - electMasterService = new ElectMasterService(settings); this.transportService = transportService; this.isBootstrappedSupplier = isBootstrappedSupplier; this.joinHelper = joinHelper; @@ -107,12 +109,9 @@ public DiscoveryUpgradeService(Settings settings, ClusterSettings clusterSetting this.bwcPingTimeout = BWC_PING_TIMEOUT_SETTING.get(settings); this.enableUnsafeBootstrappingOnUpgrade = ENABLE_UNSAFE_BOOTSTRAPPING_ON_UPGRADE_SETTING.get(settings); this.clusterName = CLUSTER_NAME_SETTING.get(settings); - - clusterSettings.addSettingsUpdateConsumer(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING, - electMasterService::minimumMasterNodes); // TODO reject update if the new value is too large } - public void activate(Optional lastKnownLeader) { + public void activate(Optional lastKnownLeader, ClusterState lastAcceptedClusterState) { // called under coordinator mutex if (isBootstrappedSupplier.getAsBoolean()) { @@ -122,8 +121,13 @@ public void activate(Optional lastKnownLeader) { assert lastKnownLeader.isPresent() == false || Coordinator.isZen1Node(lastKnownLeader.get()) : lastKnownLeader; // if there was a leader and it's not a old node then we must have been bootstrapped + final Settings dynamicSettings = lastAcceptedClusterState.metaData().settings(); + final int minimumMasterNodes = DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.exists(dynamicSettings) + ? DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(dynamicSettings) + : lastAcceptedClusterState.getMinimumMasterNodesOnPublishingMaster(); + assert joiningRound == null : joiningRound; - joiningRound = new JoiningRound(lastKnownLeader.isPresent()); + joiningRound = new JoiningRound(enableUnsafeBootstrappingOnUpgrade && lastKnownLeader.isPresent(), minimumMasterNodes); joiningRound.scheduleNextAttempt(); } @@ -160,15 +164,22 @@ void countDown() { private class JoiningRound { private final boolean upgrading; + private final int minimumMasterNodes; - JoiningRound(boolean upgrading) { + JoiningRound(boolean upgrading, int minimumMasterNodes) { this.upgrading = upgrading; + this.minimumMasterNodes = minimumMasterNodes; } private boolean isRunning() { return joiningRound == this && isBootstrappedSupplier.getAsBoolean() == false; } + private boolean canBootstrap(Set discoveryNodes) { + return upgrading && 1 <= minimumMasterNodes + && minimumMasterNodes <= discoveryNodes.stream().filter(DiscoveryNode::isMasterNode).count(); + } + void scheduleNextAttempt() { if (isRunning() == false) { return; @@ -189,26 +200,22 @@ public void run() { // this set of nodes is reasonably fresh - the PeerFinder cleans up nodes to which the transport service is not // connected each time it wakes up (every second by default) - logger.debug("nodes: {}", discoveryNodes); - - if (electMasterService.hasEnoughMasterNodes(discoveryNodes)) { - if (discoveryNodes.stream().anyMatch(Coordinator::isZen1Node)) { - electBestOldMaster(discoveryNodes); - } else if (upgrading && enableUnsafeBootstrappingOnUpgrade) { - // no Zen1 nodes found, but the last-known master was a Zen1 node, so this is a rolling upgrade - transportService.getThreadPool().generic().execute(() -> { - try { - initialConfigurationConsumer.accept(new VotingConfiguration(discoveryNodes.stream() - .map(DiscoveryNode::getId).collect(Collectors.toSet()))); - } catch (Exception e) { - logger.debug("exception during bootstrapping upgrade, retrying", e); - } finally { - scheduleNextAttempt(); - } - }); - } else { - scheduleNextAttempt(); - } + logger.debug("upgrading={}, minimumMasterNodes={}, nodes={}", upgrading, minimumMasterNodes, discoveryNodes); + + if (discoveryNodes.stream().anyMatch(Coordinator::isZen1Node)) { + electBestOldMaster(discoveryNodes); + } else if (canBootstrap(discoveryNodes)) { + // no Zen1 nodes found, but the last-known master was a Zen1 node, so this is a rolling upgrade + transportService.getThreadPool().generic().execute(() -> { + try { + initialConfigurationConsumer.accept(new VotingConfiguration(discoveryNodes.stream() + .map(DiscoveryNode::getId).collect(Collectors.toSet()))); + } catch (Exception e) { + logger.debug("exception during bootstrapping upgrade, retrying", e); + } finally { + scheduleNextAttempt(); + } + }); } else { scheduleNextAttempt(); } @@ -232,7 +239,7 @@ public void onResponse(Void value) { // If the only Zen1 nodes left are stale, and we can bootstrap, maybe we should bootstrap? // Do we ever need to elect a freshly-started Zen1 node? if (isRunning()) { - final MasterCandidate electedMaster = electMasterService.electMaster(masterCandidates); + final MasterCandidate electedMaster = permissiveElectMasterService.electMaster(masterCandidates); logger.debug("elected {}, sending join", electedMaster); joinHelper.sendJoinRequest(electedMaster.getNode(), Optional.empty(), JoiningRound.this::scheduleNextAttempt); From 153da7ef8c719cbd7a3393942dbbec6e6d7aa9b1 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 22 Jan 2019 16:43:02 +0000 Subject: [PATCH 3/9] Update ClusterState XContent examples --- .../admin/cluster/reroute/ClusterRerouteResponseTests.java | 4 +++- .../collector/cluster/ClusterStatsMonitoringDocTests.java | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index 1999a18a92b31..095a8660b66fc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -73,6 +73,7 @@ public void testToXContent() throws IOException { " \"version\" : 0,\n" + " \"state_uuid\" : \"" + clusterState.stateUUID() + "\",\n" + " \"master_node\" : \"node0\",\n" + + " \"minimum_master_nodes\" : -1,\n" + " \"blocks\" : { },\n" + " \"nodes\" : {\n" + " \"node0\" : {\n" + @@ -146,7 +147,8 @@ public void testToXContent() throws IOException { " \"cluster_uuid\" : \"_na_\",\n" + " \"version\" : 0,\n" + " \"state_uuid\" : \"" + clusterState.stateUUID() + "\",\n" + - " \"master_node\" : \"node0\"\n" + + " \"master_node\" : \"node0\",\n" + + " \"minimum_master_nodes\" : -1\n" + " },\n" + " \"explanations\" : [\n" + " {\n" + diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java index fbb923c3f2f69..6726ab02a87bc 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java @@ -527,6 +527,7 @@ public void testToXContent() throws IOException { + "\"version\":12," + "\"state_uuid\":\"_state_uuid\"," + "\"master_node\":\"_node\"," + + "\"minimum_master_nodes\":-1," + "\"nodes\":{" + "\"_node_id\":{" + "\"name\":\"_node_name\"," From 92ae2d872258efd9ab7abf8f9cd2b734ba3c205f Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 23 Jan 2019 09:23:26 +0000 Subject: [PATCH 4/9] Whitespace --- .../src/main/java/org/elasticsearch/cluster/ClusterState.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 163ec1a808caf..1ae56d3421bae 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -294,7 +294,9 @@ public Set getVotingConfigExclusions() { return coordinationMetaData().getVotingConfigExclusions(); } - public int getMinimumMasterNodesOnPublishingMaster() { return minimumMasterNodesOnPublishingMaster; } + public int getMinimumMasterNodesOnPublishingMaster() { + return minimumMasterNodesOnPublishingMaster; + } // Used for testing and logging to determine how this cluster state was send over the wire public boolean wasReadFromDiff() { From 6f9194b37b3d8e5d0bb18c56661a190137b08733 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 23 Jan 2019 09:39:21 +0000 Subject: [PATCH 5/9] Remove minimum_master_nodes from XContent output --- .../src/main/java/org/elasticsearch/cluster/ClusterState.java | 1 - .../admin/cluster/reroute/ClusterRerouteResponseTests.java | 4 +--- .../collector/cluster/ClusterStatsMonitoringDocTests.java | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index 1ae56d3421bae..d719072e33816 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -444,7 +444,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (metrics.contains(Metric.MASTER_NODE)) { builder.field("master_node", nodes().getMasterNodeId()); - builder.field("minimum_master_nodes", minimumMasterNodesOnPublishingMaster); } if (metrics.contains(Metric.BLOCKS)) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index 095a8660b66fc..1999a18a92b31 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -73,7 +73,6 @@ public void testToXContent() throws IOException { " \"version\" : 0,\n" + " \"state_uuid\" : \"" + clusterState.stateUUID() + "\",\n" + " \"master_node\" : \"node0\",\n" + - " \"minimum_master_nodes\" : -1,\n" + " \"blocks\" : { },\n" + " \"nodes\" : {\n" + " \"node0\" : {\n" + @@ -147,8 +146,7 @@ public void testToXContent() throws IOException { " \"cluster_uuid\" : \"_na_\",\n" + " \"version\" : 0,\n" + " \"state_uuid\" : \"" + clusterState.stateUUID() + "\",\n" + - " \"master_node\" : \"node0\",\n" + - " \"minimum_master_nodes\" : -1\n" + + " \"master_node\" : \"node0\"\n" + " },\n" + " \"explanations\" : [\n" + " {\n" + diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java index 6726ab02a87bc..fbb923c3f2f69 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java @@ -527,7 +527,6 @@ public void testToXContent() throws IOException { + "\"version\":12," + "\"state_uuid\":\"_state_uuid\"," + "\"master_node\":\"_node\"," - + "\"minimum_master_nodes\":-1," + "\"nodes\":{" + "\"_node_id\":{" + "\"name\":\"_node_name\"," From 846e19479a2cf81cf754e45ac9cb505d7d643090 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 23 Jan 2019 09:50:22 +0000 Subject: [PATCH 6/9] Add javadoc --- .../main/java/org/elasticsearch/cluster/ClusterState.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index d719072e33816..200f5b59d5416 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -294,6 +294,13 @@ public Set getVotingConfigExclusions() { return coordinationMetaData().getVotingConfigExclusions(); } + /** + * The node-level `discovery.zen.minimum_master_nodes` setting on the master node that published this cluster state, for use in rolling + * upgrades from 6.x to 7.x. Once all the 6.x master-eligible nodes have left the cluster, the 7.x nodes use this value to determine how + * many master-eligible nodes must be discovered before the cluster can be bootstrapped. Note that this method returns the node-level + * value of this setting, and ignores any cluster-level override that was set via the API. Callers are expected to combine this value + * with any value set in the cluster-level settings. This should be removed once we no longer need support for {@link Version#V_6_7_0}. + */ public int getMinimumMasterNodesOnPublishingMaster() { return minimumMasterNodesOnPublishingMaster; } From e47b407a9012b1b3ae93c11fd044f59f3561c646 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 23 Jan 2019 09:54:36 +0000 Subject: [PATCH 7/9] Rename permissiveElectMasterService and add javadoc --- .../cluster/coordination/DiscoveryUpgradeService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java index 015585bb779da..c122d57b2cc41 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java @@ -82,7 +82,11 @@ public class DiscoveryUpgradeService { public static final Setting ENABLE_UNSAFE_BOOTSTRAPPING_ON_UPGRADE_SETTING = Setting.boolSetting("discovery.zen.unsafe_rolling_upgrades_enabled", true, Setting.Property.NodeScope); - private static final ElectMasterService permissiveElectMasterService = new ElectMasterService(Settings.EMPTY); + /** + * Dummy {@link ElectMasterService} that is only used to choose the best 6.x master from the discovered nodes, ignoring the + * `minimum_master_nodes` setting. + */ + private static final ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY); private final TransportService transportService; private final BooleanSupplier isBootstrappedSupplier; @@ -239,7 +243,7 @@ public void onResponse(Void value) { // If the only Zen1 nodes left are stale, and we can bootstrap, maybe we should bootstrap? // Do we ever need to elect a freshly-started Zen1 node? if (isRunning()) { - final MasterCandidate electedMaster = permissiveElectMasterService.electMaster(masterCandidates); + final MasterCandidate electedMaster = electMasterService.electMaster(masterCandidates); logger.debug("elected {}, sending join", electedMaster); joinHelper.sendJoinRequest(electedMaster.getNode(), Optional.empty(), JoiningRound.this::scheduleNextAttempt); From c615d4b5f34e07aae84aba9894c232490fae99d0 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 23 Jan 2019 09:58:04 +0000 Subject: [PATCH 8/9] Unused parameter --- .../org/elasticsearch/cluster/coordination/Coordinator.java | 2 +- .../cluster/coordination/DiscoveryUpgradeService.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index c0a69c876e31c..b2c98148162b5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -168,7 +168,7 @@ public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSe this.reconfigurator = new Reconfigurator(settings, clusterSettings); this.clusterBootstrapService = new ClusterBootstrapService(settings, transportService, this::getFoundPeers, this::isInitialConfigurationSet, this::setInitialConfiguration); - this.discoveryUpgradeService = new DiscoveryUpgradeService(settings, clusterSettings, transportService, + this.discoveryUpgradeService = new DiscoveryUpgradeService(settings, transportService, this::isInitialConfigurationSet, joinHelper, peerFinder::getFoundPeers, this::setInitialConfiguration); this.lagDetector = new LagDetector(settings, transportService.getThreadPool(), n -> removeNode(n, "lagging"), transportService::getLocalNode); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java index c122d57b2cc41..6c1567af72933 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java @@ -29,7 +29,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -100,7 +99,7 @@ public class DiscoveryUpgradeService { @Nullable // null if no active joining round private volatile JoiningRound joiningRound; - public DiscoveryUpgradeService(Settings settings, ClusterSettings clusterSettings, TransportService transportService, + public DiscoveryUpgradeService(Settings settings, TransportService transportService, BooleanSupplier isBootstrappedSupplier, JoinHelper joinHelper, Supplier> peersSupplier, Consumer initialConfigurationConsumer) { From a69fdab7345a7097b3a6ee329278f4a7cae4c45c Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 23 Jan 2019 12:28:38 +0000 Subject: [PATCH 9/9] Do not check that minimumMasterNodes is actually set --- .../cluster/coordination/DiscoveryUpgradeService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java index 6c1567af72933..56102704848c8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/DiscoveryUpgradeService.java @@ -179,8 +179,7 @@ private boolean isRunning() { } private boolean canBootstrap(Set discoveryNodes) { - return upgrading && 1 <= minimumMasterNodes - && minimumMasterNodes <= discoveryNodes.stream().filter(DiscoveryNode::isMasterNode).count(); + return upgrading && minimumMasterNodes <= discoveryNodes.stream().filter(DiscoveryNode::isMasterNode).count(); } void scheduleNextAttempt() {