From a798db352bf83cb6e1c14d393de7b098b860aa56 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 9 Oct 2015 23:20:04 +0300 Subject: [PATCH 01/19] initial commit --- .../elasticsearch/cluster/ClusterState.java | 52 ++++++--- .../cluster/metadata/IndexMetaData.java | 97 ++++++++++++++--- .../cluster/routing/IndexRoutingTable.java | 20 ++-- .../routing/IndexShardRoutingTable.java | 28 +++-- .../cluster/routing/RoutingTable.java | 10 +- .../cluster/routing/ShardRouting.java | 41 +++++-- .../ExceptionSerializationTests.java | 2 +- .../health/ClusterHealthResponsesTests.java | 9 +- .../TransportBroadcastByNodeActionTests.java | 18 +--- .../ClusterStateCreationUtils.java | 22 ++-- .../cluster/ClusterStateDiffIT.java | 3 +- .../elasticsearch/cluster/DiskUsageTests.java | 28 ++--- .../metadata/ToAndFromJsonMetaDataTests.java | 46 ++++---- .../cluster/routing/AllocationIdTests.java | 10 +- .../cluster/routing/RoutingTableTests.java | 101 +++++++++++++++++- .../cluster/routing/ShardRoutingHelper.java | 2 +- .../cluster/routing/ShardRoutingTests.java | 81 +++++++++----- .../cluster/routing/TestShardRouting.java | 27 +++-- .../cluster/routing/UnassignedInfoTests.java | 2 +- .../allocation/CatAllocationTestCase.java | 6 +- .../allocation/StartedShardsRoutingTests.java | 28 +++-- .../decider/DiskThresholdDeciderTests.java | 48 ++++----- .../DiskThresholdDeciderUnitTests.java | 32 +++--- .../allocation/decider/ShardStateIT.java | 66 ++++++++++++ .../gateway/PrimaryShardAllocatorTests.java | 8 +- .../gateway/PriorityComparatorTests.java | 21 ++-- .../gateway/ReplicaShardAllocatorTests.java | 8 +- .../index/shard/IndexShardTests.java | 14 ++- .../indices/flush/SyncedFlushUnitTests.java | 2 +- .../store/IndicesStoreIntegrationIT.java | 2 +- .../indices/store/IndicesStoreTests.java | 33 +++--- 31 files changed, 586 insertions(+), 281 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/ShardStateIT.java diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java index 5b84ec4367a47..d45a4d0950c79 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -21,7 +21,6 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; - import org.elasticsearch.cluster.DiffableUtils.KeyedReader; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -31,12 +30,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.service.InternalClusterService; import org.elasticsearch.common.Nullable; @@ -57,11 +51,7 @@ import org.elasticsearch.discovery.zen.publish.PublishClusterStateAction; import java.io.IOException; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Set; +import java.util.*; /** * Represents the current state of the cluster. @@ -137,7 +127,7 @@ public static T lookupPrototype(String type) { public static T lookupPrototypeSafe(String type) { @SuppressWarnings("unchecked") - T proto = (T)customPrototypes.get(type); + T proto = (T) customPrototypes.get(type); if (proto == null) { throw new IllegalArgumentException("No custom state prototype registered for type [" + type + "]"); } @@ -663,6 +653,34 @@ public ClusterState build() { if (UNKNOWN_UUID.equals(uuid)) { uuid = Strings.randomBase64UUID(); } + // make sure index meta data and routing tables are in sync w.r.t primaryTerm + MetaData.Builder metaDataBuilder = null; + for (IndexRoutingTable indexRoutingTable : routingTable) { + final IndexMetaData indexMetaData = metaData.index(indexRoutingTable.getIndex()); + IndexMetaData.Builder indexMetaDataBuilder = null; + for (IndexShardRoutingTable shardRoutings : indexRoutingTable) { + final ShardRouting primary = shardRoutings.primaryShard(); + final int shardId = primary.shardId().id(); + if (primary.primaryTerm() != indexMetaData.primaryTerm(shardId)) { + assert primary.primaryTerm() > indexMetaData.primaryTerm(shardId) : + "primary term should only increase. Index primary term [" + + indexMetaData.primaryTerm(shardId) + "] but primary routing is " + primary; + if (indexMetaDataBuilder == null) { + indexMetaDataBuilder = IndexMetaData.builder(indexMetaData); + } + indexMetaDataBuilder.primaryTerm(shardId, primary.primaryTerm()); + } + } + if (indexMetaDataBuilder != null) { + if (metaDataBuilder == null) { + metaDataBuilder = MetaData.builder(metaData); + } + metaDataBuilder.put(indexMetaDataBuilder); + } + } + if (metaDataBuilder != null) { + metaData = metaDataBuilder.build(); + } return new ClusterState(clusterName, version, uuid, metaData, routingTable, nodes, blocks, customs.build(), fromDiff); } @@ -673,16 +691,16 @@ public static byte[] toBytes(ClusterState state) throws IOException { } /** - * @param data input bytes - * @param localNode used to set the local node in the cluster state. + * @param data input bytes + * @param localNode used to set the local node in the cluster state. */ public static ClusterState fromBytes(byte[] data, DiscoveryNode localNode) throws IOException { return readFrom(StreamInput.wrap(data), localNode); } /** - * @param in input stream - * @param localNode used to set the local node in the cluster state. can be null. + * @param in input stream + * @param localNode used to set the local node in the cluster state. can be null. */ public static ClusterState readFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException { return PROTO.readFrom(in, localNode); diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 42e9a4b2244b5..db80c3f2c5b21 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.metadata; +import com.carrotsearch.hppc.IntArrayList; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.Version; @@ -46,10 +47,7 @@ import java.io.IOException; import java.text.ParseException; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; +import java.util.*; import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.AND; import static org.elasticsearch.cluster.node.DiscoveryNodeFilters.OpType.OR; @@ -58,7 +56,7 @@ /** * */ -public class IndexMetaData implements Diffable, FromXContentBuilder, ToXContent { +public class IndexMetaData implements Diffable, FromXContentBuilder, ToXContent { public static final IndexMetaData PROTO = IndexMetaData.builder("") .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) @@ -145,6 +143,7 @@ public static State fromString(String state) { throw new IllegalStateException("No state match for [" + state + "]"); } } + public static final String INDEX_SETTING_PREFIX = "index."; public static final String SETTING_NUMBER_OF_SHARDS = "index.number_of_shards"; public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas"; @@ -173,6 +172,7 @@ public static State fromString(String state) { private final String index; private final long version; + private final int[] primaryTerms; private final State state; @@ -194,7 +194,7 @@ public static State fromString(String state) { private final Version indexUpgradedVersion; private final org.apache.lucene.util.Version minimumCompatibleLuceneVersion; - private IndexMetaData(String index, long version, State state, Settings settings, ImmutableOpenMap mappings, ImmutableOpenMap aliases, ImmutableOpenMap customs) { + private IndexMetaData(String index, long version, int[] primaryTerms, State state, Settings settings, ImmutableOpenMap mappings, ImmutableOpenMap aliases, ImmutableOpenMap customs) { Integer maybeNumberOfShards = settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null); if (maybeNumberOfShards == null) { throw new IllegalArgumentException("must specify numberOfShards for index [" + index + "]"); @@ -212,10 +212,12 @@ private IndexMetaData(String index, long version, State state, Settings settings if (numberOfReplicas < 0) { throw new IllegalArgumentException("must specify non-negative number of shards for index [" + index + "]"); } + this.settings = settings; this.index = index; this.version = version; + this.primaryTerms = primaryTerms; + assert primaryTerms.length == numberOfShards; this.state = state; - this.settings = settings; this.mappings = mappings; this.customs = customs; this.numberOfShards = numberOfShards; @@ -248,7 +250,7 @@ private IndexMetaData(String index, long version, State state, Settings settings try { this.minimumCompatibleLuceneVersion = org.apache.lucene.util.Version.parse(stringLuceneVersion); } catch (ParseException ex) { - throw new IllegalStateException("Cannot parse lucene version [" + stringLuceneVersion + "] in the [" + SETTING_VERSION_MINIMUM_COMPATIBLE +"] setting", ex); + throw new IllegalStateException("Cannot parse lucene version [" + stringLuceneVersion + "] in the [" + SETTING_VERSION_MINIMUM_COMPATIBLE + "] setting", ex); } } else { this.minimumCompatibleLuceneVersion = null; @@ -279,6 +281,15 @@ public long getVersion() { return this.version; } + + /** + * this number is incremented when a replica shard is promoted to a primary (see {@link ShardRouting#moveToPrimary()}) or + * a first primary is created after a full cluster restart. + */ + public int primaryTerm(int shardId) { + return this.primaryTerms[shardId]; + } + /** * Return the {@link Version} on which this index has been created. This * information is typically useful for backward compatibility. @@ -390,6 +401,10 @@ public boolean equals(Object o) { IndexMetaData that = (IndexMetaData) o; + if (version != that.version) { + return false; + } + if (!aliases.equals(that.aliases)) { return false; } @@ -408,6 +423,10 @@ public boolean equals(Object o) { if (!customs.equals(that.customs)) { return false; } + + if (Arrays.equals(primaryTerms, that.primaryTerms)) { + return false; + } return true; } @@ -446,6 +465,7 @@ private static class IndexMetaDataDiff implements Diff { private final String index; private final long version; + private final int[] primaryTerms; private final State state; private final Settings settings; private final Diff> mappings; @@ -457,6 +477,7 @@ public IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) { version = after.version; state = after.state; settings = after.settings; + primaryTerms = after.primaryTerms; mappings = DiffableUtils.diff(before.mappings, after.mappings); aliases = DiffableUtils.diff(before.aliases, after.aliases); customs = DiffableUtils.diff(before.customs, after.customs); @@ -467,6 +488,8 @@ public IndexMetaDataDiff(StreamInput in) throws IOException { version = in.readLong(); state = State.fromId(in.readByte()); settings = Settings.readSettingsFromStream(in); + // must be done after settings so shard number will be set + primaryTerms = in.readIntArray(); mappings = DiffableUtils.readImmutableOpenMapDiff(in, MappingMetaData.PROTO); aliases = DiffableUtils.readImmutableOpenMapDiff(in, AliasMetaData.PROTO); customs = DiffableUtils.readImmutableOpenMapDiff(in, new DiffableUtils.KeyedReader() { @@ -488,6 +511,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(version); out.writeByte(state.id); Settings.writeSettingsToStream(settings, out); + // must be done after settings so shard number will be set + out.writeIntArray(primaryTerms); mappings.writeTo(out); aliases.writeTo(out); customs.writeTo(out); @@ -499,6 +524,7 @@ public IndexMetaData apply(IndexMetaData part) { builder.version(version); builder.state(state); builder.settings(settings); + builder.primaryTerms(primaryTerms); builder.mappings.putAll(mappings.apply(part.mappings)); builder.aliases.putAll(aliases.apply(part.aliases)); builder.customs.putAll(customs.apply(part.customs)); @@ -512,6 +538,8 @@ public IndexMetaData readFrom(StreamInput in) throws IOException { builder.version(in.readLong()); builder.state(State.fromId(in.readByte())); builder.settings(readSettingsFromStream(in)); + // must be done after settings so the array will not be overwritten + builder.primaryTerms(in.readIntArray()); int mappingsSize = in.readVInt(); for (int i = 0; i < mappingsSize; i++) { MappingMetaData mappingMd = MappingMetaData.PROTO.readFrom(in); @@ -537,6 +565,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(version); out.writeByte(state.id()); writeSettingsToStream(settings, out); + out.writeIntArray(primaryTerms); out.writeVInt(mappings.size()); for (ObjectCursor cursor : mappings.values()) { cursor.value.writeTo(out); @@ -565,6 +594,7 @@ public static class Builder { private String index; private State state = State.OPEN; private long version = 1; + private int[] primaryTerms; private Settings settings = Settings.Builder.EMPTY_SETTINGS; private final ImmutableOpenMap.Builder mappings; private final ImmutableOpenMap.Builder aliases; @@ -575,6 +605,7 @@ public Builder(String index) { this.mappings = ImmutableOpenMap.builder(); this.aliases = ImmutableOpenMap.builder(); this.customs = ImmutableOpenMap.builder(); + this.primaryTerms = new int[0]; } public Builder(IndexMetaData indexMetaData) { @@ -582,6 +613,7 @@ public Builder(IndexMetaData indexMetaData) { this.state = indexMetaData.state; this.version = indexMetaData.version; this.settings = indexMetaData.getSettings(); + this.primaryTerms = indexMetaData.primaryTerms; this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings); this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases); this.customs = ImmutableOpenMap.builder(indexMetaData.customs); @@ -598,6 +630,7 @@ public Builder index(String index) { public Builder numberOfShards(int numberOfShards) { settings = settingsBuilder().put(settings).put(SETTING_NUMBER_OF_SHARDS, numberOfShards).build(); + primaryTerms = new int[numberOfShards]; return this; } @@ -613,7 +646,7 @@ public Builder numberOfReplicas(int numberOfReplicas) { public int numberOfReplicas() { return settings.getAsInt(SETTING_NUMBER_OF_REPLICAS, -1); } - + public Builder creationDate(long creationDate) { settings = settingsBuilder().put(settings).put(SETTING_CREATION_DATE, creationDate).build(); return this; @@ -624,12 +657,17 @@ public long creationDate() { } public Builder settings(Settings.Builder settings) { - this.settings = settings.build(); - return this; + return settings(settings.build()); } public Builder settings(Settings settings) { this.settings = settings; + if (numberOfShards() >= 0) { + primaryTerms = new int[numberOfShards()]; + } else { + // meh - what can you do... + primaryTerms = new int[0]; + } return this; } @@ -702,6 +740,21 @@ public Builder version(long version) { return this; } + public int primaryTerm(int shardId) { + assert numberOfShards() >= 0 : "primaryTerm(shardId) is called without settings numberOfShards() first"; + return this.primaryTerms[shardId]; + } + + public Builder primaryTerm(int shardId, int primaryTerm) { + assert numberOfShards() >= 0 : "primaryTerm(shardId, primaryTerm) is called without settings numberOfShards() first"; + this.primaryTerms[shardId] = primaryTerm; + return this; + } + + private void primaryTerms(int[] primaryTerms) { + this.primaryTerms = primaryTerms; + } + public IndexMetaData build() { ImmutableOpenMap.Builder tmpAliases = aliases; Settings tmpSettings = settings; @@ -714,7 +767,7 @@ public IndexMetaData build() { } } - return new IndexMetaData(index, version, state, tmpSettings, mappings.build(), tmpAliases.build(), customs.build()); + return new IndexMetaData(index, version, primaryTerms, state, tmpSettings, mappings.build(), tmpAliases.build(), customs.build()); } public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { @@ -757,6 +810,11 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build } builder.endObject(); + builder.startArray("primary_terms"); + for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) { + builder.value(indexMetaData.primaryTerm(i)); + } + builder.endArray(); builder.endObject(); } @@ -816,6 +874,21 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti } } } + } else if ("primary_terms".equals(currentFieldName)) { + IntArrayList list = new IntArrayList(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_NUMBER) { + list.add(parser.intValue()); + } else { + throw new IllegalStateException("found a non-numeric value under [primary_terms]"); + } + } + if (builder.numberOfShards() < 0) { + builder.numberOfShards(list.size()); // alocation terms array + } + for (int i = 0; i < list.size(); i++) { + builder.primaryTerm(i, list.get(i)); + } } } else if (token.isValue()) { if ("state".equals(currentFieldName)) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index ca071c811e3e1..b2bbcaab5a57f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -99,14 +99,14 @@ public String getIndex() { } /** - * creates a new {@link IndexRoutingTable} with all shard versions normalized + * creates a new {@link IndexRoutingTable} with all shard versions & primary terms set to the highest found. * * @return new {@link IndexRoutingTable} */ - public IndexRoutingTable normalizeVersions() { + public IndexRoutingTable normalizeVersionsAndPrimaryTerms() { IndexRoutingTable.Builder builder = new Builder(this.index); for (IntObjectCursor cursor : shards) { - builder.addIndexShard(cursor.value.normalizeVersions()); + builder.addIndexShard(cursor.value.normalizeVersionsAndPrimaryTerms()); } return builder.build(); } @@ -422,11 +422,12 @@ private Builder initializeAsRestore(IndexMetaData indexMetaData, RestoreSource r for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) { IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId)); for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) { + final int primaryTerm = indexMetaData.primaryTerm(shardId); if (asNew && ignoreShards.contains(shardId)) { // This shards wasn't completely snapshotted - restore it as new shard - indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo)); + indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, primaryTerm, i == 0, unassignedInfo)); } else { - indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, i == 0 ? restoreSource : null, i == 0, unassignedInfo)); + indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, i == 0 ? restoreSource : null, primaryTerm, i == 0, unassignedInfo)); } } shards.put(shardId, indexShardRoutingBuilder.build()); @@ -442,9 +443,10 @@ private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unas throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); } for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) { + final int primaryTerm = indexMetaData.primaryTerm(shardId); IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId)); for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) { - indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo)); + indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null,primaryTerm, i == 0, unassignedInfo)); } shards.put(shardId, indexShardRoutingBuilder.build()); } @@ -455,9 +457,11 @@ public Builder addReplica() { for (IntCursor cursor : shards.keys()) { int shardId = cursor.value; // version 0, will get updated when reroute will happen - ShardRouting shard = ShardRouting.newUnassigned(index, shardId, null, false, new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null)); + final IndexShardRoutingTable shardRoutingTable = shards.get(shardId); + ShardRouting shard = ShardRouting.newUnassigned(index, shardId, null, shardRoutingTable.primary.primaryTerm(), false, + new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null)); shards.put(shardId, - new IndexShardRoutingTable.Builder(shards.get(shard.id())).addShard(shard).build() + new IndexShardRoutingTable.Builder(shardRoutingTable).addShard(shard).build() ); } return this; diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java index 6512ee5cef799..7eda4d63dd780 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java @@ -28,14 +28,7 @@ import org.elasticsearch.index.shard.ShardId; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.ThreadLocalRandom; import static java.util.Collections.emptyMap; @@ -120,34 +113,37 @@ public class IndexShardRoutingTable implements Iterable { } /** - * Normalizes all shard routings to the same version. + * Normalizes all shard routings to the same (highest found) version & primary terms. */ - public IndexShardRoutingTable normalizeVersions() { + public IndexShardRoutingTable normalizeVersionsAndPrimaryTerms() { if (shards.isEmpty()) { return this; } + if (shards.size() == 1) { return this; } long highestVersion = shards.get(0).version(); + int highestPrimaryTerm = shards.get(0).primaryTerm(); boolean requiresNormalization = false; for (int i = 1; i < shards.size(); i++) { - if (shards.get(i).version() != highestVersion) { + final long version = shards.get(i).version(); + final int primaryTerm = shards.get(i).primaryTerm(); + if (highestVersion != version || highestPrimaryTerm != primaryTerm) { requiresNormalization = true; } - if (shards.get(i).version() > highestVersion) { - highestVersion = shards.get(i).version(); - } + highestVersion = Math.max(highestVersion, version); + highestPrimaryTerm = Math.max(highestPrimaryTerm, primaryTerm); } if (!requiresNormalization) { return this; } List shardRoutings = new ArrayList<>(shards.size()); for (int i = 0; i < shards.size(); i++) { - if (shards.get(i).version() == highestVersion) { + if (shards.get(i).version() == highestVersion && shards.get(i).primaryTerm() == highestPrimaryTerm) { shardRoutings.add(shards.get(i)); } else { - shardRoutings.add(new ShardRouting(shards.get(i), highestVersion)); + shardRoutings.add(new ShardRouting(shards.get(i), highestVersion, highestPrimaryTerm)); } } return new IndexShardRoutingTable(shardId, Collections.unmodifiableList(shardRoutings)); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index d27c1c098ae67..59dd5fcd9aada 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -22,7 +22,6 @@ import com.carrotsearch.hppc.IntSet; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; - import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.Diffable; import org.elasticsearch.cluster.DiffableUtils; @@ -35,12 +34,7 @@ import org.elasticsearch.index.IndexNotFoundException; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Predicate; /** @@ -540,7 +534,7 @@ public RoutingTable build() { } // normalize the versions right before we build it... for (ObjectCursor indexRoutingTable : indicesRouting.values()) { - indicesRouting.put(indexRoutingTable.value.index(), indexRoutingTable.value.normalizeVersions()); + indicesRouting.put(indexRoutingTable.value.index(), indexRoutingTable.value.normalizeVersionsAndPrimaryTerms()); } RoutingTable table = new RoutingTable(version, indicesRouting.build()); indicesRouting = null; diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index 8ee8205ca60a6..59d6d2ba07a56 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -47,6 +47,7 @@ public final class ShardRouting implements Streamable, ToXContent { private String currentNodeId; private String relocatingNodeId; private boolean primary; + private int primaryTerm; private ShardRoutingState state; private long version; private RestoreSource restoreSource; @@ -62,11 +63,15 @@ private ShardRouting() { } public ShardRouting(ShardRouting copy) { - this(copy, copy.version()); + this(copy, copy.version(), copy.primaryTerm()); } public ShardRouting(ShardRouting copy, long version) { - this(copy.index(), copy.id(), copy.currentNodeId(), copy.relocatingNodeId(), copy.restoreSource(), copy.primary(), copy.state(), version, copy.unassignedInfo(), copy.allocationId(), true, copy.getExpectedShardSize()); + this(copy, version, copy.primaryTerm()); + } + + public ShardRouting(ShardRouting copy, long version, int primaryTerm) { + this(copy.index(), copy.id(), copy.currentNodeId(), copy.relocatingNodeId(), copy.restoreSource(), primaryTerm, copy.primary(), copy.state(), version, copy.unassignedInfo(), copy.allocationId(), true, copy.getExpectedShardSize()); } /** @@ -74,12 +79,13 @@ public ShardRouting(ShardRouting copy, long version) { * by either this class or tests. Visible for testing. */ ShardRouting(String index, int shardId, String currentNodeId, - String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state, long version, + String relocatingNodeId, RestoreSource restoreSource, int primaryTerm, boolean primary, ShardRoutingState state, long version, UnassignedInfo unassignedInfo, AllocationId allocationId, boolean internal, long expectedShardSize) { this.index = index; this.shardId = shardId; this.currentNodeId = currentNodeId; this.relocatingNodeId = relocatingNodeId; + this.primaryTerm = primaryTerm; this.primary = primary; this.state = state; this.asList = Collections.singletonList(this); @@ -103,8 +109,8 @@ public ShardRouting(ShardRouting copy, long version) { /** * Creates a new unassigned shard. */ - public static ShardRouting newUnassigned(String index, int shardId, RestoreSource restoreSource, boolean primary, UnassignedInfo unassignedInfo) { - return new ShardRouting(index, shardId, null, null, restoreSource, primary, ShardRoutingState.UNASSIGNED, 0, unassignedInfo, null, true, UNAVAILABLE_EXPECTED_SHARD_SIZE); + public static ShardRouting newUnassigned(String index, int shardId, RestoreSource restoreSource, int primaryTerm, boolean primary, UnassignedInfo unassignedInfo) { + return new ShardRouting(index, shardId, null, null, restoreSource, primaryTerm, primary, ShardRoutingState.UNASSIGNED, 0, unassignedInfo, null, true, UNAVAILABLE_EXPECTED_SHARD_SIZE); } /** @@ -214,7 +220,7 @@ public String relocatingNodeId() { */ public ShardRouting buildTargetRelocatingShard() { assert relocating(); - return new ShardRouting(index, shardId, relocatingNodeId, currentNodeId, restoreSource, primary, ShardRoutingState.INITIALIZING, version, unassignedInfo, + return new ShardRouting(index, shardId, relocatingNodeId, currentNodeId, restoreSource, primaryTerm, primary, ShardRoutingState.INITIALIZING, version, unassignedInfo, AllocationId.newTargetRelocation(allocationId), true, expectedShardSize); } @@ -249,6 +255,13 @@ public boolean primary() { return this.primary; } + /** + * Returns the term of the current primary shard for this shard. The term is incremented with every primary promotion/initial assignment + */ + public int primaryTerm() { + return this.primaryTerm; + } + /** * The shard state. */ @@ -318,6 +331,7 @@ public void readFromThin(StreamInput in) throws IOException { } primary = in.readBoolean(); + primaryTerm = in.readInt(); state = ShardRoutingState.fromValue(in.readByte()); restoreSource = RestoreSource.readOptionalRestoreSource(in); @@ -363,6 +377,7 @@ public void writeToThin(StreamOutput out) throws IOException { } out.writeBoolean(primary); + out.writeInt(primaryTerm); out.writeByte(state.value()); if (restoreSource != null) { @@ -507,6 +522,7 @@ void moveToPrimary() { throw new IllegalShardRoutingStateException(this, "Already primary, can't move to primary"); } primary = true; + primaryTerm++; } /** @@ -562,6 +578,9 @@ public boolean isRelocationTargetOf(ShardRouting other) { assert b == false || this.primary == other.primary : "ShardRouting is a relocation target but primary flag is different. This [" + this + "], target [" + other + "]"; + assert b == false || this.primaryTerm == other.primaryTerm : + "ShardRouting is a relocation target but primary term is different. This [" + this + "], target [" + other + "]"; + return b; } @@ -589,10 +608,13 @@ public boolean isRelocationSourceOf(ShardRouting other) { assert b == false || this.primary == other.primary : "ShardRouting is a relocation source but primary flag is different. This [" + this + "], target [" + other + "]"; + assert b == false || this.primaryTerm == other.primaryTerm : + "ShardRouting is a relocation source but primary term is different. This [" + this + "], target [" + other + "]"; + return b; } - /** returns true if the current routing is identical to the other routing in all but meta fields, i.e., version and unassigned info */ + /** returns true if the current routing is identical to the other routing in all but meta fields, i.e., version, primary term and unassigned info */ public boolean equalsIgnoringMetaData(ShardRouting other) { if (primary != other.primary) { return false; @@ -637,6 +659,9 @@ public boolean equals(Object o) { if (unassignedInfo != null ? !unassignedInfo.equals(that.unassignedInfo) : that.unassignedInfo != null) { return false; } + if (primaryTerm != that.primaryTerm) { + return false; + } return equalsIgnoringMetaData(that); } @@ -682,6 +707,7 @@ public String shortSummary() { sb.append("[R]"); } sb.append(", v[").append(version).append("]"); + sb.append(", t[").append(primaryTerm).append("]"); if (this.restoreSource != null) { sb.append(", restoring[" + restoreSource + "]"); } @@ -703,6 +729,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject() .field("state", state()) .field("primary", primary()) + .field("primary_term", primaryTerm()) .field("node", currentNodeId()) .field("relocating_node", relocatingNodeId()) .field("shard", shardId().id()) diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index 55dc2e4211380..b65ab41e5b6e5 100644 --- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -206,7 +206,7 @@ private T serialize(T exception) throws IOException { } public void testIllegalShardRoutingStateException() throws IOException { - final ShardRouting routing = TestShardRouting.newShardRouting("test", 0, "xyz", "def", false, ShardRoutingState.STARTED, 0); + final ShardRouting routing = TestShardRouting.newShardRouting("test", 0, "xyz", "def", 1, false, ShardRoutingState.STARTED, 0); final String routingAsString = routing.toString(); IllegalShardRoutingStateException serialize = serialize(new IllegalShardRoutingStateException(routing, "foo", new NullPointerException())); assertNotNull(serialize.shard()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java index ec050232057b4..7660fd4c93955 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java @@ -132,11 +132,14 @@ private ShardRouting genShardRouting(String index, int shardId, boolean primary) switch (state) { case STARTED: - return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), null, null, primary, ShardRoutingState.STARTED, 1); + return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), null, null, 1, primary, + ShardRoutingState.STARTED, 1); case INITIALIZING: - return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), null, null, primary, ShardRoutingState.INITIALIZING, 1); + return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), null, null, 1, primary, + ShardRoutingState.INITIALIZING, 1); case RELOCATING: - return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), "node_" + Integer.toString(node_id++), null, primary, ShardRoutingState.RELOCATING, 1); + return TestShardRouting.newShardRouting(index, shardId, "node_" + Integer.toString(node_id++), + "node_" + Integer.toString(node_id++), null, 1, primary, ShardRoutingState.RELOCATING, 1); default: throw new ElasticsearchException("Unknown state: " + state.name()); } diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java index 18a86b1dea0c2..ac4e399cd626d 100644 --- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java @@ -38,13 +38,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.ShardsIterator; -import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.DummyTransportAddress; @@ -63,12 +57,7 @@ import org.junit.BeforeClass; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -209,7 +198,8 @@ void setClusterState(TestClusterService clusterService, String index) { int numberOfShards = randomIntBetween(1, 10); for (int j = 0; j < numberOfShards; j++) { final ShardId shardId = new ShardId(index, ++shardIndex); - ShardRouting shard = TestShardRouting.newShardRouting(index, shardId.getId(), node.id(), true, ShardRoutingState.STARTED, 1); + final int primaryTerm = randomInt(200); + ShardRouting shard = TestShardRouting.newShardRouting(index, shardId.getId(), node.id(), primaryTerm, true, ShardRoutingState.STARTED, 1); IndexShardRoutingTable.Builder indexShard = new IndexShardRoutingTable.Builder(shardId); indexShard.addShard(shard); indexRoutingTable.addIndexShard(indexShard.build()); diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java index 913d52d5b173e..9cec081a2facc 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java @@ -40,12 +40,8 @@ import java.util.HashSet; import java.util.Set; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED; -import static org.elasticsearch.test.ESTestCase.randomFrom; -import static org.elasticsearch.test.ESTestCase.randomIntBetween; +import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.test.ESTestCase.*; /** * Helper methods for generating cluster states @@ -109,7 +105,9 @@ public static ClusterState state(String index, boolean primaryLocal, ShardRoutin } else { unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null); } - indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, 0, primaryNode, relocatingNode, null, true, primaryState, 0, unassignedInfo)); + final int primaryTerm = randomInt(200); + indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, 0, primaryNode, relocatingNode, null, primaryTerm, + true, primaryState, 0, unassignedInfo)); for (ShardRoutingState replicaState : replicaStates) { String replicaNode = null; @@ -125,7 +123,8 @@ public static ClusterState state(String index, boolean primaryLocal, ShardRoutin unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null); } indexShardRoutingBuilder.addShard( - TestShardRouting.newShardRouting(index, shardId.id(), replicaNode, relocatingNode, null, false, replicaState, 0, unassignedInfo)); + TestShardRouting.newShardRouting(index, shardId.id(), replicaNode, relocatingNode, null, primaryTerm, false, + replicaState, 0, unassignedInfo)); } ClusterState.Builder state = ClusterState.builder(new ClusterName("test")); @@ -156,13 +155,16 @@ public static ClusterState stateWithAssignedPrimariesAndOneReplica(String index, state.nodes(discoBuilder); state.metaData(MetaData.builder().put(indexMetaData, false).generateClusterUuidIfNeeded()); IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index); + final int primaryTerm = randomInt(200); for (int i = 0; i < numberOfShards; i++) { RoutingTable.Builder routing = new RoutingTable.Builder(); routing.addAsNew(indexMetaData); final ShardId shardId = new ShardId(index, i); IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); - indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(0).id(), null, null, true, ShardRoutingState.STARTED, 0, null)); - indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(1).id(), null, null, false, ShardRoutingState.STARTED, 0, null)); + indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(0).id(), null, null, primaryTerm, true, + ShardRoutingState.STARTED, 0, null)); + indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, i, newNode(1).id(), null, null, primaryTerm, false, + ShardRoutingState.STARTED, 0, null)); indexRoutingTableBuilder.addIndexShard(indexShardRoutingBuilder.build()); } state.routingTable(RoutingTable.builder().add(indexRoutingTableBuilder.build()).build()); diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index 301dc8bb5bbb5..71f95919f0a46 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -236,13 +236,14 @@ private IndexRoutingTable randomIndexRoutingTable(String index, String[] nodeIds for (int i = 0; i < shardCount; i++) { IndexShardRoutingTable.Builder indexShard = new IndexShardRoutingTable.Builder(new ShardId(index, i)); int replicaCount = randomIntBetween(1, 10); + int term = randomInt(200); for (int j = 0; j < replicaCount; j++) { UnassignedInfo unassignedInfo = null; if (randomInt(5) == 1) { unassignedInfo = new UnassignedInfo(randomReason(), randomAsciiOfLength(10)); } indexShard.addShard( - TestShardRouting.newShardRouting(index, i, randomFrom(nodeIds), null, null, j == 0, + TestShardRouting.newShardRouting(index, i, randomFrom(nodeIds), null, null, term, j == 0, ShardRoutingState.fromValue((byte) randomIntBetween(2, 4)), 1, unassignedInfo)); } builder.addIndexShard(indexShard.build()); diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java index 595dbc9a94fdd..86df3f423b72d 100644 --- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java @@ -52,19 +52,19 @@ public void diskUsageCalcTest() { // Test that DiskUsage handles invalid numbers, as reported by some // filesystems (ZFS & NTFS) - DiskUsage du2 = new DiskUsage("node1", "n1","random", 100, 101); + DiskUsage du2 = new DiskUsage("node1", "n1", "random", 100, 101); assertThat(du2.getFreeDiskAsPercentage(), equalTo(101.0)); assertThat(du2.getFreeBytes(), equalTo(101L)); assertThat(du2.getUsedBytes(), equalTo(-1L)); assertThat(du2.getTotalBytes(), equalTo(100L)); - DiskUsage du3 = new DiskUsage("node1", "n1", "random",-1, -1); + DiskUsage du3 = new DiskUsage("node1", "n1", "random", -1, -1); assertThat(du3.getFreeDiskAsPercentage(), equalTo(100.0)); assertThat(du3.getFreeBytes(), equalTo(-1L)); assertThat(du3.getUsedBytes(), equalTo(0L)); assertThat(du3.getTotalBytes(), equalTo(-1L)); - DiskUsage du4 = new DiskUsage("node1", "n1","random", 0, 0); + DiskUsage du4 = new DiskUsage("node1", "n1", "random", 0, 0); assertThat(du4.getFreeDiskAsPercentage(), equalTo(100.0)); assertThat(du4.getFreeBytes(), equalTo(0L)); assertThat(du4.getUsedBytes(), equalTo(0L)); @@ -95,21 +95,21 @@ public void randomDiskUsageTest() { } public void testFillShardLevelInfo() { - ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, 1, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); ShardRoutingHelper.initialize(test_0, "node1"); ShardRoutingHelper.moveToStarted(test_0); Path test0Path = createTempDir().resolve("indices").resolve("test").resolve("0"); CommonStats commonStats0 = new CommonStats(); commonStats0.store = new StoreStats(100, 1); - ShardRouting test_1 = ShardRouting.newUnassigned("test", 1, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_1 = ShardRouting.newUnassigned("test", 1, null, 1, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); ShardRoutingHelper.initialize(test_1, "node2"); ShardRoutingHelper.moveToStarted(test_1); Path test1Path = createTempDir().resolve("indices").resolve("test").resolve("1"); CommonStats commonStats1 = new CommonStats(); commonStats1.store = new StoreStats(1000, 1); - ShardStats[] stats = new ShardStats[] { - new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, "0xdeadbeef", test_0.shardId()), commonStats0 , null), - new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, "0xdeadbeef", test_1.shardId()), commonStats1 , null) + ShardStats[] stats = new ShardStats[]{ + new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, "0xdeadbeef", test_0.shardId()), commonStats0, null), + new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, "0xdeadbeef", test_1.shardId()), commonStats1, null) }; ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder(); ImmutableOpenMap.Builder routingToPath = ImmutableOpenMap.builder(); @@ -135,21 +135,21 @@ public void testFillDiskUsage() { new FsInfo.Path("/least", "/dev/sdb", 200, 190, 70), new FsInfo.Path("/most", "/dev/sdc", 300, 290, 280), }; - FsInfo.Path[] node2FSInfo = new FsInfo.Path[] { + FsInfo.Path[] node2FSInfo = new FsInfo.Path[]{ new FsInfo.Path("/least_most", "/dev/sda", 100, 90, 80), }; - FsInfo.Path[] node3FSInfo = new FsInfo.Path[] { + FsInfo.Path[] node3FSInfo = new FsInfo.Path[]{ new FsInfo.Path("/least", "/dev/sda", 100, 90, 70), new FsInfo.Path("/most", "/dev/sda", 100, 90, 80), }; - NodeStats[] nodeStats = new NodeStats[] { + NodeStats[] nodeStats = new NodeStats[]{ new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Version.CURRENT), 0, - null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null), + null, null, null, null, null, new FsInfo(0, node1FSInfo), null, null, null, null), new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null), + null, null, null, null, null, new FsInfo(0, node2FSInfo), null, null, null, null), new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, Version.CURRENT), 0, - null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null) + null, null, null, null, null, new FsInfo(0, node3FSInfo), null, null, null, null) }; InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvaiableUsages, newMostAvaiableUsages); DiskUsage leastNode_1 = newLeastAvaiableUsages.get("node_1"); diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java index dba510afa0bea..f925df3e9e05e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java @@ -42,11 +42,14 @@ public void testSimpleJsonFromAndTo() throws IOException { .put(IndexMetaData.builder("test1") .settings(settings(Version.CURRENT)) .numberOfShards(1) - .numberOfReplicas(2)) + .numberOfReplicas(2) + .primaryTerm(0, 1)) .put(IndexMetaData.builder("test2") .settings(settings(Version.CURRENT).put("setting1", "value1").put("setting2", "value2")) .numberOfShards(2) - .numberOfReplicas(3)) + .numberOfReplicas(3) + .primaryTerm(0, 2) + .primaryTerm(1, 2)) .put(IndexMetaData.builder("test3") .settings(settings(Version.CURRENT)) .numberOfShards(1) @@ -113,15 +116,15 @@ public void testSimpleJsonFromAndTo() throws IOException { .putAlias(newAliasMetaDataBuilder("alias1").filter(ALIAS_FILTER1)) .putAlias(newAliasMetaDataBuilder("alias2")) .putAlias(newAliasMetaDataBuilder("alias4").filter(ALIAS_FILTER2))) - .put(IndexTemplateMetaData.builder("foo") - .template("bar") - .order(1) - .settings(settingsBuilder() - .put("setting1", "value1") - .put("setting2", "value2")) - .putAlias(newAliasMetaDataBuilder("alias-bar1")) - .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}")) - .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar"))) + .put(IndexTemplateMetaData.builder("foo") + .template("bar") + .order(1) + .settings(settingsBuilder() + .put("setting1", "value1") + .put("setting2", "value2")) + .putAlias(newAliasMetaDataBuilder("alias-bar1")) + .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}")) + .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar"))) .put(IndexMetaData.builder("test12") .settings(settings(Version.CURRENT) .put("setting1", "value1") @@ -134,15 +137,15 @@ public void testSimpleJsonFromAndTo() throws IOException { .putAlias(newAliasMetaDataBuilder("alias1").filter(ALIAS_FILTER1)) .putAlias(newAliasMetaDataBuilder("alias2")) .putAlias(newAliasMetaDataBuilder("alias4").filter(ALIAS_FILTER2))) - .put(IndexTemplateMetaData.builder("foo") - .template("bar") - .order(1) - .settings(settingsBuilder() - .put("setting1", "value1") - .put("setting2", "value2")) - .putAlias(newAliasMetaDataBuilder("alias-bar1")) - .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}")) - .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar"))) + .put(IndexTemplateMetaData.builder("foo") + .template("bar") + .order(1) + .settings(settingsBuilder() + .put("setting1", "value1") + .put("setting2", "value2")) + .putAlias(newAliasMetaDataBuilder("alias-bar1")) + .putAlias(newAliasMetaDataBuilder("alias-bar2").filter("{\"term\":{\"user\":\"kimchy\"}}")) + .putAlias(newAliasMetaDataBuilder("alias-bar3").routing("routing-bar"))) .build(); String metaDataSource = MetaData.Builder.toXContent(metaData); @@ -151,6 +154,7 @@ public void testSimpleJsonFromAndTo() throws IOException { MetaData parsedMetaData = MetaData.Builder.fromXContent(XContentFactory.xContent(XContentType.JSON).createParser(metaDataSource)); IndexMetaData indexMetaData = parsedMetaData.index("test1"); + assertThat(indexMetaData.primaryTerm(0), equalTo(1)); assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); assertThat(indexMetaData.getCreationDate(), equalTo(-1l)); @@ -160,6 +164,8 @@ public void testSimpleJsonFromAndTo() throws IOException { indexMetaData = parsedMetaData.index("test2"); assertThat(indexMetaData.getNumberOfShards(), equalTo(2)); assertThat(indexMetaData.getNumberOfReplicas(), equalTo(3)); + assertThat(indexMetaData.primaryTerm(0), equalTo(2)); + assertThat(indexMetaData.primaryTerm(1), equalTo(2)); assertThat(indexMetaData.getCreationDate(), equalTo(-1l)); assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5)); assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1")); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java index 262b3db630b75..b0f572c98fa98 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java @@ -31,7 +31,7 @@ public class AllocationIdTests extends ESTestCase { @Test public void testShardToStarted() { logger.info("-- create unassigned shard"); - ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, 1, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); assertThat(shard.allocationId(), nullValue()); logger.info("-- initialize the shard"); @@ -52,7 +52,7 @@ public void testShardToStarted() { @Test public void testSuccessfulRelocation() { logger.info("-- build started shard"); - ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, 1, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); shard.initialize("node1", -1); shard.moveToStarted(); @@ -76,7 +76,7 @@ public void testSuccessfulRelocation() { @Test public void testCancelRelocation() { logger.info("-- build started shard"); - ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, 1, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); shard.initialize("node1", -1); shard.moveToStarted(); @@ -97,7 +97,7 @@ public void testCancelRelocation() { @Test public void testMoveToUnassigned() { logger.info("-- build started shard"); - ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, 1, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); shard.initialize("node1", -1); shard.moveToStarted(); @@ -109,7 +109,7 @@ public void testMoveToUnassigned() { @Test public void testReinitializing() { logger.info("-- build started shard"); - ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned("test", 0, null, 1, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); shard.initialize("node1", -1); shard.moveToStarted(); AllocationId allocationId = shard.allocationId(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java index 2a7ed6a4a93c0..c17deea6c56e7 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.node.DiscoveryNodes.Builder; import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; @@ -33,11 +34,11 @@ import org.junit.Before; import org.junit.Test; +import java.util.*; + import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; public class RoutingTableTests extends ESAllocationTestCase { @@ -56,8 +57,10 @@ public class RoutingTableTests extends ESAllocationTestCase { .build()); private ClusterState clusterState; + private final Map primaryTermsPerIndex = new HashMap<>(); + private final Map versionsPerIndex = new HashMap<>(); + @Override - @Before public void setUp() throws Exception { super.setUp(); this.numberOfShards = randomIntBetween(1, 5); @@ -66,6 +69,7 @@ public void setUp() throws Exception { this.totalNumberOfShards = this.shardsPerIndex * 2; logger.info("Setup test with " + this.numberOfShards + " shards and " + this.numberOfReplicas + " replicas."); this.emptyRoutingTable = new RoutingTable.Builder().build(); + this.primaryTermsPerIndex.clear(); MetaData metaData = MetaData.builder() .put(createIndexMetaData(TEST_INDEX_1)) .put(createIndexMetaData(TEST_INDEX_2)) @@ -75,6 +79,10 @@ public void setUp() throws Exception { .add(new IndexRoutingTable.Builder(TEST_INDEX_1).initializeAsNew(metaData.index(TEST_INDEX_1)).build()) .add(new IndexRoutingTable.Builder(TEST_INDEX_2).initializeAsNew(metaData.index(TEST_INDEX_2)).build()) .build(); + this.versionsPerIndex.clear(); + this.versionsPerIndex.put(TEST_INDEX_1, new long[numberOfShards]); + this.versionsPerIndex.put(TEST_INDEX_2, new long[numberOfShards]); + this.clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(testRoutingTable).build(); } @@ -92,6 +100,22 @@ private void initPrimaries() { this.testRoutingTable = rerouteResult.routingTable(); assertThat(rerouteResult.changed(), is(true)); this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); + versionsPerIndex.keySet().forEach(this::incrementVersion); + } + + private void incrementVersion(String index) { + final long[] versions = versionsPerIndex.get(index); + for (int i = 0; i < versions.length; i++) { + versions[i]++; + } + } + + private void incrementVersion(String index, int shard) { + versionsPerIndex.get(index)[shard]++; + } + + private void incrementPrimaryTerm(String index, int shard) { + primaryTermsPerIndex.get(index)[shard]++; } private void startInitializingShards(String index) { @@ -100,13 +124,58 @@ private void startInitializingShards(String index) { RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.applyStartedShards(this.clusterState, this.clusterState.getRoutingNodes().shardsWithState(index, INITIALIZING)); this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); this.testRoutingTable = rerouteResult.routingTable(); + incrementVersion(index); + } + + private void failSomePrimaries(String index) { + this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build(); + final IndexRoutingTable indexShardRoutingTable = testRoutingTable.index(index); + Set shardIdsToFail = new HashSet<>(); + for (int i = 1 + randomInt(numberOfShards - 1); i > 0; i--) { + shardIdsToFail.add(randomInt(numberOfShards - 1)); + } + logger.info("failing primary shards {} for index [{}]", shardIdsToFail, index); + List failedShards = new ArrayList<>(); + for (int shard : shardIdsToFail) { + failedShards.add(new FailedRerouteAllocation.FailedShard(indexShardRoutingTable.shard(shard).primaryShard(), "test", null)); + incrementPrimaryTerm(index, shard); // the primary failure should increment the primary term; + incrementVersion(index, shard); // version is incremented once when the primary is unassigned + incrementVersion(index, shard); // and another time when the primary flag is set to false + } + RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.applyFailedShards(this.clusterState, failedShards); + this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); + this.testRoutingTable = rerouteResult.routingTable(); } private IndexMetaData.Builder createIndexMetaData(String indexName) { - return new IndexMetaData.Builder(indexName) + primaryTermsPerIndex.put(indexName, new int[numberOfShards]); + final IndexMetaData.Builder builder = new IndexMetaData.Builder(indexName) .settings(DEFAULT_SETTINGS) .numberOfReplicas(this.numberOfReplicas) .numberOfShards(this.numberOfShards); + for (int i = 0; i < numberOfShards; i++) { + builder.primaryTerm(i, randomInt(200)); + primaryTermsPerIndex.get(indexName)[i] = builder.primaryTerm(i); + } + return builder; + } + + private void assertAllVersionAndPrimaryTerm() { + versionsPerIndex.keySet().forEach(this::assertVersionAndPrimaryTerm); + } + + private void assertVersionAndPrimaryTerm(String index) { + final long[] versions = versionsPerIndex.get(index); + final int[] terms = primaryTermsPerIndex.get(index); + final IndexMetaData indexMetaData = clusterState.metaData().index(index); + for (IndexShardRoutingTable shardRoutingTable : this.testRoutingTable.index(index)) { + final int shard = shardRoutingTable.shardId().id(); + for (ShardRouting routing : shardRoutingTable) { + assertThat("wrong version in " + routing, routing.version(), equalTo(versions[shard])); + assertThat("wrong primary term in " + routing, routing.primaryTerm(), equalTo(terms[shard])); + } + assertThat(indexMetaData.primaryTerm(shard), equalTo(terms[shard])); + } } @Test @@ -168,6 +237,28 @@ public void testShardsWithState() { assertThat(this.testRoutingTable.shardsWithState(ShardRoutingState.STARTED).size(), is(this.totalNumberOfShards)); } + @Test + public void testVersionAndPrimaryTermNormalization() { + assertAllVersionAndPrimaryTerm(); + + initPrimaries(); + assertAllVersionAndPrimaryTerm(); + + startInitializingShards(TEST_INDEX_1); + assertAllVersionAndPrimaryTerm(); + + startInitializingShards(TEST_INDEX_2); + assertAllVersionAndPrimaryTerm(); + + // now start all replicas too + startInitializingShards(TEST_INDEX_1); + startInitializingShards(TEST_INDEX_2); + assertAllVersionAndPrimaryTerm(); + + failSomePrimaries(TEST_INDEX_1); + assertAllVersionAndPrimaryTerm(); + } + @Test public void testActivePrimaryShardsGrouped() { assertThat(this.emptyRoutingTable.activePrimaryShardsGrouped(new String[0], true).size(), is(0)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java index 2139cc29ae1ba..dfd7230b6a5e3 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java @@ -53,6 +53,6 @@ public static void moveToUnassigned(ShardRouting routing, UnassignedInfo info) { } public static ShardRouting newWithRestoreSource(ShardRouting routing, RestoreSource restoreSource) { - return new ShardRouting(routing.index(), routing.shardId().id(), routing.currentNodeId(), routing.relocatingNodeId(), restoreSource, routing.primary(), routing.state(), routing.version(), routing.unassignedInfo(), routing.allocationId(), true, routing.getExpectedShardSize()); + return new ShardRouting(routing.index(), routing.shardId().id(), routing.currentNodeId(), routing.relocatingNodeId(), restoreSource, routing.primaryTerm(), routing.primary(), routing.state(), routing.version(), routing.unassignedInfo(), routing.allocationId(), true, routing.getExpectedShardSize()); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java index 146e80c766552..947e8cb6d0d36 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java @@ -30,10 +30,13 @@ import java.io.IOException; +import static org.hamcrest.Matchers.equalTo; + public class ShardRoutingTests extends ESTestCase { public void testFrozenAfterRead() throws IOException { - ShardRouting routing = TestShardRouting.newShardRouting("foo", 1, "node_1", null, null, false, ShardRoutingState.INITIALIZING, 1); + int term = randomInt(200); + ShardRouting routing = TestShardRouting.newShardRouting("foo", 1, "node_1", null, null, term, false, ShardRoutingState.INITIALIZING, 1); routing.moveToPrimary(); assertTrue(routing.primary()); routing.moveFromPrimary(); @@ -49,11 +52,23 @@ public void testFrozenAfterRead() throws IOException { } } + public void testPrimaryTermIncrement() { + int term = randomInt(200); + ShardRouting routing = TestShardRouting.newShardRouting("foo", 1, "node_1", null, null, term, false, ShardRoutingState.STARTED, 1); + routing.moveToPrimary(); + assertTrue(routing.primary()); + assertThat(routing.primaryTerm(), equalTo(term + 1)); + routing.moveFromPrimary(); + assertFalse(routing.primary()); + assertThat(routing.primaryTerm(), equalTo(term + 1)); + } + public void testIsSameAllocation() { - ShardRouting unassignedShard0 = TestShardRouting.newShardRouting("test", 0, null, false, ShardRoutingState.UNASSIGNED, 1); - ShardRouting unassignedShard1 = TestShardRouting.newShardRouting("test", 1, null, false, ShardRoutingState.UNASSIGNED, 1); - ShardRouting initializingShard0 = TestShardRouting.newShardRouting("test", 0, "1", randomBoolean(), ShardRoutingState.INITIALIZING, 1); - ShardRouting initializingShard1 = TestShardRouting.newShardRouting("test", 1, "1", randomBoolean(), ShardRoutingState.INITIALIZING, 1); + int term = randomInt(200); + ShardRouting unassignedShard0 = TestShardRouting.newShardRouting("test", 0, null, term, false, ShardRoutingState.UNASSIGNED, 1); + ShardRouting unassignedShard1 = TestShardRouting.newShardRouting("test", 1, null, term, false, ShardRoutingState.UNASSIGNED, 1); + ShardRouting initializingShard0 = TestShardRouting.newShardRouting("test", 0, "1", term, randomBoolean(), ShardRoutingState.INITIALIZING, 1); + ShardRouting initializingShard1 = TestShardRouting.newShardRouting("test", 1, "1", term, randomBoolean(), ShardRoutingState.INITIALIZING, 1); ShardRouting startedShard0 = new ShardRouting(initializingShard0); startedShard0.moveToStarted(); ShardRouting startedShard1 = new ShardRouting(initializingShard1); @@ -91,13 +106,14 @@ public void testIsSameShard() { private ShardRouting randomShardRouting(String index, int shard) { ShardRoutingState state = randomFrom(ShardRoutingState.values()); - return TestShardRouting.newShardRouting(index, shard, state == ShardRoutingState.UNASSIGNED ? null : "1", state != ShardRoutingState.UNASSIGNED && randomBoolean(), state, randomInt(5)); + return TestShardRouting.newShardRouting(index, shard, state == ShardRoutingState.UNASSIGNED ? null : "1", randomInt(200), + state != ShardRoutingState.UNASSIGNED && randomBoolean(), state, randomInt(5)); } public void testIsSourceTargetRelocation() { - ShardRouting unassignedShard0 = TestShardRouting.newShardRouting("test", 0, null, false, ShardRoutingState.UNASSIGNED, 1); - ShardRouting initializingShard0 = TestShardRouting.newShardRouting("test", 0, "node1", randomBoolean(), ShardRoutingState.INITIALIZING, 1); - ShardRouting initializingShard1 = TestShardRouting.newShardRouting("test", 1, "node1", randomBoolean(), ShardRoutingState.INITIALIZING, 1); + ShardRouting unassignedShard0 = TestShardRouting.newShardRouting("test", 0, null, randomInt(200), false, ShardRoutingState.UNASSIGNED, 1); + ShardRouting initializingShard0 = TestShardRouting.newShardRouting("test", 0, "node1", randomInt(200), randomBoolean(), ShardRoutingState.INITIALIZING, 1); + ShardRouting initializingShard1 = TestShardRouting.newShardRouting("test", 1, "node1", randomInt(200), randomBoolean(), ShardRoutingState.INITIALIZING, 1); ShardRouting startedShard0 = new ShardRouting(initializingShard0); startedShard0.moveToStarted(); ShardRouting startedShard1 = new ShardRouting(initializingShard1); @@ -139,13 +155,14 @@ public void testIsSourceTargetRelocation() { assertFalse(startedShard0.isRelocationSourceOf(sourceShard0a)); } - public void testEqualsIgnoringVersion() { + public void testEqualsIgnoringMetaData() { ShardRouting routing = randomShardRouting("test", 0); ShardRouting otherRouting = new ShardRouting(routing); - assertTrue("expected equality\nthis " + routing + ",\nother " + otherRouting, routing.equalsIgnoringMetaData(otherRouting)); - otherRouting = new ShardRouting(routing, 1); + otherRouting = new ShardRouting(routing, + randomBoolean() ? routing.version() : routing.version() + 1, + randomBoolean() ? routing.primaryTerm() : routing.primaryTerm() + 1); assertTrue("expected equality\nthis " + routing + ",\nother " + otherRouting, routing.equalsIgnoringMetaData(otherRouting)); @@ -155,36 +172,42 @@ public void testEqualsIgnoringVersion() { switch (changeId) { case 0: // change index - otherRouting = TestShardRouting.newShardRouting(otherRouting.index() + "a", otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(), - otherRouting.restoreSource(), otherRouting.primary(), otherRouting.state(), otherRouting.version(), otherRouting.unassignedInfo()); + otherRouting = TestShardRouting.newShardRouting(otherRouting.index() + "a", otherRouting.id(), otherRouting.currentNodeId(), + otherRouting.relocatingNodeId(), otherRouting.restoreSource(), otherRouting.primaryTerm(), otherRouting.primary(), + otherRouting.state(), otherRouting.version(), otherRouting.unassignedInfo()); break; case 1: // change shard id - otherRouting = TestShardRouting.newShardRouting(otherRouting.index(), otherRouting.id() + 1, otherRouting.currentNodeId(), otherRouting.relocatingNodeId(), - otherRouting.restoreSource(), otherRouting.primary(), otherRouting.state(), otherRouting.version(), otherRouting.unassignedInfo()); + otherRouting = TestShardRouting.newShardRouting(otherRouting.index(), otherRouting.id() + 1, otherRouting.currentNodeId(), + otherRouting.relocatingNodeId(), otherRouting.restoreSource(), otherRouting.primaryTerm(), otherRouting.primary(), + otherRouting.state(), otherRouting.version(), otherRouting.unassignedInfo()); break; case 2: // change current node otherRouting = TestShardRouting.newShardRouting(otherRouting.index(), otherRouting.id(), otherRouting.currentNodeId() == null ? "1" : otherRouting.currentNodeId() + "_1", otherRouting.relocatingNodeId(), - otherRouting.restoreSource(), otherRouting.primary(), otherRouting.state(), otherRouting.version(), otherRouting.unassignedInfo()); + otherRouting.restoreSource(), otherRouting.primaryTerm(), otherRouting.primary(), otherRouting.state(), + otherRouting.version(), otherRouting.unassignedInfo()); break; case 3: // change relocating node otherRouting = TestShardRouting.newShardRouting(otherRouting.index(), otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId() == null ? "1" : otherRouting.relocatingNodeId() + "_1", - otherRouting.restoreSource(), otherRouting.primary(), otherRouting.state(), otherRouting.version(), otherRouting.unassignedInfo()); + otherRouting.restoreSource(), otherRouting.primaryTerm(), otherRouting.primary(), otherRouting.state(), + otherRouting.version(), otherRouting.unassignedInfo()); break; case 4: // change restore source otherRouting = TestShardRouting.newShardRouting(otherRouting.index(), otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(), otherRouting.restoreSource() == null ? new RestoreSource(new SnapshotId("test", "s1"), Version.CURRENT, "test") : new RestoreSource(otherRouting.restoreSource().snapshotId(), Version.CURRENT, otherRouting.index() + "_1"), - otherRouting.primary(), otherRouting.state(), otherRouting.version(), otherRouting.unassignedInfo()); + otherRouting.primaryTerm(), otherRouting.primary(), otherRouting.state(), otherRouting.version(), + otherRouting.unassignedInfo()); break; case 5: // change primary flag - otherRouting = TestShardRouting.newShardRouting(otherRouting.index(), otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(), - otherRouting.restoreSource(), otherRouting.primary() == false, otherRouting.state(), otherRouting.version(), otherRouting.unassignedInfo()); + otherRouting = TestShardRouting.newShardRouting(otherRouting.index(), otherRouting.id(), otherRouting.currentNodeId(), + otherRouting.relocatingNodeId(), otherRouting.restoreSource(), otherRouting.primaryTerm(), + otherRouting.primary() == false, otherRouting.state(), otherRouting.version(), otherRouting.unassignedInfo()); break; case 6: // change state @@ -198,20 +221,26 @@ public void testEqualsIgnoringVersion() { unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test"); } - otherRouting = TestShardRouting.newShardRouting(otherRouting.index(), otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(), - otherRouting.restoreSource(), otherRouting.primary(), newState, otherRouting.version(), unassignedInfo); + otherRouting = TestShardRouting.newShardRouting(otherRouting.index(), otherRouting.id(), otherRouting.currentNodeId(), + otherRouting.relocatingNodeId(), otherRouting.restoreSource(), otherRouting.primaryTerm(), otherRouting.primary(), + newState, otherRouting.version(), unassignedInfo); break; } if (randomBoolean()) { // change version - otherRouting = new ShardRouting(otherRouting, otherRouting.version() + 1); + otherRouting = new ShardRouting(otherRouting, otherRouting.version() + 1, otherRouting.primaryTerm()); + } + if (randomBoolean()) { + // increase term + otherRouting = new ShardRouting(otherRouting, otherRouting.version(), otherRouting.primaryTerm() + 1); } if (randomBoolean()) { // change unassigned info - otherRouting = TestShardRouting.newShardRouting(otherRouting.index(), otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(), - otherRouting.restoreSource(), otherRouting.primary(), otherRouting.state(), otherRouting.version(), + otherRouting = TestShardRouting.newShardRouting(otherRouting.index(), otherRouting.id(), otherRouting.currentNodeId(), + otherRouting.relocatingNodeId(), otherRouting.restoreSource(), otherRouting.primaryTerm(), otherRouting.primary(), + otherRouting.state(), otherRouting.version(), otherRouting.unassignedInfo() == null ? new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test") : new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, otherRouting.unassignedInfo().getMessage() + "_1")); } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/core/src/test/java/org/elasticsearch/cluster/routing/TestShardRouting.java index df9e1f8af24ef..0dae058f7ba62 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/TestShardRouting.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/TestShardRouting.java @@ -27,26 +27,31 @@ */ public class TestShardRouting { - public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, boolean primary, ShardRoutingState state, long version) { - return new ShardRouting(index, shardId, currentNodeId, null, null, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1); + public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, int primaryTerm, boolean primary, + ShardRoutingState state, long version) { + return new ShardRouting(index, shardId, currentNodeId, null, null, primaryTerm, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1); } - public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state, long version) { - return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1); + public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, + int primaryTerm, boolean primary, ShardRoutingState state, long version) { + return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primaryTerm, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1); } - public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state, AllocationId allocationId, long version) { - return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primary, state, version, buildUnassignedInfo(state), allocationId, true, -1); + public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, + int primaryTerm, boolean primary, ShardRoutingState state, AllocationId allocationId, long version) { + return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primaryTerm, primary, state, version, buildUnassignedInfo(state), allocationId, true, -1); } - public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state, long version) { - return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1); + public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, + RestoreSource restoreSource, int primaryTerm, boolean primary, + ShardRoutingState state, long version) { + return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primaryTerm, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1); } public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, - String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state, long version, - UnassignedInfo unassignedInfo) { - return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, version, unassignedInfo, buildAllocationId(state), true, -1); + String relocatingNodeId, RestoreSource restoreSource, int primaryTerm, boolean primary, + ShardRoutingState state, long version, UnassignedInfo unassignedInfo) { + return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primaryTerm, primary, state, version, unassignedInfo, buildAllocationId(state), true, -1); } private static AllocationId buildAllocationId(ShardRoutingState state) { diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java index fc7ce64c082cd..549f157241c7a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java @@ -197,7 +197,7 @@ public void testReplicaAdded() { */ @Test public void testStateTransitionMetaHandling() { - ShardRouting shard = TestShardRouting.newShardRouting("test", 1, null, null, null, true, ShardRoutingState.UNASSIGNED, 1, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = TestShardRouting.newShardRouting("test", 1, null, null, null, 1, true, ShardRoutingState.UNASSIGNED, 1, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); ShardRouting mutable = new ShardRouting(shard); assertThat(mutable.unassignedInfo(), notNullValue()); mutable.initialize("test_node", -1); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java index 6253febaec893..f39f094bae3f8 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java @@ -19,7 +19,6 @@ package org.elasticsearch.cluster.routing.allocation; -import java.nio.charset.StandardCharsets; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -32,13 +31,14 @@ import java.io.BufferedReader; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.common.settings.Settings.settingsBuilder; /** @@ -75,7 +75,7 @@ public void run() throws IOException { ShardRoutingState state = ShardRoutingState.valueOf(matcher.group(4)); String ip = matcher.group(5); nodes.add(ip); - ShardRouting routing = TestShardRouting.newShardRouting(index, shard, ip, null, null, primary, state, 1); + ShardRouting routing = TestShardRouting.newShardRouting(index, shard, ip, null, null, 1, primary, state, 1); idx.add(routing); logger.debug("Add routing {}", routing); } else { diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java index 1e8a5fbc1ec16..66b43d43db2b1 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java @@ -54,9 +54,9 @@ public void tesStartedShardsMatching() { .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))) .metaData(MetaData.builder().put(indexMetaData, false)); - final ShardRouting initShard = TestShardRouting.newShardRouting("test", 0, "node1", randomBoolean(), ShardRoutingState.INITIALIZING, 1); - final ShardRouting startedShard = TestShardRouting.newShardRouting("test", 1, "node2", randomBoolean(), ShardRoutingState.STARTED, 1); - final ShardRouting relocatingShard = TestShardRouting.newShardRouting("test", 2, "node1", "node2", randomBoolean(), ShardRoutingState.RELOCATING, 1); + final ShardRouting initShard = TestShardRouting.newShardRouting("test", 0, "node1", 1, randomBoolean(), ShardRoutingState.INITIALIZING, 1); + final ShardRouting startedShard = TestShardRouting.newShardRouting("test", 1, "node2", 1, randomBoolean(), ShardRoutingState.STARTED, 1); + final ShardRouting relocatingShard = TestShardRouting.newShardRouting("test", 2, "node1", "node2", 1, randomBoolean(), ShardRoutingState.RELOCATING, 1); stateBuilder.routingTable(RoutingTable.builder().add(IndexRoutingTable.builder("test") .addIndexShard(new IndexShardRoutingTable.Builder(initShard.shardId()).addShard(initShard).build()) .addIndexShard(new IndexShardRoutingTable.Builder(startedShard.shardId()).addShard(startedShard).build()) @@ -67,7 +67,8 @@ public void tesStartedShardsMatching() { logger.info("--> test starting of shard"); RoutingAllocation.Result result = allocation.applyStartedShards(state, Arrays.asList( - TestShardRouting.newShardRouting(initShard.index(), initShard.id(), initShard.currentNodeId(), initShard.relocatingNodeId(), initShard.primary(), + TestShardRouting.newShardRouting(initShard.index(), initShard.id(), initShard.currentNodeId(), initShard.relocatingNodeId(), + initShard.primaryTerm(), initShard.primary(), ShardRoutingState.INITIALIZING, initShard.allocationId(), randomInt())), false); assertTrue("failed to start " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); assertTrue(initShard + "isn't started \ncurrent routing table:" + result.routingTable().prettyPrint(), @@ -77,29 +78,32 @@ public void tesStartedShardsMatching() { logger.info("--> testing shard variants that shouldn't match the initializing shard"); result = allocation.applyStartedShards(state, Arrays.asList( - TestShardRouting.newShardRouting(initShard.index(), initShard.id(), initShard.currentNodeId(), initShard.relocatingNodeId(), initShard.primary(), + TestShardRouting.newShardRouting(initShard.index(), initShard.id(), initShard.currentNodeId(), initShard.relocatingNodeId(), + initShard.primaryTerm(), initShard.primary(), ShardRoutingState.INITIALIZING, 1)), false); assertFalse("wrong allocation id flag shouldn't start shard " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); result = allocation.applyStartedShards(state, Arrays.asList( - TestShardRouting.newShardRouting(initShard.index(), initShard.id(), "some_node", initShard.currentNodeId(), initShard.primary(), + TestShardRouting.newShardRouting(initShard.index(), initShard.id(), "some_node", initShard.currentNodeId(), + initShard.primaryTerm(), initShard.primary(), ShardRoutingState.INITIALIZING, AllocationId.newTargetRelocation(AllocationId.newRelocation(initShard.allocationId())) , 1)), false); assertFalse("relocating shard from node shouldn't start shard " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); - logger.info("--> testing double starting"); result = allocation.applyStartedShards(state, Arrays.asList( - TestShardRouting.newShardRouting(startedShard.index(), startedShard.id(), startedShard.currentNodeId(), startedShard.relocatingNodeId(), startedShard.primary(), + TestShardRouting.newShardRouting(startedShard.index(), startedShard.id(), startedShard.currentNodeId(), startedShard.relocatingNodeId(), + startedShard.primaryTerm(), startedShard.primary(), ShardRoutingState.INITIALIZING, startedShard.allocationId(), 1)), false); assertFalse("duplicate starting of the same shard should be ignored \ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); logger.info("--> testing starting of relocating shards"); final AllocationId targetAllocationId = AllocationId.newTargetRelocation(relocatingShard.allocationId()); result = allocation.applyStartedShards(state, Arrays.asList( - TestShardRouting.newShardRouting(relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), relocatingShard.currentNodeId(), relocatingShard.primary(), + TestShardRouting.newShardRouting(relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), + relocatingShard.currentNodeId(), relocatingShard.primaryTerm(), relocatingShard.primary(), ShardRoutingState.INITIALIZING, targetAllocationId, randomInt())), false); assertTrue("failed to start " + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); @@ -111,12 +115,14 @@ public void tesStartedShardsMatching() { logger.info("--> testing shard variants that shouldn't match the initializing relocating shard"); result = allocation.applyStartedShards(state, Arrays.asList( - TestShardRouting.newShardRouting(relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), relocatingShard.currentNodeId(), relocatingShard.primary(), + TestShardRouting.newShardRouting(relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), + relocatingShard.currentNodeId(), relocatingShard.primaryTerm(), relocatingShard.primary(), ShardRoutingState.INITIALIZING, relocatingShard.version()))); assertFalse("wrong allocation id shouldn't start shard" + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); result = allocation.applyStartedShards(state, Arrays.asList( - TestShardRouting.newShardRouting(relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), relocatingShard.currentNodeId(), relocatingShard.primary(), + TestShardRouting.newShardRouting(relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), + relocatingShard.currentNodeId(), relocatingShard.primaryTerm(), relocatingShard.primary(), ShardRoutingState.INITIALIZING, relocatingShard.allocationId(), randomInt())), false); assertFalse("wrong allocation id shouldn't start shard even if relocatingId==shard.id" + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 525c4465363e3..86d2493216f98 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -29,14 +29,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators; @@ -56,14 +49,9 @@ import java.util.HashSet; import java.util.Map; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; -import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; public class DiskThresholdDeciderTests extends ESAllocationTestCase { @@ -126,8 +114,8 @@ public void addListener(Listener listener) { logger.info("--> adding two nodes"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder() - .put(newNode("node1")) - .put(newNode("node2")) + .put(newNode("node1")) + .put(newNode("node2")) ).build(); routingTable = strategy.reroute(clusterState).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); @@ -157,7 +145,7 @@ public void addListener(Listener listener) { logger.info("--> adding node3"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) - .put(newNode("node3")) + .put(newNode("node3")) ).build(); routingTable = strategy.reroute(clusterState).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); @@ -244,7 +232,7 @@ public void addListener(Listener listener) { logger.info("--> adding node4"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) - .put(newNode("node4")) + .put(newNode("node4")) ).build(); routingTable = strategy.reroute(clusterState).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); @@ -393,7 +381,7 @@ public void addListener(Listener listener) { logger.info("--> adding node3"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) - .put(newNode("node3")) + .put(newNode("node3")) ).build(); routingTable = strategy.reroute(clusterState).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); @@ -480,7 +468,7 @@ public void addListener(Listener listener) { logger.info("--> adding node4"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) - .put(newNode("node4")) + .put(newNode("node4")) ).build(); routingTable = strategy.reroute(clusterState).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); @@ -853,8 +841,8 @@ public void testCanRemainWithShardRelocatingAway() { .build(); // Two shards consuming each 80% of disk space while 70% is allowed, so shard 0 isn't allowed here - ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, null, true, ShardRoutingState.STARTED, 1); - ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", null, null, true, ShardRoutingState.STARTED, 1); + ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, null, 1, true, ShardRoutingState.STARTED, 1); + ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", null, null, 1, true, ShardRoutingState.STARTED, 1); RoutingNode firstRoutingNode = new RoutingNode("node1", discoveryNode1, Arrays.asList(firstRouting, secondRouting)); RoutingTable.Builder builder = RoutingTable.builder().add( IndexRoutingTable.builder("test") @@ -873,8 +861,8 @@ public void testCanRemainWithShardRelocatingAway() { assertThat(decision.type(), equalTo(Decision.Type.NO)); // Two shards consuming each 80% of disk space while 70% is allowed, but one is relocating, so shard 0 can stay - firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, null, true, ShardRoutingState.STARTED, 1); - secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", "node2", null, true, ShardRoutingState.RELOCATING, 1); + firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, null, 1, true, ShardRoutingState.STARTED, 1); + secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", "node2", null, 1, true, ShardRoutingState.RELOCATING, 1); firstRoutingNode = new RoutingNode("node1", discoveryNode1, Arrays.asList(firstRouting, secondRouting)); builder = RoutingTable.builder().add( IndexRoutingTable.builder("test") @@ -906,7 +894,7 @@ public void addListener(Listener listener) { } }; AllocationDeciders deciders = new AllocationDeciders(Settings.EMPTY, new HashSet<>(Arrays.asList( - new SameShardAllocationDecider(Settings.EMPTY), diskThresholdDecider + new SameShardAllocationDecider(Settings.EMPTY), diskThresholdDecider ))); AllocationService strategy = new AllocationService(settingsBuilder() .put("cluster.routing.allocation.concurrent_recoveries", 10) @@ -971,8 +959,8 @@ public void testForSingleDataNode() { .build(); // Two shards consumes 80% of disk space in data node, but we have only one data node, shards should remain. - ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, null, true, ShardRoutingState.STARTED, 1); - ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", null, null, true, ShardRoutingState.STARTED, 1); + ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, null, 1, true, ShardRoutingState.STARTED, 1); + ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", null, null, 1, true, ShardRoutingState.STARTED, 1); RoutingNode firstRoutingNode = new RoutingNode("node2", discoveryNode2, Arrays.asList(firstRouting, secondRouting)); RoutingTable.Builder builder = RoutingTable.builder().add( @@ -1028,8 +1016,8 @@ public void addListener(Listener listener) { ClusterState updateClusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .put(discoveryNode3)).build(); - firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, null, true, ShardRoutingState.STARTED, 1); - secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", "node3", null, true, ShardRoutingState.RELOCATING, 1); + firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, null, 1, true, ShardRoutingState.STARTED, 1); + secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", "node3", null, 1, true, ShardRoutingState.RELOCATING, 1); firstRoutingNode = new RoutingNode("node2", discoveryNode2, Arrays.asList(firstRouting, secondRouting)); builder = RoutingTable.builder().add( IndexRoutingTable.builder("test") diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index 5417a9b85f3e1..809990cb66228 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -20,21 +20,13 @@ package org.elasticsearch.cluster.routing.allocation.decider; import org.elasticsearch.Version; -import org.elasticsearch.cluster.ClusterInfo; -import org.elasticsearch.cluster.ClusterInfoService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.DiskUsage; -import org.elasticsearch.cluster.EmptyClusterInfoService; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.MockInternalClusterInfoService.DevNullClusterInfo; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingHelper; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; @@ -101,7 +93,7 @@ public void testCanAllocateUsesMaxAvailableSpace() { ClusterInfoService cis = EmptyClusterInfoService.INSTANCE; DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss, cis, null); - ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, 1, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); DiscoveryNode node_0 = new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, Version.CURRENT); DiscoveryNode node_1 = new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Version.CURRENT); @@ -146,22 +138,22 @@ public void testCanRemainUsesLeastAvailableSpace() { DiscoveryNode node_0 = new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, Version.CURRENT); DiscoveryNode node_1 = new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Version.CURRENT); - ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, 1, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); ShardRoutingHelper.initialize(test_0, node_0.getId()); ShardRoutingHelper.moveToStarted(test_0); shardRoutingMap.put(test_0, "/node0/least"); - ShardRouting test_1 = ShardRouting.newUnassigned("test", 1, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_1 = ShardRouting.newUnassigned("test", 1, null, 1, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); ShardRoutingHelper.initialize(test_1, node_1.getId()); ShardRoutingHelper.moveToStarted(test_1); shardRoutingMap.put(test_1, "/node1/least"); - ShardRouting test_2 = ShardRouting.newUnassigned("test", 2, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_2 = ShardRouting.newUnassigned("test", 2, null, 1, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); ShardRoutingHelper.initialize(test_2, node_1.getId()); ShardRoutingHelper.moveToStarted(test_2); shardRoutingMap.put(test_2, "/node1/most"); - ShardRouting test_3 = ShardRouting.newUnassigned("test", 3, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_3 = ShardRouting.newUnassigned("test", 3, null, 1, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); ShardRoutingHelper.initialize(test_3, node_1.getId()); ShardRoutingHelper.moveToStarted(test_3); // Intentionally not in the shardRoutingMap. We want to test what happens when we don't know where it is. @@ -226,17 +218,17 @@ public void testShardSizeAndRelocatingSize() { shardSizes.put("[test][2][r]", 1000L); shardSizes.put("[other][0][p]", 10000L); ClusterInfo info = new DevNullClusterInfo(ImmutableOpenMap.of(), ImmutableOpenMap.of(), shardSizes.build()); - ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_0 = ShardRouting.newUnassigned("test", 0, null, 1, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); ShardRoutingHelper.initialize(test_0, "node1"); ShardRoutingHelper.moveToStarted(test_0); ShardRoutingHelper.relocate(test_0, "node2"); - ShardRouting test_1 = ShardRouting.newUnassigned("test", 1, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_1 = ShardRouting.newUnassigned("test", 1, null, 1, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); ShardRoutingHelper.initialize(test_1, "node2"); ShardRoutingHelper.moveToStarted(test_1); ShardRoutingHelper.relocate(test_1, "node1"); - ShardRouting test_2 = ShardRouting.newUnassigned("test", 2, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_2 = ShardRouting.newUnassigned("test", 2, null, 1, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); ShardRoutingHelper.initialize(test_2, "node1"); ShardRoutingHelper.moveToStarted(test_2); @@ -250,13 +242,13 @@ public void testShardSizeAndRelocatingSize() { assertEquals(0l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/some/other/dev")); assertEquals(0l, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/some/other/dev")); - ShardRouting test_3 = ShardRouting.newUnassigned("test", 3, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_3 = ShardRouting.newUnassigned("test", 3, null, 1, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); ShardRoutingHelper.initialize(test_3, "node1"); ShardRoutingHelper.moveToStarted(test_3); assertEquals(0l, DiskThresholdDecider.getShardSize(test_3, info)); - ShardRouting other_0 = ShardRouting.newUnassigned("other", 0, null, randomBoolean(), new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting other_0 = ShardRouting.newUnassigned("other", 0, null, 1, randomBoolean(), new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); ShardRoutingHelper.initialize(other_0, "node2"); ShardRoutingHelper.moveToStarted(other_0); ShardRoutingHelper.relocate(other_0, "node1"); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/ShardStateIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/ShardStateIT.java new file mode 100644 index 0000000000000..f3ea59c973a27 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/ShardStateIT.java @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.routing.allocation.decider; + +import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.test.ESIntegTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class ShardStateIT extends ESIntegTestCase { + + public void testPrimaryFailureIncreasesTerm() throws Exception { + internalCluster().ensureAtLeastNumDataNodes(2); + prepareCreate("test").setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).get(); + ensureGreen(); + ClusterState state = client().admin().cluster().prepareState().get().getState(); + IndexMetaData metaData = state.metaData().index("test"); + assertThat(metaData.primaryTerm(0), equalTo(0)); + assertThat(metaData.primaryTerm(1), equalTo(0)); + + logger.info("--> disabling allocation to capture shard failure"); + disableAllocation("test"); + + final int shard = randomBoolean() ? 0 : 1; + final String nodeId = state.routingTable().index("test").shard(shard).primaryShard().currentNodeId(); + final String node = state.nodes().get(nodeId).name(); + logger.info("--> failing primary of [{}] on node [{}]", shard, node); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + indicesService.indexService("test").getShard(shard).failShard("simulated test failure", null); + + logger.info("--> waiting for a yellow index"); + assertBusy(() -> assertThat(client().admin().cluster().prepareHealth().get().getStatus(), equalTo(ClusterHealthStatus.YELLOW))); + + state = client().admin().cluster().prepareState().get().getState(); + metaData = state.metaData().index("test"); + assertThat(metaData.primaryTerm(shard), equalTo(1)); + assertThat(metaData.primaryTerm(shard ^ 1), equalTo(0)); + + logger.info("--> enabling allocation"); + enableAllocation("test"); + ensureGreen(); + state = client().admin().cluster().prepareState().get().getState(); + metaData = state.metaData().index("test"); + assertThat(metaData.primaryTerm(shard), equalTo(1)); + assertThat(metaData.primaryTerm(shard ^ 1), equalTo(0)); + } +} diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java index ce6a8b0a6e638..7aad4492e51bd 100644 --- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java @@ -66,13 +66,13 @@ public void buildTestAllocator() { */ @Test public void testNoProcessReplica() { - ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, false, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)); + ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, 1, false, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)); assertThat(testAllocator.needToFindPrimaryCopy(shard), equalTo(false)); } @Test public void testNoProcessPrimayNotAllcoatedBefore() { - ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, true, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, 1, true, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); assertThat(testAllocator.needToFindPrimaryCopy(shard), equalTo(false)); } @@ -281,7 +281,7 @@ public void testEnoughCopiesFoundForAllocationWithDifferentVersion() { @Test public void testAllocationOnAnyNodeWithSharedFs() { - ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, false, + ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, 1, false, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)); @@ -307,7 +307,7 @@ public void testAllocationOnAnyNodeWithSharedFs() { @Test public void testAllocationOnAnyNodeShouldPutNodesWithExceptionsLast() { - ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, false, + ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, 1, false, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)); diff --git a/core/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java index 88499bf96cd9e..9a114bd11d5fa 100644 --- a/core/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/PriorityComparatorTests.java @@ -30,8 +30,9 @@ public class PriorityComparatorTests extends ESTestCase { public void testPreferNewIndices() { RoutingNodes.UnassignedShards shards = new RoutingNodes.UnassignedShards((RoutingNodes) null); List shardRoutings = Arrays.asList(TestShardRouting.newShardRouting("oldest", 0, null, null, null, - randomBoolean(), ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")), TestShardRouting.newShardRouting("newest", 0, null, null, null, - randomBoolean(), ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar"))); + 1, randomBoolean(), ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")), + TestShardRouting.newShardRouting("newest", 0, null, null, null, + 1, randomBoolean(), ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar"))); Collections.shuffle(shardRoutings, random()); for (ShardRouting routing : shardRoutings) { shards.add(routing); @@ -59,9 +60,10 @@ protected Settings getIndexSettings(String index) { public void testPreferPriorityIndices() { RoutingNodes.UnassignedShards shards = new RoutingNodes.UnassignedShards((RoutingNodes) null); - List shardRoutings = Arrays.asList(TestShardRouting.newShardRouting("oldest", 0, null, null, null, - randomBoolean(), ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")), TestShardRouting.newShardRouting("newest", 0, null, null, null, - randomBoolean(), ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar"))); + List shardRoutings = Arrays.asList(TestShardRouting.newShardRouting("oldest", 0, null, null, null, 1, + randomBoolean(), ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar")), + TestShardRouting.newShardRouting("newest", 0, null, null, null, 1, + randomBoolean(), ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar"))); Collections.shuffle(shardRoutings, random()); for (ShardRouting routing : shardRoutings) { shards.add(routing); @@ -97,15 +99,16 @@ public void testPriorityComparatorSort() { if (frequently()) { indices[i] = new IndexMeta("idx_2015_04_" + String.format(Locale.ROOT, "%02d", i), randomIntBetween(1, 1000), randomIntBetween(1, 10000)); } else { // sometimes just use defaults - indices[i] = new IndexMeta("idx_2015_04_" + String.format(Locale.ROOT, "%02d", i)); + indices[i] = new IndexMeta("idx_2015_04_" + String.format(Locale.ROOT, "%02d", i)); } map.put(indices[i].name, indices[i]); } int numShards = randomIntBetween(10, 100); for (int i = 0; i < numShards; i++) { IndexMeta indexMeta = randomFrom(indices); - shards.add(TestShardRouting.newShardRouting(indexMeta.name, randomIntBetween(1, 5), null, null, null, - randomBoolean(), ShardRoutingState.UNASSIGNED, randomIntBetween(0, 100), new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar"))); + shards.add(TestShardRouting.newShardRouting(indexMeta.name, randomIntBetween(1, 5), null, null, null, 1, + randomBoolean(), ShardRoutingState.UNASSIGNED, randomIntBetween(0, 100), + new UnassignedInfo(randomFrom(UnassignedInfo.Reason.values()), "foobar"))); } shards.sort(new PriorityComparator() { @Override @@ -128,7 +131,7 @@ protected Settings getIndexSettings(String index) { assertTrue("creationDate mismatch, expected:" + currentMeta.creationDate + " after " + prevMeta.creationDate, prevMeta.creationDate > currentMeta.creationDate); } } else { - assertTrue("priority mismatch, expected:" + currentMeta.priority + " after " + prevMeta.priority, prevMeta.priority > currentMeta.priority); + assertTrue("priority mismatch, expected:" + currentMeta.priority + " after " + prevMeta.priority, prevMeta.priority > currentMeta.priority); } } previous = routing; diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java index e692b620d2f4f..82b0d860ae09c 100644 --- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java @@ -304,8 +304,8 @@ private RoutingAllocation onePrimaryOnNode1And1Replica(AllocationDeciders decide RoutingTable routingTable = RoutingTable.builder() .add(IndexRoutingTable.builder(shardId.getIndex()) .addIndexShard(new IndexShardRoutingTable.Builder(shardId) - .addShard(TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node1.id(), true, ShardRoutingState.STARTED, 10)) - .addShard(ShardRouting.newUnassigned(shardId.getIndex(), shardId.getId(), null, false, new UnassignedInfo(reason, null))) + .addShard(TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node1.id(), 1, true, ShardRoutingState.STARTED, 10)) + .addShard(ShardRouting.newUnassigned(shardId.getIndex(), shardId.getId(), null, 1, false, new UnassignedInfo(reason, null))) .build()) ) .build(); @@ -323,8 +323,8 @@ private RoutingAllocation onePrimaryOnNode1And1ReplicaRecovering(AllocationDecid RoutingTable routingTable = RoutingTable.builder() .add(IndexRoutingTable.builder(shardId.getIndex()) .addIndexShard(new IndexShardRoutingTable.Builder(shardId) - .addShard(TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node1.id(), true, ShardRoutingState.STARTED, 10)) - .addShard(TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node2.id(), null, null, false, ShardRoutingState.INITIALIZING, 10, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null))) + .addShard(TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node1.id(), 1, true, ShardRoutingState.STARTED, 10)) + .addShard(TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node2.id(), null, null, 1, false, ShardRoutingState.INITIALIZING, 10, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null))) .build()) ) .build(); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index baf4ee6077537..16eedfe118e2d 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -21,7 +21,9 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.*; -import org.apache.lucene.search.*; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.Constants; @@ -207,8 +209,10 @@ public void testPersistenceStateMetadataPersistence() throws Exception { assertEquals(shardStateMetaData, getShardStateMetadata(shard)); assertEquals(shardStateMetaData, new ShardStateMetaData(routing.version(), routing.primary(), shard.indexSettings.get(IndexMetaData.SETTING_INDEX_UUID))); - // test if we still write it even if the shard is not active - ShardRouting inactiveRouting = TestShardRouting.newShardRouting(shard.shardRouting.index(), shard.shardRouting.shardId().id(), shard.shardRouting.currentNodeId(), null, null, true, ShardRoutingState.INITIALIZING, shard.shardRouting.version() + 1); + // test if we don't write it if the shard is not active + ShardRouting inactiveRouting = TestShardRouting.newShardRouting(shard.shardRouting.index(), shard.shardRouting.shardId().id(), + shard.shardRouting.currentNodeId(), null, null, shard.shardRouting.primaryTerm(), true, ShardRoutingState.INITIALIZING, + shard.shardRouting.version() + 1); shard.persistMetadata(inactiveRouting, shard.shardRouting); shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId)); assertEquals("inactive shard state shouldn't be persisted", shardStateMetaData, getShardStateMetadata(shard)); @@ -246,7 +250,9 @@ public void testDeleteShardState() throws IOException { ShardStateMetaData shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId)); assertEquals(shardStateMetaData, getShardStateMetadata(shard)); - routing = TestShardRouting.newShardRouting(shard.shardId.index().getName(), shard.shardId.id(), routing.currentNodeId(), null, routing.primary(), ShardRoutingState.INITIALIZING, shard.shardRouting.allocationId(), shard.shardRouting.version() + 1); + routing = TestShardRouting.newShardRouting(shard.shardId.index().getName(), shard.shardId.id(), routing.currentNodeId(), null, + shard.shardRouting.primaryTerm(), routing.primary(), ShardRoutingState.INITIALIZING, shard.shardRouting.allocationId(), + shard.shardRouting.version() + 1); shard.updateRoutingEntry(routing, true); shard.deleteShardState(); diff --git a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUnitTests.java b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUnitTests.java index 19cce93c6e4e6..8fad646d8b828 100644 --- a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUnitTests.java +++ b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUnitTests.java @@ -108,7 +108,7 @@ protected TestPlan createTestPlan() { Map shardResponses = new HashMap<>(); for (int copy = 0; copy < replicas + 1; copy++) { final ShardRouting shardRouting = TestShardRouting.newShardRouting(index, shard, "node_" + shardId + "_" + copy, null, - copy == 0, ShardRoutingState.STARTED, 0); + 1, copy == 0, ShardRoutingState.STARTED, 0); if (randomInt(5) < 2) { // shard copy failure failed++; diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java index 8562b0991c272..7c0820a9899c3 100644 --- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java +++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java @@ -413,7 +413,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { for (int i = 0; i < numShards; i++) { indexRoutingTableBuilder.addIndexShard( new IndexShardRoutingTable.Builder(new ShardId("test", i)) - .addShard(TestShardRouting.newShardRouting("test", i, masterId, true, ShardRoutingState.STARTED, shardVersions[shardIds[i]])) + .addShard(TestShardRouting.newShardRouting("test", i, masterId, 1, true, ShardRoutingState.STARTED, shardVersions[shardIds[i]])) .build() ); } diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java index 890b94d158392..91797a0b6994b 100644 --- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java +++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java @@ -26,7 +26,10 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.transport.LocalTransportAddress; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; @@ -39,7 +42,6 @@ import static org.elasticsearch.Version.CURRENT; import static org.elasticsearch.test.VersionUtils.randomVersion; -import static org.hamcrest.Matchers.is; /** */ @@ -83,10 +85,10 @@ public void testShardCanBeDeleted_noShardStarted() throws Exception { ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test")); clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas))); IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1)); - + final int primaryTerm = randomInt(200); for (int i = 0; i < numShards; i++) { int unStartedShard = randomInt(numReplicas); - for (int j=0; j <= numReplicas; j++) { + for (int j = 0; j <= numReplicas; j++) { ShardRoutingState state; if (j == unStartedShard) { state = randomFrom(NOT_STARTED_STATES); @@ -97,7 +99,7 @@ public void testShardCanBeDeleted_noShardStarted() throws Exception { if (state == ShardRoutingState.UNASSIGNED) { unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null); } - routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, null, j == 0, state, 0, unassignedInfo)); + routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, null, primaryTerm, j == 0, state, 0, unassignedInfo)); } } assertFalse(indicesStore.shardCanBeDeleted(clusterState.build(), routingTable.build())); @@ -113,12 +115,13 @@ public void testShardCanBeDeleted_shardExistsLocally() throws Exception { clusterState.nodes(DiscoveryNodes.builder().localNodeId(localNode.id()).put(localNode).put(new DiscoveryNode("xyz", new LocalTransportAddress("xyz"), Version.CURRENT))); IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1)); int localShardId = randomInt(numShards - 1); + final int primaryTerm = randomInt(200); for (int i = 0; i < numShards; i++) { String nodeId = i == localShardId ? localNode.getId() : randomBoolean() ? "abc" : "xyz"; String relocationNodeId = randomBoolean() ? null : randomBoolean() ? localNode.getId() : "xyz"; - routingTable.addShard(TestShardRouting.newShardRouting("test", i, nodeId, relocationNodeId, true, ShardRoutingState.STARTED, 0)); + routingTable.addShard(TestShardRouting.newShardRouting("test", i, nodeId, relocationNodeId, primaryTerm, true, ShardRoutingState.STARTED, 0)); for (int j = 0; j < numReplicas; j++) { - routingTable.addShard(TestShardRouting.newShardRouting("test", i, nodeId, relocationNodeId, false, ShardRoutingState.STARTED, 0)); + routingTable.addShard(TestShardRouting.newShardRouting("test", i, nodeId, relocationNodeId, primaryTerm, false, ShardRoutingState.STARTED, 0)); } } @@ -135,11 +138,12 @@ public void testShardCanBeDeleted_nodeNotInList() throws Exception { clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas))); clusterState.nodes(DiscoveryNodes.builder().localNodeId(localNode.id()).put(localNode)); IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1)); + final int primaryTerm = randomInt(200); for (int i = 0; i < numShards; i++) { String relocatingNodeId = randomBoolean() ? null : "def"; - routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", relocatingNodeId, true, ShardRoutingState.STARTED, 0)); + routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", relocatingNodeId, primaryTerm, true, ShardRoutingState.STARTED, 0)); for (int j = 0; j < numReplicas; j++) { - routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", relocatingNodeId, false, ShardRoutingState.STARTED, 0)); + routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", relocatingNodeId, primaryTerm, false, ShardRoutingState.STARTED, 0)); } } @@ -158,10 +162,11 @@ public void testShardCanBeDeleted_nodeVersion() throws Exception { clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas))); clusterState.nodes(DiscoveryNodes.builder().localNodeId(localNode.id()).put(localNode).put(new DiscoveryNode("xyz", new LocalTransportAddress("xyz"), nodeVersion))); IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1)); + final int primaryTerm = randomInt(200); for (int i = 0; i < numShards; i++) { - routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, true, ShardRoutingState.STARTED, 0)); + routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, primaryTerm, true, ShardRoutingState.STARTED, 0)); for (int j = 0; j < numReplicas; j++) { - routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, false, ShardRoutingState.STARTED, 0)); + routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", null, primaryTerm, false, ShardRoutingState.STARTED, 0)); } } @@ -177,7 +182,7 @@ public void testShardCanBeDeleted_relocatingNode() throws Exception { ClusterState.Builder clusterState = ClusterState.builder(new ClusterName("test")); clusterState.metaData(MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas))); final Version nodeVersion = randomBoolean() ? CURRENT : randomVersion(random()); - + final int primaryTerm = randomInt(200); clusterState.nodes(DiscoveryNodes.builder().localNodeId(localNode.id()) .put(localNode) .put(new DiscoveryNode("xyz", new LocalTransportAddress("xyz"), Version.CURRENT)) @@ -185,9 +190,9 @@ public void testShardCanBeDeleted_relocatingNode() throws Exception { )); IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", 1)); for (int i = 0; i < numShards; i++) { - routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", "def", true, ShardRoutingState.STARTED, 0)); + routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", "def", primaryTerm, true, ShardRoutingState.STARTED, 0)); for (int j = 0; j < numReplicas; j++) { - routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", "def", false, ShardRoutingState.STARTED, 0)); + routingTable.addShard(TestShardRouting.newShardRouting("test", i, "xyz", "def", primaryTerm, false, ShardRoutingState.STARTED, 0)); } } From 905404c4c43b3bf2d2c6f1068341ca8dd6d59383 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sat, 10 Oct 2015 12:01:46 +0300 Subject: [PATCH 02/19] better handling of primary term in builder --- .../cluster/metadata/IndexMetaData.java | 41 +++++++++++-------- .../TransportBroadcastByNodeActionTests.java | 12 +++++- .../gateway/MetaStateServiceTests.java | 2 +- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index db80c3f2c5b21..433e9538349ab 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -424,7 +424,7 @@ public boolean equals(Object o) { return false; } - if (Arrays.equals(primaryTerms, that.primaryTerms)) { + if (Arrays.equals(primaryTerms, that.primaryTerms) == false) { return false; } return true; @@ -488,7 +488,6 @@ public IndexMetaDataDiff(StreamInput in) throws IOException { version = in.readLong(); state = State.fromId(in.readByte()); settings = Settings.readSettingsFromStream(in); - // must be done after settings so shard number will be set primaryTerms = in.readIntArray(); mappings = DiffableUtils.readImmutableOpenMapDiff(in, MappingMetaData.PROTO); aliases = DiffableUtils.readImmutableOpenMapDiff(in, AliasMetaData.PROTO); @@ -511,7 +510,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(version); out.writeByte(state.id); Settings.writeSettingsToStream(settings, out); - // must be done after settings so shard number will be set out.writeIntArray(primaryTerms); mappings.writeTo(out); aliases.writeTo(out); @@ -594,7 +592,7 @@ public static class Builder { private String index; private State state = State.OPEN; private long version = 1; - private int[] primaryTerms; + private int[] primaryTerms = null; private Settings settings = Settings.Builder.EMPTY_SETTINGS; private final ImmutableOpenMap.Builder mappings; private final ImmutableOpenMap.Builder aliases; @@ -605,7 +603,6 @@ public Builder(String index) { this.mappings = ImmutableOpenMap.builder(); this.aliases = ImmutableOpenMap.builder(); this.customs = ImmutableOpenMap.builder(); - this.primaryTerms = new int[0]; } public Builder(IndexMetaData indexMetaData) { @@ -662,12 +659,6 @@ public Builder settings(Settings.Builder settings) { public Builder settings(Settings settings) { this.settings = settings; - if (numberOfShards() >= 0) { - primaryTerms = new int[numberOfShards()]; - } else { - // meh - what can you do... - primaryTerms = new int[0]; - } return this; } @@ -741,12 +732,16 @@ public Builder version(long version) { } public int primaryTerm(int shardId) { - assert numberOfShards() >= 0 : "primaryTerm(shardId) is called without settings numberOfShards() first"; + if (primaryTerms == null) { + allocatePrimaryTerms(); + } return this.primaryTerms[shardId]; } public Builder primaryTerm(int shardId, int primaryTerm) { - assert numberOfShards() >= 0 : "primaryTerm(shardId, primaryTerm) is called without settings numberOfShards() first"; + if (primaryTerms == null) { + allocatePrimaryTerms(); + } this.primaryTerms[shardId] = primaryTerm; return this; } @@ -755,6 +750,15 @@ private void primaryTerms(int[] primaryTerms) { this.primaryTerms = primaryTerms; } + private void allocatePrimaryTerms() { + assert primaryTerms == null; + if (numberOfShards() < 0) { + throw new IllegalStateException("you must set the number of shards before setting/reading primary terms"); + } + primaryTerms = new int[numberOfShards()]; + } + + public IndexMetaData build() { ImmutableOpenMap.Builder tmpAliases = aliases; Settings tmpSettings = settings; @@ -767,6 +771,13 @@ public IndexMetaData build() { } } + if (primaryTerms == null) { + allocatePrimaryTerms(); + } else if (primaryTerms.length != numberOfShards()) { + throw new IllegalStateException("primaryTerms length is [" + primaryTerms.length + + "] but should be equal to number of shards [" + numberOfShards() + "]"); + } + return new IndexMetaData(index, version, primaryTerms, state, tmpSettings, mappings.build(), tmpAliases.build(), customs.build()); } @@ -883,12 +894,10 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti throw new IllegalStateException("found a non-numeric value under [primary_terms]"); } } - if (builder.numberOfShards() < 0) { - builder.numberOfShards(list.size()); // alocation terms array - } for (int i = 0; i < list.size(); i++) { builder.primaryTerm(i, list.get(i)); } + builder.primaryTerms(list.toArray()); } } else if (token.isValue()) { if ("state".equals(currentFieldName)) { diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java index ac4e399cd626d..4cdf1bd8f8e1e 100644 --- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java @@ -35,7 +35,9 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.*; @@ -192,10 +194,12 @@ void setClusterState(TestClusterService clusterService, String index) { IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index); int shardIndex = -1; + int totalIndexShards = 0; for (int i = 0; i < numberOfNodes; i++) { final DiscoveryNode node = newNode(i); discoBuilder = discoBuilder.put(node); int numberOfShards = randomIntBetween(1, 10); + totalIndexShards += numberOfShards; for (int j = 0; j < numberOfShards; j++) { final ShardId shardId = new ShardId(index, ++shardIndex); final int primaryTerm = randomInt(200); @@ -209,6 +213,12 @@ void setClusterState(TestClusterService clusterService, String index) { discoBuilder.masterNodeId(newNode(numberOfNodes - 1).id()); ClusterState.Builder stateBuilder = ClusterState.builder(new ClusterName(TEST_CLUSTER)); stateBuilder.nodes(discoBuilder); + final IndexMetaData.Builder indexMetaData = IndexMetaData.builder(index) + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfReplicas(0) + .numberOfShards(totalIndexShards); + + stateBuilder.metaData(MetaData.builder().put(indexMetaData)); stateBuilder.routingTable(RoutingTable.builder().add(indexRoutingTable.build()).build()); ClusterState clusterState = stateBuilder.build(); clusterService.setState(clusterState); @@ -300,7 +310,7 @@ public void testOperationExecution() throws Exception { TransportResponse response = channel.getCapturedResponse(); assertTrue(response instanceof TransportBroadcastByNodeAction.NodeResponse); - TransportBroadcastByNodeAction.NodeResponse nodeResponse = (TransportBroadcastByNodeAction.NodeResponse)response; + TransportBroadcastByNodeAction.NodeResponse nodeResponse = (TransportBroadcastByNodeAction.NodeResponse) response; // check the operation was executed on the correct node assertEquals("node id", nodeId, nodeResponse.getNodeId()); diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java index de66f9519c7b7..0c6a9f662b271 100644 --- a/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java @@ -90,7 +90,7 @@ public void testWriteGlobalStateWithIndexAndNoIndexIsLoaded() throws Exception { } @Test - public void tesLoadGlobal() throws Exception { + public void testLoadGlobal() throws Exception { try (NodeEnvironment env = newNodeEnvironment()) { MetaStateService metaStateService = new MetaStateService(randomSettings(), env); From 3dff0ea68ed2031b83d83cef2af69b9d20528626 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sat, 10 Oct 2015 12:30:59 +0300 Subject: [PATCH 03/19] fix StartedShardsRoutingTests --- .../allocation/StartedShardsRoutingTests.java | 68 +++++++++++++------ 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java index 66b43d43db2b1..209f18f840d8b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java @@ -24,43 +24,66 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.AllocationId; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.test.ESAllocationTestCase; import org.junit.Test; import java.util.Arrays; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; + public class StartedShardsRoutingTests extends ESAllocationTestCase { @Test - public void tesStartedShardsMatching() { + public void testStartedShardsMatching() { AllocationService allocation = createAllocationService(); logger.info("--> building initial cluster state"); final IndexMetaData indexMetaData = IndexMetaData.builder("test") - .settings(settings(Version.CURRENT)) - .numberOfShards(3).numberOfReplicas(0) + .settings(settings(Version.CURRENT).put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, "none")) + .numberOfShards(3).numberOfReplicas(1) .build(); ClusterState.Builder stateBuilder = ClusterState.builder(ClusterName.DEFAULT) - .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))) + .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3"))) .metaData(MetaData.builder().put(indexMetaData, false)); - final ShardRouting initShard = TestShardRouting.newShardRouting("test", 0, "node1", 1, randomBoolean(), ShardRoutingState.INITIALIZING, 1); - final ShardRouting startedShard = TestShardRouting.newShardRouting("test", 1, "node2", 1, randomBoolean(), ShardRoutingState.STARTED, 1); - final ShardRouting relocatingShard = TestShardRouting.newShardRouting("test", 2, "node1", "node2", 1, randomBoolean(), ShardRoutingState.RELOCATING, 1); - stateBuilder.routingTable(RoutingTable.builder().add(IndexRoutingTable.builder("test") - .addIndexShard(new IndexShardRoutingTable.Builder(initShard.shardId()).addShard(initShard).build()) - .addIndexShard(new IndexShardRoutingTable.Builder(startedShard.shardId()).addShard(startedShard).build()) - .addIndexShard(new IndexShardRoutingTable.Builder(relocatingShard.shardId()).addShard(relocatingShard).build())).build()); + final ShardRouting initShard; + final ShardRouting startedShard; + final ShardRouting relocatingShard; + final IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder("test"); + if (randomBoolean()) { + initShard = TestShardRouting.newShardRouting("test", 0, "node1", 1, true, ShardRoutingState.INITIALIZING, 1); + ShardRouting replica = TestShardRouting.newShardRouting("test", 0, null, 1, false, ShardRoutingState.UNASSIGNED, 1); + indexRoutingTable.addIndexShard(new IndexShardRoutingTable.Builder(initShard.shardId()).addShard(initShard).addShard(replica).build()); + } else { + ShardRouting primaryShard = TestShardRouting.newShardRouting("test", 0, "node2", 1, true, ShardRoutingState.STARTED, 1); + initShard = TestShardRouting.newShardRouting("test", 0, "node1", 1, false, ShardRoutingState.INITIALIZING, 1); + indexRoutingTable.addIndexShard(new IndexShardRoutingTable.Builder(initShard.shardId()).addShard(primaryShard).addShard(initShard).build()); + } + if (randomBoolean()) { + startedShard = TestShardRouting.newShardRouting("test", 1, "node2", 1, true, ShardRoutingState.STARTED, 1); + ShardRouting replica = TestShardRouting.newShardRouting("test", 1, null, 1, false, ShardRoutingState.UNASSIGNED, 1); + indexRoutingTable.addIndexShard(new IndexShardRoutingTable.Builder(startedShard.shardId()).addShard(startedShard).addShard(replica).build()); + } else { + ShardRouting primaryShard = TestShardRouting.newShardRouting("test", 1, "node1", 1, true, ShardRoutingState.STARTED, 1); + startedShard = TestShardRouting.newShardRouting("test", 1, "node2", 1, false, ShardRoutingState.STARTED, 1); + indexRoutingTable.addIndexShard(new IndexShardRoutingTable.Builder(startedShard.shardId()).addShard(primaryShard).addShard(startedShard).build()); + } + + if (randomBoolean()) { + relocatingShard = TestShardRouting.newShardRouting("test", 2, "node1", "node2", 1, true, ShardRoutingState.RELOCATING, 1); + ShardRouting replica = TestShardRouting.newShardRouting("test", 2, null, 1, false, ShardRoutingState.UNASSIGNED, 1); + indexRoutingTable.addIndexShard(new IndexShardRoutingTable.Builder(relocatingShard.shardId()).addShard(relocatingShard).addShard(replica).build()); + } else { + ShardRouting primaryShard = TestShardRouting.newShardRouting("test", 2, "node3", 1, true, ShardRoutingState.STARTED, 1); + relocatingShard = TestShardRouting.newShardRouting("test", 2, "node1", "node2", 1, false, ShardRoutingState.RELOCATING, 1); + indexRoutingTable.addIndexShard(new IndexShardRoutingTable.Builder(relocatingShard.shardId()) + .addShard(primaryShard).addShard(relocatingShard).build()); + } + + stateBuilder.routingTable(RoutingTable.builder().add(indexRoutingTable).build()); ClusterState state = stateBuilder.build(); @@ -71,8 +94,9 @@ public void tesStartedShardsMatching() { initShard.primaryTerm(), initShard.primary(), ShardRoutingState.INITIALIZING, initShard.allocationId(), randomInt())), false); assertTrue("failed to start " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed()); - assertTrue(initShard + "isn't started \ncurrent routing table:" + result.routingTable().prettyPrint(), - result.routingTable().index("test").shard(initShard.id()).allShardsStarted()); + final ShardRouting resultRouting = result.routingTable().index("test").shard(initShard.id()).activeShards() + .stream().filter(routing -> routing.isSameAllocation(initShard)).findFirst().get(); + assertThat(initShard + "isn't started \ncurrent routing table:" + result.routingTable().prettyPrint(), resultRouting, notNullValue()); logger.info("--> testing shard variants that shouldn't match the initializing shard"); From 47ff7277f01593c6549bec5125c26528e2ba2b28 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sat, 10 Oct 2015 12:37:33 +0300 Subject: [PATCH 04/19] fix ClusterStateCreationUtils --- .../action/support/replication/ClusterStateCreationUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java index 9cec081a2facc..fb7c59f353c66 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java @@ -149,7 +149,7 @@ public static ClusterState stateWithAssignedPrimariesAndOneReplica(String index, discoBuilder.masterNodeId(newNode(1).id()); // we need a non-local master to test shard failures IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(Settings.builder() .put(SETTING_VERSION_CREATED, Version.CURRENT) - .put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 1) + .put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1) .put(SETTING_CREATION_DATE, System.currentTimeMillis())).build(); ClusterState.Builder state = ClusterState.builder(new ClusterName("test")); state.nodes(discoBuilder); From 849876b1c759e0864705609afcfb0165d9988383 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sat, 10 Oct 2015 20:57:15 +0300 Subject: [PATCH 05/19] move primary term syncing with meta data to AllocationService. --- .../elasticsearch/cluster/ClusterState.java | 29 +------ .../cluster/routing/RoutingNodes.java | 4 +- .../cluster/routing/ShardRouting.java | 5 +- .../routing/allocation/AllocationService.java | 82 ++++++++++++++++--- .../routing/allocation/RoutingAllocation.java | 24 ++++-- .../cluster/routing/RoutingTableTests.java | 8 ++ .../cluster/routing/ShardRoutingTests.java | 2 +- .../{decider => }/ShardStateIT.java | 14 ++-- .../zen/NodeJoinControllerTests.java | 6 +- 9 files changed, 114 insertions(+), 60 deletions(-) rename core/src/test/java/org/elasticsearch/cluster/routing/allocation/{decider => }/ShardStateIT.java (86%) diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java index d45a4d0950c79..bfff23868c2f3 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -583,6 +583,7 @@ public Builder nodes(DiscoveryNodes nodes) { public Builder routingResult(RoutingAllocation.Result routingResult) { this.routingTable = routingResult.routingTable(); + this.metaData = routingResult.metaData(); return this; } @@ -653,34 +654,6 @@ public ClusterState build() { if (UNKNOWN_UUID.equals(uuid)) { uuid = Strings.randomBase64UUID(); } - // make sure index meta data and routing tables are in sync w.r.t primaryTerm - MetaData.Builder metaDataBuilder = null; - for (IndexRoutingTable indexRoutingTable : routingTable) { - final IndexMetaData indexMetaData = metaData.index(indexRoutingTable.getIndex()); - IndexMetaData.Builder indexMetaDataBuilder = null; - for (IndexShardRoutingTable shardRoutings : indexRoutingTable) { - final ShardRouting primary = shardRoutings.primaryShard(); - final int shardId = primary.shardId().id(); - if (primary.primaryTerm() != indexMetaData.primaryTerm(shardId)) { - assert primary.primaryTerm() > indexMetaData.primaryTerm(shardId) : - "primary term should only increase. Index primary term [" - + indexMetaData.primaryTerm(shardId) + "] but primary routing is " + primary; - if (indexMetaDataBuilder == null) { - indexMetaDataBuilder = IndexMetaData.builder(indexMetaData); - } - indexMetaDataBuilder.primaryTerm(shardId, primary.primaryTerm()); - } - } - if (indexMetaDataBuilder != null) { - if (metaDataBuilder == null) { - metaDataBuilder = MetaData.builder(metaData); - } - metaDataBuilder.put(indexMetaDataBuilder); - } - } - if (metaDataBuilder != null) { - metaData = metaDataBuilder.build(); - } return new ClusterState(clusterName, version, uuid, metaData, routingTable, nodes, blocks, customs.build(), fromDiff); } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index d5ed922b1206c..ee33a118520c0 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -609,10 +609,12 @@ public ShardRouting next() { /** * Initializes the current unassigned shard and moves it from the unassigned list. + * + * If a primary is initalized, it's term is incremented. */ public void initialize(String nodeId, long version, long expectedShardSize) { innerRemove(); - nodes.initialize(new ShardRouting(current, version), nodeId, expectedShardSize); + nodes.initialize(new ShardRouting(current, version, current.primary() ? current.primaryTerm() + 1 : current.primaryTerm()), nodeId, expectedShardSize); } /** diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index 59d6d2ba07a56..6bdae16e12b9e 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -435,7 +435,7 @@ void moveToUnassigned(UnassignedInfo unassignedInfo) { } /** - * Initializes an unassigned shard on a node. + * Initializes an unassigned shard on a node. If the shard is primary, it's term is incremented. */ void initialize(String nodeId, long expectedShardSize) { ensureNotFrozen(); @@ -445,6 +445,9 @@ void initialize(String nodeId, long expectedShardSize) { state = ShardRoutingState.INITIALIZING; currentNodeId = nodeId; allocationId = AllocationId.newInitializing(); + if (primary) { + primaryTerm++; + } this.expectedShardSize = expectedShardSize; } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index de3a763e205e5..d34b016e339e9 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -23,13 +23,9 @@ import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators; import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; @@ -79,13 +75,70 @@ public RoutingAllocation.Result applyStartedShards(ClusterState clusterState, Li StartedRerouteAllocation allocation = new StartedRerouteAllocation(allocationDeciders, routingNodes, clusterState.nodes(), startedShards, clusterInfoService.getClusterInfo()); boolean changed = applyStartedShards(routingNodes, startedShards); if (!changed) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } shardsAllocators.applyStartedShards(allocation); if (withReroute) { reroute(allocation); } - return new RoutingAllocation.Result(true, new RoutingTable.Builder().updateNodes(routingNodes).build().validateRaiseException(clusterState.metaData())); + return buildChangedResult(clusterState.metaData(), routingNodes); + } + + + protected RoutingAllocation.Result buildChangedResult(MetaData metaData, RoutingNodes routingNodes) { + return buildChangedResult(metaData, routingNodes, new RoutingExplanations()); + + } + protected RoutingAllocation.Result buildChangedResult(MetaData metaData, RoutingNodes routingNodes, RoutingExplanations explanations) { + final RoutingTable routingTable = new RoutingTable.Builder().updateNodes(routingNodes).build(); + MetaData newMetaData = updateMetaDataWithRoutingTable(metaData,routingTable); + return new RoutingAllocation.Result(true, routingTable.validateRaiseException(newMetaData), newMetaData, explanations); + } + + /** + * Updates the current {@link MetaData} based on the newly created {@link RoutingTable}. + * + * @param currentMetaData {@link MetaData} object from before the routing table was changed. + * @param newRoutingTable new {@link RoutingTable} created by the allocation change + * @return adpated {@link MetaData}, potentially the original one if no change was needed. + */ + static MetaData updateMetaDataWithRoutingTable(MetaData currentMetaData, RoutingTable newRoutingTable) { + // make sure index meta data and routing tables are in sync w.r.t primaryTerm + MetaData.Builder metaDataBuilder = null; + for (IndexRoutingTable indexRoutingTable : newRoutingTable) { + final IndexMetaData indexMetaData = currentMetaData.index(indexRoutingTable.getIndex()); + if (indexMetaData == null) { + throw new IllegalStateException("no metadata found for index [" + indexRoutingTable.index() + "]"); + } + IndexMetaData.Builder indexMetaDataBuilder = null; + for (IndexShardRoutingTable shardRoutings : indexRoutingTable) { + final ShardRouting primary = shardRoutings.primaryShard(); + if (primary == null) { + throw new IllegalStateException("missing primary shard for " + shardRoutings.shardId()); + } + final int shardId = primary.shardId().id(); + if (primary.primaryTerm() != indexMetaData.primaryTerm(shardId)) { + assert primary.primaryTerm() > indexMetaData.primaryTerm(shardId) : + "primary term should only increase. Index primary term [" + + indexMetaData.primaryTerm(shardId) + "] but primary routing is " + primary; + if (indexMetaDataBuilder == null) { + indexMetaDataBuilder = IndexMetaData.builder(indexMetaData); + } + indexMetaDataBuilder.primaryTerm(shardId, primary.primaryTerm()); + } + } + if (indexMetaDataBuilder != null) { + if (metaDataBuilder == null) { + metaDataBuilder = MetaData.builder(currentMetaData); + } + metaDataBuilder.put(indexMetaDataBuilder); + } + } + if (metaDataBuilder != null) { + return metaDataBuilder.build(); + } else { + return currentMetaData; + } } public RoutingAllocation.Result applyFailedShard(ClusterState clusterState, ShardRouting failedShard) { @@ -107,11 +160,11 @@ public RoutingAllocation.Result applyFailedShards(ClusterState clusterState, Lis changed |= applyFailedShard(allocation, failedShard.shard, true, new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, failedShard.message, failedShard.failure)); } if (!changed) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } shardsAllocators.applyFailedShards(allocation); reroute(allocation); - return new RoutingAllocation.Result(true, new RoutingTable.Builder().updateNodes(routingNodes).build().validateRaiseException(clusterState.metaData())); + return buildChangedResult(clusterState.metaData(), routingNodes); } public RoutingAllocation.Result reroute(ClusterState clusterState, AllocationCommands commands) { @@ -134,9 +187,12 @@ public RoutingAllocation.Result reroute(ClusterState clusterState, AllocationCom // the assumption is that commands will move / act on shards (or fail through exceptions) // so, there will always be shard "movements", so no need to check on reroute reroute(allocation); - return new RoutingAllocation.Result(true, new RoutingTable.Builder().updateNodes(routingNodes).build().validateRaiseException(clusterState.metaData()), explanations); + + return buildChangedResult(clusterState.metaData(), routingNodes, explanations); } + + /** * Reroutes the routing table based on the live nodes. *

@@ -158,9 +214,9 @@ public RoutingAllocation.Result reroute(ClusterState clusterState, boolean debug RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState.nodes(), clusterInfoService.getClusterInfo()); allocation.debugDecision(debug); if (!reroute(allocation)) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } - return new RoutingAllocation.Result(true, new RoutingTable.Builder().updateNodes(routingNodes).build().validateRaiseException(clusterState.metaData())); + return buildChangedResult(clusterState.metaData(), routingNodes); } private boolean reroute(RoutingAllocation allocation) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index 1874a7b020b6d..678b855841b31 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -52,29 +52,33 @@ public static class Result { private final RoutingTable routingTable; + private final MetaData metaData; + private RoutingExplanations explanations = new RoutingExplanations(); /** * Creates a new {@link RoutingAllocation.Result} - * - * @param changed a flag to determine whether the actual {@link RoutingTable} has been changed + * @param changed a flag to determine whether the actual {@link RoutingTable} has been changed * @param routingTable the {@link RoutingTable} this Result references + * @param metaData the {@link MetaData} this result refrences */ - public Result(boolean changed, RoutingTable routingTable) { + public Result(boolean changed, RoutingTable routingTable, MetaData metaData) { this.changed = changed; this.routingTable = routingTable; + this.metaData = metaData; } /** * Creates a new {@link RoutingAllocation.Result} - * - * @param changed a flag to determine whether the actual {@link RoutingTable} has been changed + * @param changed a flag to determine whether the actual {@link RoutingTable} has been changed * @param routingTable the {@link RoutingTable} this Result references + * @param metaData the {@link MetaData} this Result references * @param explanations Explanation for the reroute actions */ - public Result(boolean changed, RoutingTable routingTable, RoutingExplanations explanations) { + public Result(boolean changed, RoutingTable routingTable, MetaData metaData, RoutingExplanations explanations) { this.changed = changed; this.routingTable = routingTable; + this.metaData = metaData; this.explanations = explanations; } @@ -85,6 +89,14 @@ public boolean changed() { return this.changed; } + /** + * Get the {@link MetaData} referenced by this result + * @return referenced {@link MetaData} + */ + public MetaData metaData() { + return metaData; + } + /** * Get the {@link RoutingTable} referenced by this result * @return referenced {@link RoutingTable} diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java index c17deea6c56e7..3e93c5a92f227 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java @@ -101,6 +101,7 @@ private void initPrimaries() { assertThat(rerouteResult.changed(), is(true)); this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); versionsPerIndex.keySet().forEach(this::incrementVersion); + primaryTermsPerIndex.keySet().forEach(this::incrementPrimaryTerm); } private void incrementVersion(String index) { @@ -114,6 +115,13 @@ private void incrementVersion(String index, int shard) { versionsPerIndex.get(index)[shard]++; } + private void incrementPrimaryTerm(String index) { + final int[] primaryTerms = primaryTermsPerIndex.get(index); + for (int i = 0; i < primaryTerms.length; i++) { + primaryTerms[i]++; + } + } + private void incrementPrimaryTerm(String index, int shard) { primaryTermsPerIndex.get(index)[shard]++; } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java index 947e8cb6d0d36..183ff7484f3d7 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java @@ -52,7 +52,7 @@ public void testFrozenAfterRead() throws IOException { } } - public void testPrimaryTermIncrement() { + public void testPrimaryTermIncrementOnPromotion() { int term = randomInt(200); ShardRouting routing = TestShardRouting.newShardRouting("foo", 1, "node_1", null, null, term, false, ShardRoutingState.STARTED, 1); routing.moveToPrimary(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/ShardStateIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java similarity index 86% rename from core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/ShardStateIT.java rename to core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java index f3ea59c973a27..93b92f37b244f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/ShardStateIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.cluster.routing.allocation.decider; +package org.elasticsearch.cluster.routing.allocation; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.ClusterState; @@ -34,8 +34,8 @@ public void testPrimaryFailureIncreasesTerm() throws Exception { ensureGreen(); ClusterState state = client().admin().cluster().prepareState().get().getState(); IndexMetaData metaData = state.metaData().index("test"); - assertThat(metaData.primaryTerm(0), equalTo(0)); - assertThat(metaData.primaryTerm(1), equalTo(0)); + assertThat(metaData.primaryTerm(0), equalTo(1)); + assertThat(metaData.primaryTerm(1), equalTo(1)); logger.info("--> disabling allocation to capture shard failure"); disableAllocation("test"); @@ -52,15 +52,15 @@ public void testPrimaryFailureIncreasesTerm() throws Exception { state = client().admin().cluster().prepareState().get().getState(); metaData = state.metaData().index("test"); - assertThat(metaData.primaryTerm(shard), equalTo(1)); - assertThat(metaData.primaryTerm(shard ^ 1), equalTo(0)); + assertThat(metaData.primaryTerm(shard), equalTo(2)); + assertThat(metaData.primaryTerm(shard ^ 1), equalTo(1)); logger.info("--> enabling allocation"); enableAllocation("test"); ensureGreen(); state = client().admin().cluster().prepareState().get().getState(); metaData = state.metaData().index("test"); - assertThat(metaData.primaryTerm(shard), equalTo(1)); - assertThat(metaData.primaryTerm(shard ^ 1), equalTo(0)); + assertThat(metaData.primaryTerm(shard), equalTo(2)); + assertThat(metaData.primaryTerm(shard ^ 1), equalTo(1)); } } diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java index 70c9430b53125..1a2bf630fecc7 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java @@ -487,17 +487,17 @@ public NoopAllocationService(Settings settings) { @Override public RoutingAllocation.Result applyStartedShards(ClusterState clusterState, List startedShards, boolean withReroute) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } @Override public RoutingAllocation.Result applyFailedShards(ClusterState clusterState, List failedShards) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } @Override public RoutingAllocation.Result reroute(ClusterState clusterState, boolean debug) { - return new RoutingAllocation.Result(false, clusterState.routingTable()); + return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData()); } } From f8c6e6b8621e92a43bc1f5a7039df5bff1da1447 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 11 Oct 2015 08:06:38 +0300 Subject: [PATCH 06/19] add primary terms to ClusterState's XContent rendering --- .../main/java/org/elasticsearch/cluster/ClusterState.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java index bfff23868c2f3..055e63fda17c0 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -468,6 +468,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endArray(); + builder.startObject("primary_terms"); + for (int shard = 0; shard < indexMetaData.numberOfShards(); shard++) { + builder.field(Integer.toString(shard), indexMetaData.primaryTerm(shard)); + } + builder.endObject(); + builder.endObject(); } builder.endObject(); From 2ea979dcba3d80c828c19bd75a72c4c6bb215d82 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 11 Oct 2015 08:32:11 +0300 Subject: [PATCH 07/19] add some assertions to RecoveryFromGatewayIT --- .../gateway/RecoveryFromGatewayIT.java | 76 +++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 2a4e8b8c00820..2496090974319 100644 --- a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -19,11 +19,16 @@ package org.elasticsearch.gateway; +import com.carrotsearch.hppc.cursors.ObjectCursor; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; import org.elasticsearch.common.settings.Settings; @@ -39,6 +44,10 @@ import org.elasticsearch.test.store.MockFSDirectoryService; import org.junit.Test; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.IntStream; + import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -48,10 +57,7 @@ import static org.elasticsearch.test.ESIntegTestCase.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.*; @ClusterScope(numDataNodes = 0, scope = Scope.TEST) public class RecoveryFromGatewayIT extends ESIntegTestCase { @@ -81,10 +87,13 @@ public void testOneNodeRecoverFromGateway() throws Exception { assertHitCount(client().prepareCount().setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); ensureYellow("test"); // wait for primary allocations here otherwise if we have a lot of shards we might have a // shard that is still in post recovery when we restart and the ensureYellow() below will timeout + + Map primaryTerms = assertAndCapturePrimaryTerms(null); internalCluster().fullRestart(); logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); client().admin().indices().prepareRefresh().execute().actionGet(); assertHitCount(client().prepareCount().setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); @@ -93,11 +102,46 @@ public void testOneNodeRecoverFromGateway() throws Exception { logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); client().admin().indices().prepareRefresh().execute().actionGet(); assertHitCount(client().prepareCount().setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); } + private Map assertAndCapturePrimaryTerms(Map previousTerms) { + if (previousTerms == null) { + previousTerms = new HashMap<>(); + } + final Map result = new HashMap<>(); + final ClusterState state = client().admin().cluster().prepareState().get().getState(); + for (ObjectCursor cursor : state.metaData().indices().values()) { + final IndexMetaData indexMetaData = cursor.value; + final String index = indexMetaData.index(); + final int[] previous = previousTerms.get(index); + final int[] current = IntStream.range(0, indexMetaData.numberOfShards()).map(indexMetaData::primaryTerm).toArray(); + if (previous == null) { + result.put(index, current); + } else { + assertThat("number of terms changed for index [" + index + "]", current.length, equalTo(previous.length)); + for (int shard = 0; shard < current.length; shard++) { + assertThat("primary term didn't increase for [" + index + "][" + shard + "]", current[shard], greaterThan(previous[shard])); + } + result.put(index, current); + } + } + + for (IndexRoutingTable indexRoutingTable : state.routingTable()) { + final int[] terms = result.get(indexRoutingTable.index()); + for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) { + + for (ShardRouting routing : shardRoutingTable.shards()) { + assertThat("wrong primary term for " + routing, routing.primaryTerm(), equalTo(terms[routing.shardId().id()])); + } + } + } + return result; + } + @Test public void testSingleNodeNoFlush() throws Exception { @@ -158,10 +202,14 @@ SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 1) logger.info("Ensure all primaries have been started"); ensureYellow(); } + + Map primaryTerms = assertAndCapturePrimaryTerms(null); + internalCluster().fullRestart(); logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); for (int i = 0; i <= randomInt(10); i++) { assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), value1Docs + value2Docs); @@ -175,6 +223,7 @@ SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 1) logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); for (int i = 0; i <= randomInt(10); i++) { assertHitCount(client().prepareCount().setQuery(matchAllQuery()).get(), value1Docs + value2Docs); @@ -183,7 +232,7 @@ SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 1) assertHitCount(client().prepareCount().setQuery(termQuery("num", 179)).get(), value1Docs); } } - + @Test public void testSingleNodeWithFlush() throws Exception { @@ -198,10 +247,13 @@ public void testSingleNodeWithFlush() throws Exception { ensureYellow("test"); // wait for primary allocations here otherwise if we have a lot of shards we might have a // shard that is still in post recovery when we restart and the ensureYellow() below will timeout + Map primaryTerms = assertAndCapturePrimaryTerms(null); + internalCluster().fullRestart(); logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); for (int i = 0; i < 10; i++) { assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 2); @@ -211,6 +263,7 @@ public void testSingleNodeWithFlush() throws Exception { logger.info("Running Cluster Health (wait for the shards to startup)"); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); for (int i = 0; i < 10; i++) { assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 2); @@ -235,6 +288,8 @@ public void testTwoNodeFirstNodeCleared() throws Exception { assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 2); } + Map primaryTerms = assertAndCapturePrimaryTerms(null); + internalCluster().fullRestart(new RestartCallback() { @Override public Settings onNodeStopped(String nodeName) throws Exception { @@ -250,6 +305,7 @@ public boolean clearData(String nodeName) { logger.info("Running Cluster Health (wait for the shards to startup)"); ensureGreen(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); for (int i = 0; i < 10; i++) { assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 2); @@ -276,6 +332,8 @@ public void testLatestVersionLoaded() throws Exception { String metaDataUuid = client().admin().cluster().prepareState().execute().get().getState().getMetaData().clusterUUID(); assertThat(metaDataUuid, not(equalTo("_na_"))); + Map primaryTerms = assertAndCapturePrimaryTerms(null); + logger.info("--> closing first node, and indexing more data to the second node"); internalCluster().fullRestart(new RestartCallback() { @@ -315,6 +373,7 @@ public void doAfterNodes(int numNodes, Client client) throws Exception { logger.info("--> running cluster_health (wait for the shards to startup)"); ensureGreen(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); assertThat(client().admin().cluster().prepareState().execute().get().getState().getMetaData().clusterUUID(), equalTo(metaDataUuid)); @@ -389,11 +448,15 @@ public void testReusePeerRecovery() throws Exception { .setTransientSettings(settingsBuilder() .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, EnableAllocationDecider.Allocation.NONE)) .get(); + + Map primaryTerms = assertAndCapturePrimaryTerms(null); + logger.info("--> full cluster restart"); internalCluster().fullRestart(); logger.info("--> waiting for cluster to return to green after {}shutdown", useSyncIds ? "" : "second "); ensureGreen(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); if (useSyncIds) { assertSyncIdsNotNull(); @@ -449,6 +512,8 @@ public void testRecoveryDifferentNodeOrderStartup() throws Exception { internalCluster().startNode(settingsBuilder().put("path.data", createTempDir()).build()); ensureGreen(); + Map primaryTerms = assertAndCapturePrimaryTerms(null); + internalCluster().fullRestart(new RestartCallback() { @@ -459,6 +524,7 @@ public boolean doRestart(String nodeName) { }); ensureYellow(); + primaryTerms = assertAndCapturePrimaryTerms(primaryTerms); assertThat(client().admin().indices().prepareExists("test").execute().actionGet().isExists(), equalTo(true)); assertHitCount(client().prepareCount("test").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(), 1); From f1434f400cb12743ab0b2788970a34fc894978c9 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 11 Oct 2015 09:54:37 +0300 Subject: [PATCH 08/19] failed to increment routing table and meta data table --- .../cluster/metadata/IndexMetaData.java | 1 - .../routing/allocation/AllocationService.java | 5 +-- .../cluster/routing/RoutingTableTests.java | 2 ++ .../PrimaryElectionRoutingTests.java | 34 +++++++++++-------- .../routing/allocation/ShardStateIT.java | 28 ++++++++------- .../gateway/RecoveryFromGatewayIT.java | 1 - 6 files changed, 40 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 433e9538349ab..53e45751a342a 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -536,7 +536,6 @@ public IndexMetaData readFrom(StreamInput in) throws IOException { builder.version(in.readLong()); builder.state(State.fromId(in.readByte())); builder.settings(readSettingsFromStream(in)); - // must be done after settings so the array will not be overwritten builder.primaryTerms(in.readIntArray()); int mappingsSize = in.readVInt(); for (int i = 0; i < mappingsSize; i++) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index d34b016e339e9..4800fce101eac 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -90,7 +90,8 @@ protected RoutingAllocation.Result buildChangedResult(MetaData metaData, Routing } protected RoutingAllocation.Result buildChangedResult(MetaData metaData, RoutingNodes routingNodes, RoutingExplanations explanations) { - final RoutingTable routingTable = new RoutingTable.Builder().updateNodes(routingNodes).build(); + final RoutingTable routingTable = new RoutingTable.Builder().updateNodes(routingNodes) + .version(routingNodes.getRoutingTable().version() + 1).build(); MetaData newMetaData = updateMetaDataWithRoutingTable(metaData,routingTable); return new RoutingAllocation.Result(true, routingTable.validateRaiseException(newMetaData), newMetaData, explanations); } @@ -135,7 +136,7 @@ static MetaData updateMetaDataWithRoutingTable(MetaData currentMetaData, Routing } } if (metaDataBuilder != null) { - return metaDataBuilder.build(); + return metaDataBuilder.version(currentMetaData.version() + 1).build(); } else { return currentMetaData; } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java index 3e93c5a92f227..c446b2d57e344 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java @@ -151,6 +151,8 @@ private void failSomePrimaries(String index) { incrementVersion(index, shard); // and another time when the primary flag is set to false } RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.applyFailedShards(this.clusterState, failedShards); + assertThat(rerouteResult.routingTable().version(), greaterThan(clusterState.routingTable().version())); + assertThat(rerouteResult.metaData().version(), greaterThan(clusterState.metaData().version())); this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); this.testRoutingTable = rerouteResult.routingTable(); } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java index 69824f1a24809..bddbd414dad8d 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java @@ -61,29 +61,31 @@ public void testBackupElectionToPrimaryWhenPrimaryCanBeAllocatedToAnotherNode() ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build(); logger.info("Adding two nodes and performing rerouting"); - clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build(); - RoutingTable prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); - clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build(); + RoutingAllocation.Result result = strategy.reroute(clusterState); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); + + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build(); + result = strategy.reroute(clusterState); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); logger.info("Start the primary shard (on node1)"); RoutingNodes routingNodes = clusterState.getRoutingNodes(); - prevRoutingTable = routingTable; - routingTable = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING)).routingTable(); - clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + result = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING)); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); logger.info("Start the backup shard (on node2)"); routingNodes = clusterState.getRoutingNodes(); - prevRoutingTable = routingTable; - routingTable = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING)).routingTable(); - clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + result = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING)); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); logger.info("Adding third node and reroute and kill first node"); clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3")).remove("node1")).build(); - prevRoutingTable = routingTable; - routingTable = strategy.reroute(clusterState).routingTable(); - clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + RoutingTable prevRoutingTable = clusterState.routingTable(); + result = strategy.reroute(clusterState); + clusterState = ClusterState.builder(clusterState).routingResult(result).build(); routingNodes = clusterState.getRoutingNodes(); + routingTable = clusterState.routingTable(); assertThat(prevRoutingTable != routingTable, equalTo(true)); assertThat(routingTable.index("test").shards().size(), equalTo(1)); @@ -92,6 +94,7 @@ public void testBackupElectionToPrimaryWhenPrimaryCanBeAllocatedToAnotherNode() assertThat(routingNodes.node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1)); // verify where the primary is assertThat(routingTable.index("test").shard(0).primaryShard().currentNodeId(), equalTo("node2")); + assertThat(routingTable.index("test").shard(0).primaryShard().primaryTerm(), equalTo(2)); assertThat(routingTable.index("test").shard(0).replicaShards().get(0).currentNodeId(), equalTo("node3")); } @@ -119,7 +122,7 @@ public void testRemovingInitializingReplicasIfPrimariesFails() { logger.info("Start the primary shards"); RoutingNodes routingNodes = clusterState.getRoutingNodes(); rerouteResult = allocation.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)); - clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); + clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); routingNodes = clusterState.getRoutingNodes(); assertThat(routingNodes.shardsWithState(STARTED).size(), equalTo(2)); @@ -133,12 +136,13 @@ public void testRemovingInitializingReplicasIfPrimariesFails() { .put(newNode(nodeIdRemaining)) ).build(); rerouteResult = allocation.reroute(clusterState); - clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); + clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); routingNodes = clusterState.getRoutingNodes(); assertThat(routingNodes.shardsWithState(STARTED).size(), equalTo(1)); assertThat(routingNodes.shardsWithState(INITIALIZING).size(), equalTo(1)); assertThat(routingNodes.node(nodeIdRemaining).shardsWithState(INITIALIZING).get(0).primary(), equalTo(true)); + assertThat(routingNodes.node(nodeIdRemaining).shardsWithState(INITIALIZING).get(0).primaryTerm(), equalTo(2)); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java index 93b92f37b244f..e761f4b351e58 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java @@ -32,14 +32,12 @@ public void testPrimaryFailureIncreasesTerm() throws Exception { internalCluster().ensureAtLeastNumDataNodes(2); prepareCreate("test").setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1).get(); ensureGreen(); - ClusterState state = client().admin().cluster().prepareState().get().getState(); - IndexMetaData metaData = state.metaData().index("test"); - assertThat(metaData.primaryTerm(0), equalTo(1)); - assertThat(metaData.primaryTerm(1), equalTo(1)); + assertPrimaryTerms(1, 1); logger.info("--> disabling allocation to capture shard failure"); disableAllocation("test"); + ClusterState state = client().admin().cluster().prepareState().get().getState(); final int shard = randomBoolean() ? 0 : 1; final String nodeId = state.routingTable().index("test").shard(shard).primaryShard().currentNodeId(); final String node = state.nodes().get(nodeId).name(); @@ -50,17 +48,23 @@ public void testPrimaryFailureIncreasesTerm() throws Exception { logger.info("--> waiting for a yellow index"); assertBusy(() -> assertThat(client().admin().cluster().prepareHealth().get().getStatus(), equalTo(ClusterHealthStatus.YELLOW))); - state = client().admin().cluster().prepareState().get().getState(); - metaData = state.metaData().index("test"); - assertThat(metaData.primaryTerm(shard), equalTo(2)); - assertThat(metaData.primaryTerm(shard ^ 1), equalTo(1)); + final int term0 = shard == 0 ? 2 : 1; + final int term1 = shard == 1 ? 2 : 1; + assertPrimaryTerms(term0, term1); logger.info("--> enabling allocation"); enableAllocation("test"); ensureGreen(); - state = client().admin().cluster().prepareState().get().getState(); - metaData = state.metaData().index("test"); - assertThat(metaData.primaryTerm(shard), equalTo(2)); - assertThat(metaData.primaryTerm(shard ^ 1), equalTo(1)); + assertPrimaryTerms(term0, term1); + } + + protected void assertPrimaryTerms(int term0, int term1) { + for (String node : internalCluster().getNodeNames()) { + logger.debug("--> asserting primary terms terms on [{}]", node); + ClusterState state = client(node).admin().cluster().prepareState().setLocal(true).get().getState(); + IndexMetaData metaData = state.metaData().index("test"); + assertThat(metaData.primaryTerm(0), equalTo(term0)); + assertThat(metaData.primaryTerm(1), equalTo(term1)); + } } } diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 2496090974319..b886de0bb5389 100644 --- a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -133,7 +133,6 @@ private Map assertAndCapturePrimaryTerms(Map previ for (IndexRoutingTable indexRoutingTable : state.routingTable()) { final int[] terms = result.get(indexRoutingTable.index()); for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) { - for (ShardRouting routing : shardRoutingTable.shards()) { assertThat("wrong primary term for " + routing, routing.primaryTerm(), equalTo(terms[routing.shardId().id()])); } From 3d3df93c7e3ced3b472fa14175ef75cb65170d6b Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 11 Oct 2015 13:08:10 +0300 Subject: [PATCH 09/19] truely copy the primaryTerms array --- .../java/org/elasticsearch/cluster/metadata/IndexMetaData.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 53e45751a342a..a0a6ec296dac0 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.node.DiscoveryNodeFilters; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -609,7 +610,7 @@ public Builder(IndexMetaData indexMetaData) { this.state = indexMetaData.state; this.version = indexMetaData.version; this.settings = indexMetaData.getSettings(); - this.primaryTerms = indexMetaData.primaryTerms; + this.primaryTerms = Arrays.copyOf(indexMetaData.primaryTerms, indexMetaData.primaryTerms.length); this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings); this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases); this.customs = ImmutableOpenMap.builder(indexMetaData.customs); From 82f65fa20b42718791653ce3eef5396eb0c4cc53 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 11 Oct 2015 23:55:57 +0300 Subject: [PATCH 10/19] fix RoutingTableTests to user routing results --- .../elasticsearch/cluster/routing/RoutingTableTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java index c446b2d57e344..a6f716153a82e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java @@ -99,7 +99,7 @@ private void initPrimaries() { RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.reroute(clusterState); this.testRoutingTable = rerouteResult.routingTable(); assertThat(rerouteResult.changed(), is(true)); - this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); + this.clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); versionsPerIndex.keySet().forEach(this::incrementVersion); primaryTermsPerIndex.keySet().forEach(this::incrementPrimaryTerm); } @@ -153,7 +153,7 @@ private void failSomePrimaries(String index) { RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.applyFailedShards(this.clusterState, failedShards); assertThat(rerouteResult.routingTable().version(), greaterThan(clusterState.routingTable().version())); assertThat(rerouteResult.metaData().version(), greaterThan(clusterState.metaData().version())); - this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); + this.clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); this.testRoutingTable = rerouteResult.routingTable(); } @@ -184,7 +184,7 @@ private void assertVersionAndPrimaryTerm(String index) { assertThat("wrong version in " + routing, routing.version(), equalTo(versions[shard])); assertThat("wrong primary term in " + routing, routing.primaryTerm(), equalTo(terms[shard])); } - assertThat(indexMetaData.primaryTerm(shard), equalTo(terms[shard])); + assertThat("primary term mismatch between indexMetaData of [" + index + "] and shard [" + shard + "]'s routing", indexMetaData.primaryTerm(shard), equalTo(terms[shard])); } } From 58446fcb5b3feac4c1561e141a4bfab40d2c5d44 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 12 Oct 2015 10:37:28 +0300 Subject: [PATCH 11/19] minor tweak java docs --- .../org/elasticsearch/cluster/metadata/IndexMetaData.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index a0a6ec296dac0..d4cd65c953dab 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -284,8 +284,10 @@ public long getVersion() { /** - * this number is incremented when a replica shard is promoted to a primary (see {@link ShardRouting#moveToPrimary()}) or - * a first primary is created after a full cluster restart. + * The term of the current selected primary. This is a non-negative number incremented when + * a primary shard is assigned after a full cluster restart or a replica shard is promoted + * to a primary (see {@link ShardRouting#moveToPrimary()}) + * */ public int primaryTerm(int shardId) { return this.primaryTerms[shardId]; From c32541311fda1004cb11e063f179cfbc33e04f6d Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 13 Oct 2015 13:20:06 +0300 Subject: [PATCH 12/19] fix double term increment --- .../cluster/routing/RoutingNodes.java | 2 +- .../gateway/PrimaryShardAllocatorTests.java | 46 ++++++++++--------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index ee33a118520c0..8eff3a3438272 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -614,7 +614,7 @@ public ShardRouting next() { */ public void initialize(String nodeId, long version, long expectedShardSize) { innerRemove(); - nodes.initialize(new ShardRouting(current, version, current.primary() ? current.primaryTerm() + 1 : current.primaryTerm()), nodeId, expectedShardSize); + nodes.initialize(new ShardRouting(current, version), nodeId, expectedShardSize); } /** diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java index 7aad4492e51bd..60ff0c93147d9 100644 --- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESAllocationTestCase; +import org.hamcrest.Matcher; import org.junit.Before; import org.junit.Test; @@ -71,7 +72,7 @@ public void testNoProcessReplica() { } @Test - public void testNoProcessPrimayNotAllcoatedBefore() { + public void testNoProcessPrimacyNotAllocatedBefore() { ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, 1, true, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); assertThat(testAllocator.needToFindPrimaryCopy(shard), equalTo(false)); } @@ -83,9 +84,7 @@ public void testNoProcessPrimayNotAllcoatedBefore() { public void testNoAsyncFetchData() { RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders()); boolean changed = testAllocator.allocateUnassigned(allocation); - assertThat(changed, equalTo(false)); - assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); - assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId)); + assertNotAllocated(allocation, changed); } /** @@ -96,9 +95,7 @@ public void testNoAllocationFound() { RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders()); testAllocator.addData(node1, -1); boolean changed = testAllocator.allocateUnassigned(allocation); - assertThat(changed, equalTo(false)); - assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); - assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId)); + assertNotAllocated(allocation, changed); } /** @@ -109,9 +106,14 @@ public void testStoreException() { RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(yesAllocationDeciders()); testAllocator.addData(node1, 3, new CorruptIndexException("test", "test")); boolean changed = testAllocator.allocateUnassigned(allocation); + assertNotAllocated(allocation, changed); + } + + protected void assertNotAllocated(RoutingAllocation allocation, boolean changed) { assertThat(changed, equalTo(false)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId)); + assertThat(allocation.routingNodes().unassigned().ignored().get(0).primaryTerm(), equalTo(0)); } /** @@ -123,9 +125,18 @@ public void testFoundAllocationAndAllocating() { testAllocator.addData(node1, 10); boolean changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); + assertShardAllocated(allocation, node1); + } + + protected void assertShardAllocated(RoutingAllocation allocation, DiscoveryNode... nodes) { assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); - assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(node1.id())); + final Matcher[] nodeMatchers = new Matcher[nodes.length]; + for (int i = 0; i < nodes.length; i++) { + nodeMatchers[i] = equalTo(nodes[i].id()); + } + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), anyOf((Matcher[]) nodeMatchers)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).primaryTerm(), equalTo(1)); } /** @@ -137,9 +148,7 @@ public void testFoundAllocationButThrottlingDecider() { RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas(throttleAllocationDeciders()); testAllocator.addData(node1, 10); boolean changed = testAllocator.allocateUnassigned(allocation); - assertThat(changed, equalTo(false)); - assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); - assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId)); + assertNotAllocated(allocation, changed); } /** @@ -152,9 +161,7 @@ public void testFoundAllocationButNoDecider() { testAllocator.addData(node1, 10); boolean changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); - assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); - assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); - assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(node1.id())); + assertShardAllocated(allocation, node1); } /** @@ -166,9 +173,7 @@ public void testAllocateToTheHighestVersion() { testAllocator.addData(node1, 10).addData(node2, 12); boolean changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); - assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); - assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); - assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(node2.id())); + assertShardAllocated(allocation, node2); } /** @@ -231,10 +236,8 @@ public void testEnoughCopiesFoundForAllocation() { allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); - assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(0)); assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas - assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); - assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), anyOf(equalTo(node2.id()), equalTo(node1.id()))); + assertShardAllocated(allocation, node1, node2); } /** @@ -275,8 +278,7 @@ public void testEnoughCopiesFoundForAllocationWithDifferentVersion() { assertThat(changed, equalTo(true)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(0)); assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas - assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); - assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(node2.id())); + assertShardAllocated(allocation, node2); } @Test From 2430a3b9681d08a6fe7ac56f5da6464b1209d323 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 13 Oct 2015 15:58:46 +0300 Subject: [PATCH 13/19] hash codes for the works (and some feedback) --- .../elasticsearch/cluster/metadata/IndexMetaData.java | 10 ++++++---- .../elasticsearch/cluster/routing/ShardRouting.java | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index d4cd65c953dab..5b942de38d29b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -440,9 +440,11 @@ public int hashCode() { result = 31 * result + aliases.hashCode(); result = 31 * result + settings.hashCode(); result = 31 * result + mappings.hashCode(); + result = 31 * result + Arrays.hashCode(primaryTerms); return result; } + @Override public Diff diff(IndexMetaData previousState) { return new IndexMetaDataDiff(previousState, this); @@ -735,14 +737,14 @@ public Builder version(long version) { public int primaryTerm(int shardId) { if (primaryTerms == null) { - allocatePrimaryTerms(); + initializePrimaryTerms(); } return this.primaryTerms[shardId]; } public Builder primaryTerm(int shardId, int primaryTerm) { if (primaryTerms == null) { - allocatePrimaryTerms(); + initializePrimaryTerms(); } this.primaryTerms[shardId] = primaryTerm; return this; @@ -752,7 +754,7 @@ private void primaryTerms(int[] primaryTerms) { this.primaryTerms = primaryTerms; } - private void allocatePrimaryTerms() { + private void initializePrimaryTerms() { assert primaryTerms == null; if (numberOfShards() < 0) { throw new IllegalStateException("you must set the number of shards before setting/reading primary terms"); @@ -774,7 +776,7 @@ public IndexMetaData build() { } if (primaryTerms == null) { - allocatePrimaryTerms(); + initializePrimaryTerms(); } else if (primaryTerms.length != numberOfShards()) { throw new IllegalStateException("primaryTerms length is [" + primaryTerms.length + "] but should be equal to number of shards [" + numberOfShards() + "]"); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index 6bdae16e12b9e..da3c3c623514e 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -681,6 +681,7 @@ public int hashCode() { result = 31 * result + (currentNodeId != null ? currentNodeId.hashCode() : 0); result = 31 * result + (relocatingNodeId != null ? relocatingNodeId.hashCode() : 0); result = 31 * result + (primary ? 1 : 0); + result = 31 * result + primaryTerm; result = 31 * result + (state != null ? state.hashCode() : 0); result = 31 * result + (int) (version ^ (version >>> 32)); result = 31 * result + (restoreSource != null ? restoreSource.hashCode() : 0); From 69c98d6f4abd57ef1444158ccd5b97bf655577b4 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 15 Oct 2015 08:00:40 +0300 Subject: [PATCH 14/19] feedback review --- .../cluster/metadata/IndexMetaData.java | 47 ++++++++++++------- .../cluster/routing/ShardRouting.java | 4 +- .../gateway/PrimaryShardAllocatorTests.java | 2 +- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 5b942de38d29b..05f8d9c9ed02b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -287,7 +287,6 @@ public long getVersion() { * The term of the current selected primary. This is a non-negative number incremented when * a primary shard is assigned after a full cluster restart or a replica shard is promoted * to a primary (see {@link ShardRouting#moveToPrimary()}) - * */ public int primaryTerm(int shardId) { return this.primaryTerms[shardId]; @@ -436,10 +435,12 @@ public boolean equals(Object o) { @Override public int hashCode() { int result = index.hashCode(); + result = 31 * result + (int) (version ^ (version >>> 32)); result = 31 * result + state.hashCode(); result = 31 * result + aliases.hashCode(); result = 31 * result + settings.hashCode(); result = 31 * result + mappings.hashCode(); + result = 31 * result + customs.hashCode(); result = 31 * result + Arrays.hashCode(primaryTerms); return result; } @@ -785,21 +786,31 @@ public IndexMetaData build() { return new IndexMetaData(index, version, primaryTerms, state, tmpSettings, mappings.build(), tmpAliases.build(), customs.build()); } + static final class Fields { + static final XContentBuilderString VERSION = new XContentBuilderString("version"); + static final XContentBuilderString SETTINGS = new XContentBuilderString("settings"); + static final XContentBuilderString STATE = new XContentBuilderString("state"); + static final XContentBuilderString MAPPINGS = new XContentBuilderString("mappings"); + static final XContentBuilderString ALIASES = new XContentBuilderString("aliases"); + static final XContentBuilderString PRIMARY_TERMS = new XContentBuilderString("primary_terms"); + } + + public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(indexMetaData.getIndex(), XContentBuilder.FieldCaseConversion.NONE); - builder.field("version", indexMetaData.getVersion()); - builder.field("state", indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH)); + builder.field(Fields.VERSION, indexMetaData.getVersion()); + builder.field(Fields.STATE, indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH)); boolean binary = params.paramAsBoolean("binary", false); - builder.startObject("settings"); + builder.startObject(Fields.SETTINGS); for (Map.Entry entry : indexMetaData.getSettings().getAsMap().entrySet()) { builder.field(entry.getKey(), entry.getValue()); } builder.endObject(); - builder.startArray("mappings"); + builder.startArray(Fields.MAPPINGS); for (ObjectObjectCursor cursor : indexMetaData.getMappings()) { if (binary) { builder.value(cursor.value.source().compressed()); @@ -819,13 +830,13 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build builder.endObject(); } - builder.startObject("aliases"); + builder.startObject(Fields.ALIASES); for (ObjectCursor cursor : indexMetaData.getAliases().values()) { AliasMetaData.Builder.toXContent(cursor.value, builder, params); } builder.endObject(); - builder.startArray("primary_terms"); + builder.startArray(Fields.PRIMARY_TERMS); for (int i = 0; i < indexMetaData.getNumberOfShards(); i++) { builder.value(indexMetaData.primaryTerm(i)); } @@ -834,6 +845,11 @@ public static void toXContent(IndexMetaData indexMetaData, XContentBuilder build builder.endObject(); } + // TODO move it somewhere where it will be useful for other code? + private static boolean fieldEquals(XContentBuilderString field, String currentFieldName) { + return field.underscore().getValue().equals(currentFieldName); + } + public static IndexMetaData fromXContent(XContentParser parser) throws IOException { if (parser.currentToken() == null) { // fresh parser? move to the first token parser.nextToken(); @@ -849,9 +865,9 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("settings".equals(currentFieldName)) { + if (fieldEquals(Fields.SETTINGS, currentFieldName)) { builder.settings(Settings.settingsBuilder().put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered()))); - } else if ("mappings".equals(currentFieldName)) { + } else if (fieldEquals(Fields.MAPPINGS, currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); @@ -861,7 +877,7 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti builder.putMapping(new MappingMetaData(mappingType, mappingSource)); } } - } else if ("aliases".equals(currentFieldName)) { + } else if (fieldEquals(Fields.ALIASES, currentFieldName)) { while (parser.nextToken() != XContentParser.Token.END_OBJECT) { builder.putAlias(AliasMetaData.Builder.fromXContent(parser)); } @@ -877,7 +893,7 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti } } } else if (token == XContentParser.Token.START_ARRAY) { - if ("mappings".equals(currentFieldName)) { + if (fieldEquals(Fields.MAPPINGS, currentFieldName)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { builder.putMapping(new MappingMetaData(new CompressedXContent(parser.binaryValue()))); @@ -889,7 +905,7 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti } } } - } else if ("primary_terms".equals(currentFieldName)) { + } else if (fieldEquals(Fields.PRIMARY_TERMS, currentFieldName)) { IntArrayList list = new IntArrayList(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.VALUE_NUMBER) { @@ -898,15 +914,12 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti throw new IllegalStateException("found a non-numeric value under [primary_terms]"); } } - for (int i = 0; i < list.size(); i++) { - builder.primaryTerm(i, list.get(i)); - } builder.primaryTerms(list.toArray()); } } else if (token.isValue()) { - if ("state".equals(currentFieldName)) { + if (fieldEquals(Fields.STATE, currentFieldName)) { builder.state(State.fromString(parser.text())); - } else if ("version".equals(currentFieldName)) { + } else if (fieldEquals(Fields.VERSION, currentFieldName)) { builder.version(parser.longValue()); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index da3c3c623514e..8a64516918816 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -331,7 +331,7 @@ public void readFromThin(StreamInput in) throws IOException { } primary = in.readBoolean(); - primaryTerm = in.readInt(); + primaryTerm = in.readVInt(); state = ShardRoutingState.fromValue(in.readByte()); restoreSource = RestoreSource.readOptionalRestoreSource(in); @@ -377,7 +377,7 @@ public void writeToThin(StreamOutput out) throws IOException { } out.writeBoolean(primary); - out.writeInt(primaryTerm); + out.writeVInt(primaryTerm); out.writeByte(state.value()); if (restoreSource != null) { diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java index 60ff0c93147d9..b38a791a54ba5 100644 --- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java @@ -72,7 +72,7 @@ public void testNoProcessReplica() { } @Test - public void testNoProcessPrimacyNotAllocatedBefore() { + public void testNoProcessPrimaryNotAllocatedBefore() { ShardRouting shard = TestShardRouting.newShardRouting("test", 0, null, null, null, 1, true, ShardRoutingState.UNASSIGNED, 0, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); assertThat(testAllocator.needToFindPrimaryCopy(shard), equalTo(false)); } From ae1690cf77a1b2aa98ef1eca0cba731dbf77ca77 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 15 Oct 2015 12:26:19 +0300 Subject: [PATCH 15/19] move to writVIntArray --- .../java/org/elasticsearch/cluster/ClusterState.java | 2 +- .../elasticsearch/cluster/metadata/IndexMetaData.java | 10 +++++----- .../support/replication/BroadcastReplicationTests.java | 2 +- .../elasticsearch/gateway/RecoveryFromGatewayIT.java | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java index 055e63fda17c0..b50df2268ecee 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -469,7 +469,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endArray(); builder.startObject("primary_terms"); - for (int shard = 0; shard < indexMetaData.numberOfShards(); shard++) { + for (int shard = 0; shard < indexMetaData.getNumberOfShards(); shard++) { builder.field(Integer.toString(shard), indexMetaData.primaryTerm(shard)); } builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 05f8d9c9ed02b..e1a7336dde204 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -494,7 +494,7 @@ public IndexMetaDataDiff(StreamInput in) throws IOException { version = in.readLong(); state = State.fromId(in.readByte()); settings = Settings.readSettingsFromStream(in); - primaryTerms = in.readIntArray(); + primaryTerms = in.readVIntArray(); mappings = DiffableUtils.readImmutableOpenMapDiff(in, MappingMetaData.PROTO); aliases = DiffableUtils.readImmutableOpenMapDiff(in, AliasMetaData.PROTO); customs = DiffableUtils.readImmutableOpenMapDiff(in, new DiffableUtils.KeyedReader() { @@ -516,7 +516,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(version); out.writeByte(state.id); Settings.writeSettingsToStream(settings, out); - out.writeIntArray(primaryTerms); + out.writeVIntArray(primaryTerms); mappings.writeTo(out); aliases.writeTo(out); customs.writeTo(out); @@ -542,7 +542,7 @@ public IndexMetaData readFrom(StreamInput in) throws IOException { builder.version(in.readLong()); builder.state(State.fromId(in.readByte())); builder.settings(readSettingsFromStream(in)); - builder.primaryTerms(in.readIntArray()); + builder.primaryTerms(in.readVIntArray()); int mappingsSize = in.readVInt(); for (int i = 0; i < mappingsSize; i++) { MappingMetaData mappingMd = MappingMetaData.PROTO.readFrom(in); @@ -568,7 +568,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(version); out.writeByte(state.id()); writeSettingsToStream(settings, out); - out.writeIntArray(primaryTerms); + out.writeVIntArray(primaryTerms); out.writeVInt(mappings.size()); for (ObjectCursor cursor : mappings.values()) { cursor.value.writeTo(out); @@ -911,7 +911,7 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti if (token == XContentParser.Token.VALUE_NUMBER) { list.add(parser.intValue()); } else { - throw new IllegalStateException("found a non-numeric value under [primary_terms]"); + throw new IllegalStateException("found a non-numeric value under [" + Fields.PRIMARY_TERMS.underscore() + "]"); } } builder.primaryTerms(list.toArray()); diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java index 2fe04bb923811..c850c0f84842e 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java @@ -131,7 +131,7 @@ public void testStartedPrimary() throws InterruptedException, ExecutionException @Test public void testResultCombine() throws InterruptedException, ExecutionException, IOException { final String index = "test"; - int numShards = randomInt(3); + int numShards = 1 + randomInt(3); clusterService.setState(stateWithAssignedPrimariesAndOneReplica(index, numShards)); logger.debug("--> using initial state:\n{}", clusterService.state().prettyPrint()); Future response = (broadcastReplicationAction.execute(new BroadcastRequest().indices(index))); diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index b886de0bb5389..65787127c8810 100644 --- a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -116,9 +116,9 @@ private Map assertAndCapturePrimaryTerms(Map previ final ClusterState state = client().admin().cluster().prepareState().get().getState(); for (ObjectCursor cursor : state.metaData().indices().values()) { final IndexMetaData indexMetaData = cursor.value; - final String index = indexMetaData.index(); + final String index = indexMetaData.getIndex(); final int[] previous = previousTerms.get(index); - final int[] current = IntStream.range(0, indexMetaData.numberOfShards()).map(indexMetaData::primaryTerm).toArray(); + final int[] current = IntStream.range(0, indexMetaData.getNumberOfShards()).map(indexMetaData::primaryTerm).toArray(); if (previous == null) { result.put(index, current); } else { From f5499d3c3bdb014407923892f90c421f5eb44d58 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 16 Oct 2015 21:59:09 +0200 Subject: [PATCH 16/19] fancy hashing --- .../java/org/elasticsearch/cluster/metadata/IndexMetaData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index e1a7336dde204..14d7b3df00279 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -435,7 +435,7 @@ public boolean equals(Object o) { @Override public int hashCode() { int result = index.hashCode(); - result = 31 * result + (int) (version ^ (version >>> 32)); + result = 31 * result + Long.hashCode(version); result = 31 * result + state.hashCode(); result = 31 * result + aliases.hashCode(); result = 31 * result + settings.hashCode(); From 3d5afcb82c5d031cb835e7ae19ff65797006c4e2 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 19 Oct 2015 19:09:30 +0200 Subject: [PATCH 17/19] fix double version increments + some improved docs --- .../cluster/metadata/IndexMetaData.java | 16 +++++++---- .../cluster/routing/IndexRoutingTable.java | 3 +++ .../cluster/routing/ShardRouting.java | 5 +++- .../routing/allocation/AllocationService.java | 5 ++-- .../cluster/routing/RoutingTableTests.java | 27 ++++++++++++------- 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 14d7b3df00279..bafd23bc6e058 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -285,9 +285,9 @@ public long getVersion() { /** * The term of the current selected primary. This is a non-negative number incremented when - * a primary shard is assigned after a full cluster restart or a replica shard is promoted - * to a primary (see {@link ShardRouting#moveToPrimary()}) - */ + * a primary shard is assigned after a full cluster restart (see {@link ShardRouting#initialize(java.lang.String, long)} + * or a replica shard is promoted to a primary (see {@link ShardRouting#moveToPrimary()}). + **/ public int primaryTerm(int shardId) { return this.primaryTerms[shardId]; } @@ -632,7 +632,6 @@ public Builder index(String index) { public Builder numberOfShards(int numberOfShards) { settings = settingsBuilder().put(settings).put(SETTING_NUMBER_OF_SHARDS, numberOfShards).build(); - primaryTerms = new int[numberOfShards]; return this; } @@ -736,6 +735,10 @@ public Builder version(long version) { return this; } + /** + * returns the primary term for the given shard. + * See {@link IndexMetaData#primaryTerm(int)} for more information. + */ public int primaryTerm(int shardId) { if (primaryTerms == null) { initializePrimaryTerms(); @@ -743,6 +746,10 @@ public int primaryTerm(int shardId) { return this.primaryTerms[shardId]; } + /** + * sets the primary term for the given shard. + * See {@link IndexMetaData#primaryTerm(int)} for more information. + */ public Builder primaryTerm(int shardId, int primaryTerm) { if (primaryTerms == null) { initializePrimaryTerms(); @@ -795,7 +802,6 @@ static final class Fields { static final XContentBuilderString PRIMARY_TERMS = new XContentBuilderString("primary_terms"); } - public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(indexMetaData.getIndex(), XContentBuilder.FieldCaseConversion.NONE); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index b2bbcaab5a57f..d1956245af934 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -100,6 +100,9 @@ public String getIndex() { /** * creates a new {@link IndexRoutingTable} with all shard versions & primary terms set to the highest found. + * This allows incrementing {@link ShardRouting#version()} and {@link ShardRouting#primaryTerm()} where we work on + * the individual shards without worrying about synchronization between {@link ShardRouting} instances. This method + * takes care of it. * * @return new {@link IndexRoutingTable} */ diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index 8a64516918816..393d1e6c04e41 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -256,7 +256,10 @@ public boolean primary() { } /** - * Returns the term of the current primary shard for this shard. The term is incremented with every primary promotion/initial assignment + * Returns the term of the current primary shard for this shard. + * The term is incremented with every primary promotion/initial assignment. + * + * See {@link org.elasticsearch.cluster.metadata.IndexMetaData#primaryTerm(int)} for more info. */ public int primaryTerm() { return this.primaryTerm; diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 4800fce101eac..d34b016e339e9 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -90,8 +90,7 @@ protected RoutingAllocation.Result buildChangedResult(MetaData metaData, Routing } protected RoutingAllocation.Result buildChangedResult(MetaData metaData, RoutingNodes routingNodes, RoutingExplanations explanations) { - final RoutingTable routingTable = new RoutingTable.Builder().updateNodes(routingNodes) - .version(routingNodes.getRoutingTable().version() + 1).build(); + final RoutingTable routingTable = new RoutingTable.Builder().updateNodes(routingNodes).build(); MetaData newMetaData = updateMetaDataWithRoutingTable(metaData,routingTable); return new RoutingAllocation.Result(true, routingTable.validateRaiseException(newMetaData), newMetaData, explanations); } @@ -136,7 +135,7 @@ static MetaData updateMetaDataWithRoutingTable(MetaData currentMetaData, Routing } } if (metaDataBuilder != null) { - return metaDataBuilder.version(currentMetaData.version() + 1).build(); + return metaDataBuilder.build(); } else { return currentMetaData; } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java index a6f716153a82e..cd634bd5c2de4 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESAllocationTestCase; -import org.junit.Before; import org.junit.Test; import java.util.*; @@ -97,9 +96,8 @@ private void initPrimaries() { } this.clusterState = ClusterState.builder(clusterState).nodes(discoBuilder).build(); RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.reroute(clusterState); - this.testRoutingTable = rerouteResult.routingTable(); assertThat(rerouteResult.changed(), is(true)); - this.clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); + applyRerouteResult(rerouteResult); versionsPerIndex.keySet().forEach(this::incrementVersion); primaryTermsPerIndex.keySet().forEach(this::incrementPrimaryTerm); } @@ -130,11 +128,25 @@ private void startInitializingShards(String index) { this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build(); logger.info("start primary shards for index " + index); RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.applyStartedShards(this.clusterState, this.clusterState.getRoutingNodes().shardsWithState(index, INITIALIZING)); - this.clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build(); - this.testRoutingTable = rerouteResult.routingTable(); + // TODO: this simulate the code in InternalClusterState.UpdateTask.run() we should unify this. + applyRerouteResult(rerouteResult); incrementVersion(index); } + private void applyRerouteResult(RoutingAllocation.Result rerouteResult) { + ClusterState previousClusterState = this.clusterState; + ClusterState newClusterState = ClusterState.builder(previousClusterState).routingResult(rerouteResult).build(); + ClusterState.Builder builder = ClusterState.builder(newClusterState).incrementVersion(); + if (previousClusterState.routingTable() != newClusterState.routingTable()) { + builder.routingTable(RoutingTable.builder(newClusterState.routingTable()).version(newClusterState.routingTable().version() + 1).build()); + } + if (previousClusterState.metaData() != newClusterState.metaData()) { + builder.metaData(MetaData.builder(newClusterState.metaData()).version(newClusterState.metaData().version() + 1)); + } + this.clusterState = builder.build(); + this.testRoutingTable = rerouteResult.routingTable(); + } + private void failSomePrimaries(String index) { this.clusterState = ClusterState.builder(clusterState).routingTable(this.testRoutingTable).build(); final IndexRoutingTable indexShardRoutingTable = testRoutingTable.index(index); @@ -151,10 +163,7 @@ private void failSomePrimaries(String index) { incrementVersion(index, shard); // and another time when the primary flag is set to false } RoutingAllocation.Result rerouteResult = ALLOCATION_SERVICE.applyFailedShards(this.clusterState, failedShards); - assertThat(rerouteResult.routingTable().version(), greaterThan(clusterState.routingTable().version())); - assertThat(rerouteResult.metaData().version(), greaterThan(clusterState.metaData().version())); - this.clusterState = ClusterState.builder(clusterState).routingResult(rerouteResult).build(); - this.testRoutingTable = rerouteResult.routingTable(); + applyRerouteResult(rerouteResult); } private IndexMetaData.Builder createIndexMetaData(String indexName) { From 21d26543d95c55243bfb33c48e4306fe2d10169d Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 19 Oct 2015 20:41:45 +0200 Subject: [PATCH 18/19] move primaryTerm to a long --- .../cluster/metadata/IndexMetaData.java | 32 +++++++++---------- .../cluster/routing/IndexRoutingTable.java | 4 +-- .../routing/IndexShardRoutingTable.java | 4 +-- .../cluster/routing/ShardRouting.java | 17 +++++----- .../metadata/ToAndFromJsonMetaDataTests.java | 6 ++-- .../cluster/routing/RoutingTableTests.java | 8 ++--- .../cluster/routing/ShardRoutingTests.java | 6 ++-- .../cluster/routing/TestShardRouting.java | 10 +++--- .../PrimaryElectionRoutingTests.java | 4 +-- .../routing/allocation/ShardStateIT.java | 6 ++-- .../gateway/PrimaryShardAllocatorTests.java | 4 +-- .../gateway/RecoveryFromGatewayIT.java | 24 +++++++------- 12 files changed, 63 insertions(+), 62 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index bafd23bc6e058..36cedee1cdd5b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -19,7 +19,7 @@ package org.elasticsearch.cluster.metadata; -import com.carrotsearch.hppc.IntArrayList; +import com.carrotsearch.hppc.LongArrayList; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.Version; @@ -173,7 +173,7 @@ public static State fromString(String state) { private final String index; private final long version; - private final int[] primaryTerms; + private final long[] primaryTerms; private final State state; @@ -195,7 +195,7 @@ public static State fromString(String state) { private final Version indexUpgradedVersion; private final org.apache.lucene.util.Version minimumCompatibleLuceneVersion; - private IndexMetaData(String index, long version, int[] primaryTerms, State state, Settings settings, ImmutableOpenMap mappings, ImmutableOpenMap aliases, ImmutableOpenMap customs) { + private IndexMetaData(String index, long version, long[] primaryTerms, State state, Settings settings, ImmutableOpenMap mappings, ImmutableOpenMap aliases, ImmutableOpenMap customs) { Integer maybeNumberOfShards = settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null); if (maybeNumberOfShards == null) { throw new IllegalArgumentException("must specify numberOfShards for index [" + index + "]"); @@ -288,7 +288,7 @@ public long getVersion() { * a primary shard is assigned after a full cluster restart (see {@link ShardRouting#initialize(java.lang.String, long)} * or a replica shard is promoted to a primary (see {@link ShardRouting#moveToPrimary()}). **/ - public int primaryTerm(int shardId) { + public long primaryTerm(int shardId) { return this.primaryTerms[shardId]; } @@ -471,7 +471,7 @@ private static class IndexMetaDataDiff implements Diff { private final String index; private final long version; - private final int[] primaryTerms; + private final long[] primaryTerms; private final State state; private final Settings settings; private final Diff> mappings; @@ -494,7 +494,7 @@ public IndexMetaDataDiff(StreamInput in) throws IOException { version = in.readLong(); state = State.fromId(in.readByte()); settings = Settings.readSettingsFromStream(in); - primaryTerms = in.readVIntArray(); + primaryTerms = in.readVLongArray(); mappings = DiffableUtils.readImmutableOpenMapDiff(in, MappingMetaData.PROTO); aliases = DiffableUtils.readImmutableOpenMapDiff(in, AliasMetaData.PROTO); customs = DiffableUtils.readImmutableOpenMapDiff(in, new DiffableUtils.KeyedReader() { @@ -516,7 +516,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(version); out.writeByte(state.id); Settings.writeSettingsToStream(settings, out); - out.writeVIntArray(primaryTerms); + out.writeVLongArray(primaryTerms); mappings.writeTo(out); aliases.writeTo(out); customs.writeTo(out); @@ -542,7 +542,7 @@ public IndexMetaData readFrom(StreamInput in) throws IOException { builder.version(in.readLong()); builder.state(State.fromId(in.readByte())); builder.settings(readSettingsFromStream(in)); - builder.primaryTerms(in.readVIntArray()); + builder.primaryTerms(in.readVLongArray()); int mappingsSize = in.readVInt(); for (int i = 0; i < mappingsSize; i++) { MappingMetaData mappingMd = MappingMetaData.PROTO.readFrom(in); @@ -568,7 +568,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(version); out.writeByte(state.id()); writeSettingsToStream(settings, out); - out.writeVIntArray(primaryTerms); + out.writeVLongArray(primaryTerms); out.writeVInt(mappings.size()); for (ObjectCursor cursor : mappings.values()) { cursor.value.writeTo(out); @@ -597,7 +597,7 @@ public static class Builder { private String index; private State state = State.OPEN; private long version = 1; - private int[] primaryTerms = null; + private long[] primaryTerms = null; private Settings settings = Settings.Builder.EMPTY_SETTINGS; private final ImmutableOpenMap.Builder mappings; private final ImmutableOpenMap.Builder aliases; @@ -739,7 +739,7 @@ public Builder version(long version) { * returns the primary term for the given shard. * See {@link IndexMetaData#primaryTerm(int)} for more information. */ - public int primaryTerm(int shardId) { + public long primaryTerm(int shardId) { if (primaryTerms == null) { initializePrimaryTerms(); } @@ -750,7 +750,7 @@ public int primaryTerm(int shardId) { * sets the primary term for the given shard. * See {@link IndexMetaData#primaryTerm(int)} for more information. */ - public Builder primaryTerm(int shardId, int primaryTerm) { + public Builder primaryTerm(int shardId, long primaryTerm) { if (primaryTerms == null) { initializePrimaryTerms(); } @@ -758,7 +758,7 @@ public Builder primaryTerm(int shardId, int primaryTerm) { return this; } - private void primaryTerms(int[] primaryTerms) { + private void primaryTerms(long[] primaryTerms) { this.primaryTerms = primaryTerms; } @@ -767,7 +767,7 @@ private void initializePrimaryTerms() { if (numberOfShards() < 0) { throw new IllegalStateException("you must set the number of shards before setting/reading primary terms"); } - primaryTerms = new int[numberOfShards()]; + primaryTerms = new long[numberOfShards()]; } @@ -912,10 +912,10 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti } } } else if (fieldEquals(Fields.PRIMARY_TERMS, currentFieldName)) { - IntArrayList list = new IntArrayList(); + LongArrayList list = new LongArrayList(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.VALUE_NUMBER) { - list.add(parser.intValue()); + list.add(parser.longValue()); } else { throw new IllegalStateException("found a non-numeric value under [" + Fields.PRIMARY_TERMS.underscore() + "]"); } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index d1956245af934..62054670e214f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -425,7 +425,7 @@ private Builder initializeAsRestore(IndexMetaData indexMetaData, RestoreSource r for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) { IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId)); for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) { - final int primaryTerm = indexMetaData.primaryTerm(shardId); + final long primaryTerm = indexMetaData.primaryTerm(shardId); if (asNew && ignoreShards.contains(shardId)) { // This shards wasn't completely snapshotted - restore it as new shard indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, primaryTerm, i == 0, unassignedInfo)); @@ -446,7 +446,7 @@ private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unas throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created"); } for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) { - final int primaryTerm = indexMetaData.primaryTerm(shardId); + final long primaryTerm = indexMetaData.primaryTerm(shardId); IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId)); for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) { indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null,primaryTerm, i == 0, unassignedInfo)); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java index 7eda4d63dd780..fa3da3779ac46 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java @@ -124,11 +124,11 @@ public IndexShardRoutingTable normalizeVersionsAndPrimaryTerms() { return this; } long highestVersion = shards.get(0).version(); - int highestPrimaryTerm = shards.get(0).primaryTerm(); + long highestPrimaryTerm = shards.get(0).primaryTerm(); boolean requiresNormalization = false; for (int i = 1; i < shards.size(); i++) { final long version = shards.get(i).version(); - final int primaryTerm = shards.get(i).primaryTerm(); + final long primaryTerm = shards.get(i).primaryTerm(); if (highestVersion != version || highestPrimaryTerm != primaryTerm) { requiresNormalization = true; } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index 393d1e6c04e41..28f7137a7102b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -47,7 +47,7 @@ public final class ShardRouting implements Streamable, ToXContent { private String currentNodeId; private String relocatingNodeId; private boolean primary; - private int primaryTerm; + private long primaryTerm; private ShardRoutingState state; private long version; private RestoreSource restoreSource; @@ -70,7 +70,7 @@ public ShardRouting(ShardRouting copy, long version) { this(copy, version, copy.primaryTerm()); } - public ShardRouting(ShardRouting copy, long version, int primaryTerm) { + public ShardRouting(ShardRouting copy, long version, long primaryTerm) { this(copy.index(), copy.id(), copy.currentNodeId(), copy.relocatingNodeId(), copy.restoreSource(), primaryTerm, copy.primary(), copy.state(), version, copy.unassignedInfo(), copy.allocationId(), true, copy.getExpectedShardSize()); } @@ -79,7 +79,7 @@ public ShardRouting(ShardRouting copy, long version, int primaryTerm) { * by either this class or tests. Visible for testing. */ ShardRouting(String index, int shardId, String currentNodeId, - String relocatingNodeId, RestoreSource restoreSource, int primaryTerm, boolean primary, ShardRoutingState state, long version, + String relocatingNodeId, RestoreSource restoreSource, long primaryTerm, boolean primary, ShardRoutingState state, long version, UnassignedInfo unassignedInfo, AllocationId allocationId, boolean internal, long expectedShardSize) { this.index = index; this.shardId = shardId; @@ -109,7 +109,7 @@ public ShardRouting(ShardRouting copy, long version, int primaryTerm) { /** * Creates a new unassigned shard. */ - public static ShardRouting newUnassigned(String index, int shardId, RestoreSource restoreSource, int primaryTerm, boolean primary, UnassignedInfo unassignedInfo) { + public static ShardRouting newUnassigned(String index, int shardId, RestoreSource restoreSource, long primaryTerm, boolean primary, UnassignedInfo unassignedInfo) { return new ShardRouting(index, shardId, null, null, restoreSource, primaryTerm, primary, ShardRoutingState.UNASSIGNED, 0, unassignedInfo, null, true, UNAVAILABLE_EXPECTED_SHARD_SIZE); } @@ -261,7 +261,7 @@ public boolean primary() { * * See {@link org.elasticsearch.cluster.metadata.IndexMetaData#primaryTerm(int)} for more info. */ - public int primaryTerm() { + public long primaryTerm() { return this.primaryTerm; } @@ -334,7 +334,7 @@ public void readFromThin(StreamInput in) throws IOException { } primary = in.readBoolean(); - primaryTerm = in.readVInt(); + primaryTerm = in.readVLong(); state = ShardRoutingState.fromValue(in.readByte()); restoreSource = RestoreSource.readOptionalRestoreSource(in); @@ -380,7 +380,7 @@ public void writeToThin(StreamOutput out) throws IOException { } out.writeBoolean(primary); - out.writeVInt(primaryTerm); + out.writeVLong(primaryTerm); out.writeByte(state.value()); if (restoreSource != null) { @@ -684,7 +684,8 @@ public int hashCode() { result = 31 * result + (currentNodeId != null ? currentNodeId.hashCode() : 0); result = 31 * result + (relocatingNodeId != null ? relocatingNodeId.hashCode() : 0); result = 31 * result + (primary ? 1 : 0); - result = 31 * result + primaryTerm; + result = 31 * result + (int) (primaryTerm ^ (primaryTerm >>> 32)); + ; result = 31 * result + (state != null ? state.hashCode() : 0); result = 31 * result + (int) (version ^ (version >>> 32)); result = 31 * result + (restoreSource != null ? restoreSource.hashCode() : 0); diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java index f925df3e9e05e..d6cfc7f980c45 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java @@ -154,7 +154,7 @@ public void testSimpleJsonFromAndTo() throws IOException { MetaData parsedMetaData = MetaData.Builder.fromXContent(XContentFactory.xContent(XContentType.JSON).createParser(metaDataSource)); IndexMetaData indexMetaData = parsedMetaData.index("test1"); - assertThat(indexMetaData.primaryTerm(0), equalTo(1)); + assertThat(indexMetaData.primaryTerm(0), equalTo(1l)); assertThat(indexMetaData.getNumberOfShards(), equalTo(1)); assertThat(indexMetaData.getNumberOfReplicas(), equalTo(2)); assertThat(indexMetaData.getCreationDate(), equalTo(-1l)); @@ -164,8 +164,8 @@ public void testSimpleJsonFromAndTo() throws IOException { indexMetaData = parsedMetaData.index("test2"); assertThat(indexMetaData.getNumberOfShards(), equalTo(2)); assertThat(indexMetaData.getNumberOfReplicas(), equalTo(3)); - assertThat(indexMetaData.primaryTerm(0), equalTo(2)); - assertThat(indexMetaData.primaryTerm(1), equalTo(2)); + assertThat(indexMetaData.primaryTerm(0), equalTo(2l)); + assertThat(indexMetaData.primaryTerm(1), equalTo(2l)); assertThat(indexMetaData.getCreationDate(), equalTo(-1l)); assertThat(indexMetaData.getSettings().getAsMap().size(), equalTo(5)); assertThat(indexMetaData.getSettings().get("setting1"), equalTo("value1")); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java index cd634bd5c2de4..2f80963c095ba 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java @@ -56,7 +56,7 @@ public class RoutingTableTests extends ESAllocationTestCase { .build()); private ClusterState clusterState; - private final Map primaryTermsPerIndex = new HashMap<>(); + private final Map primaryTermsPerIndex = new HashMap<>(); private final Map versionsPerIndex = new HashMap<>(); @Override @@ -114,7 +114,7 @@ private void incrementVersion(String index, int shard) { } private void incrementPrimaryTerm(String index) { - final int[] primaryTerms = primaryTermsPerIndex.get(index); + final long[] primaryTerms = primaryTermsPerIndex.get(index); for (int i = 0; i < primaryTerms.length; i++) { primaryTerms[i]++; } @@ -167,7 +167,7 @@ private void failSomePrimaries(String index) { } private IndexMetaData.Builder createIndexMetaData(String indexName) { - primaryTermsPerIndex.put(indexName, new int[numberOfShards]); + primaryTermsPerIndex.put(indexName, new long[numberOfShards]); final IndexMetaData.Builder builder = new IndexMetaData.Builder(indexName) .settings(DEFAULT_SETTINGS) .numberOfReplicas(this.numberOfReplicas) @@ -185,7 +185,7 @@ private void assertAllVersionAndPrimaryTerm() { private void assertVersionAndPrimaryTerm(String index) { final long[] versions = versionsPerIndex.get(index); - final int[] terms = primaryTermsPerIndex.get(index); + final long[] terms = primaryTermsPerIndex.get(index); final IndexMetaData indexMetaData = clusterState.metaData().index(index); for (IndexShardRoutingTable shardRoutingTable : this.testRoutingTable.index(index)) { final int shard = shardRoutingTable.shardId().id(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java index 183ff7484f3d7..c547994b3fdc8 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java @@ -35,7 +35,7 @@ public class ShardRoutingTests extends ESTestCase { public void testFrozenAfterRead() throws IOException { - int term = randomInt(200); + long term = randomInt(200); ShardRouting routing = TestShardRouting.newShardRouting("foo", 1, "node_1", null, null, term, false, ShardRoutingState.INITIALIZING, 1); routing.moveToPrimary(); assertTrue(routing.primary()); @@ -53,7 +53,7 @@ public void testFrozenAfterRead() throws IOException { } public void testPrimaryTermIncrementOnPromotion() { - int term = randomInt(200); + long term = randomInt(200); ShardRouting routing = TestShardRouting.newShardRouting("foo", 1, "node_1", null, null, term, false, ShardRoutingState.STARTED, 1); routing.moveToPrimary(); assertTrue(routing.primary()); @@ -64,7 +64,7 @@ public void testPrimaryTermIncrementOnPromotion() { } public void testIsSameAllocation() { - int term = randomInt(200); + long term = randomInt(200); ShardRouting unassignedShard0 = TestShardRouting.newShardRouting("test", 0, null, term, false, ShardRoutingState.UNASSIGNED, 1); ShardRouting unassignedShard1 = TestShardRouting.newShardRouting("test", 1, null, term, false, ShardRoutingState.UNASSIGNED, 1); ShardRouting initializingShard0 = TestShardRouting.newShardRouting("test", 0, "1", term, randomBoolean(), ShardRoutingState.INITIALIZING, 1); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/core/src/test/java/org/elasticsearch/cluster/routing/TestShardRouting.java index 0dae058f7ba62..8a173ca4393d8 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/TestShardRouting.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/TestShardRouting.java @@ -27,29 +27,29 @@ */ public class TestShardRouting { - public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, int primaryTerm, boolean primary, + public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, long primaryTerm, boolean primary, ShardRoutingState state, long version) { return new ShardRouting(index, shardId, currentNodeId, null, null, primaryTerm, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1); } public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, - int primaryTerm, boolean primary, ShardRoutingState state, long version) { + long primaryTerm, boolean primary, ShardRoutingState state, long version) { return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primaryTerm, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1); } public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, - int primaryTerm, boolean primary, ShardRoutingState state, AllocationId allocationId, long version) { + long primaryTerm, boolean primary, ShardRoutingState state, AllocationId allocationId, long version) { return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primaryTerm, primary, state, version, buildUnassignedInfo(state), allocationId, true, -1); } public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, - RestoreSource restoreSource, int primaryTerm, boolean primary, + RestoreSource restoreSource, long primaryTerm, boolean primary, ShardRoutingState state, long version) { return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primaryTerm, primary, state, version, buildUnassignedInfo(state), buildAllocationId(state), true, -1); } public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, - String relocatingNodeId, RestoreSource restoreSource, int primaryTerm, boolean primary, + String relocatingNodeId, RestoreSource restoreSource, long primaryTerm, boolean primary, ShardRoutingState state, long version, UnassignedInfo unassignedInfo) { return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primaryTerm, primary, state, version, unassignedInfo, buildAllocationId(state), true, -1); } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java index bddbd414dad8d..81ff56be3482c 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java @@ -94,7 +94,7 @@ public void testBackupElectionToPrimaryWhenPrimaryCanBeAllocatedToAnotherNode() assertThat(routingNodes.node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1)); // verify where the primary is assertThat(routingTable.index("test").shard(0).primaryShard().currentNodeId(), equalTo("node2")); - assertThat(routingTable.index("test").shard(0).primaryShard().primaryTerm(), equalTo(2)); + assertThat(routingTable.index("test").shard(0).primaryShard().primaryTerm(), equalTo(2l)); assertThat(routingTable.index("test").shard(0).replicaShards().get(0).currentNodeId(), equalTo("node3")); } @@ -142,7 +142,7 @@ public void testRemovingInitializingReplicasIfPrimariesFails() { assertThat(routingNodes.shardsWithState(STARTED).size(), equalTo(1)); assertThat(routingNodes.shardsWithState(INITIALIZING).size(), equalTo(1)); assertThat(routingNodes.node(nodeIdRemaining).shardsWithState(INITIALIZING).get(0).primary(), equalTo(true)); - assertThat(routingNodes.node(nodeIdRemaining).shardsWithState(INITIALIZING).get(0).primaryTerm(), equalTo(2)); + assertThat(routingNodes.node(nodeIdRemaining).shardsWithState(INITIALIZING).get(0).primaryTerm(), equalTo(2l)); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java index e761f4b351e58..3a413a6245e5a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardStateIT.java @@ -48,8 +48,8 @@ public void testPrimaryFailureIncreasesTerm() throws Exception { logger.info("--> waiting for a yellow index"); assertBusy(() -> assertThat(client().admin().cluster().prepareHealth().get().getStatus(), equalTo(ClusterHealthStatus.YELLOW))); - final int term0 = shard == 0 ? 2 : 1; - final int term1 = shard == 1 ? 2 : 1; + final long term0 = shard == 0 ? 2 : 1; + final long term1 = shard == 1 ? 2 : 1; assertPrimaryTerms(term0, term1); logger.info("--> enabling allocation"); @@ -58,7 +58,7 @@ public void testPrimaryFailureIncreasesTerm() throws Exception { assertPrimaryTerms(term0, term1); } - protected void assertPrimaryTerms(int term0, int term1) { + protected void assertPrimaryTerms(long term0, long term1) { for (String node : internalCluster().getNodeNames()) { logger.debug("--> asserting primary terms terms on [{}]", node); ClusterState state = client(node).admin().cluster().prepareState().setLocal(true).get().getState(); diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java index b38a791a54ba5..9857b8662ad5c 100644 --- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java @@ -113,7 +113,7 @@ protected void assertNotAllocated(RoutingAllocation allocation, boolean changed) assertThat(changed, equalTo(false)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId)); - assertThat(allocation.routingNodes().unassigned().ignored().get(0).primaryTerm(), equalTo(0)); + assertThat(allocation.routingNodes().unassigned().ignored().get(0).primaryTerm(), equalTo(0l)); } /** @@ -136,7 +136,7 @@ protected void assertShardAllocated(RoutingAllocation allocation, DiscoveryNode. nodeMatchers[i] = equalTo(nodes[i].id()); } assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), anyOf((Matcher[]) nodeMatchers)); - assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).primaryTerm(), equalTo(1)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).primaryTerm(), equalTo(1L)); } /** diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 65787127c8810..d050af54581fa 100644 --- a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -88,7 +88,7 @@ public void testOneNodeRecoverFromGateway() throws Exception { ensureYellow("test"); // wait for primary allocations here otherwise if we have a lot of shards we might have a // shard that is still in post recovery when we restart and the ensureYellow() below will timeout - Map primaryTerms = assertAndCapturePrimaryTerms(null); + Map primaryTerms = assertAndCapturePrimaryTerms(null); internalCluster().fullRestart(); logger.info("Running Cluster Health (wait for the shards to startup)"); @@ -108,17 +108,17 @@ public void testOneNodeRecoverFromGateway() throws Exception { assertHitCount(client().prepareCount().setQuery(termQuery("appAccountIds", 179)).execute().actionGet(), 2); } - private Map assertAndCapturePrimaryTerms(Map previousTerms) { + private Map assertAndCapturePrimaryTerms(Map previousTerms) { if (previousTerms == null) { previousTerms = new HashMap<>(); } - final Map result = new HashMap<>(); + final Map result = new HashMap<>(); final ClusterState state = client().admin().cluster().prepareState().get().getState(); for (ObjectCursor cursor : state.metaData().indices().values()) { final IndexMetaData indexMetaData = cursor.value; final String index = indexMetaData.getIndex(); - final int[] previous = previousTerms.get(index); - final int[] current = IntStream.range(0, indexMetaData.getNumberOfShards()).map(indexMetaData::primaryTerm).toArray(); + final long[] previous = previousTerms.get(index); + final long[] current = IntStream.range(0, indexMetaData.getNumberOfShards()).mapToLong(indexMetaData::primaryTerm).toArray(); if (previous == null) { result.put(index, current); } else { @@ -131,7 +131,7 @@ private Map assertAndCapturePrimaryTerms(Map previ } for (IndexRoutingTable indexRoutingTable : state.routingTable()) { - final int[] terms = result.get(indexRoutingTable.index()); + final long[] terms = result.get(indexRoutingTable.index()); for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) { for (ShardRouting routing : shardRoutingTable.shards()) { assertThat("wrong primary term for " + routing, routing.primaryTerm(), equalTo(terms[routing.shardId().id()])); @@ -202,7 +202,7 @@ SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, 1) ensureYellow(); } - Map primaryTerms = assertAndCapturePrimaryTerms(null); + Map primaryTerms = assertAndCapturePrimaryTerms(null); internalCluster().fullRestart(); @@ -246,7 +246,7 @@ public void testSingleNodeWithFlush() throws Exception { ensureYellow("test"); // wait for primary allocations here otherwise if we have a lot of shards we might have a // shard that is still in post recovery when we restart and the ensureYellow() below will timeout - Map primaryTerms = assertAndCapturePrimaryTerms(null); + Map primaryTerms = assertAndCapturePrimaryTerms(null); internalCluster().fullRestart(); @@ -287,7 +287,7 @@ public void testTwoNodeFirstNodeCleared() throws Exception { assertHitCount(client().prepareCount().setQuery(matchAllQuery()).execute().actionGet(), 2); } - Map primaryTerms = assertAndCapturePrimaryTerms(null); + Map primaryTerms = assertAndCapturePrimaryTerms(null); internalCluster().fullRestart(new RestartCallback() { @Override @@ -331,7 +331,7 @@ public void testLatestVersionLoaded() throws Exception { String metaDataUuid = client().admin().cluster().prepareState().execute().get().getState().getMetaData().clusterUUID(); assertThat(metaDataUuid, not(equalTo("_na_"))); - Map primaryTerms = assertAndCapturePrimaryTerms(null); + Map primaryTerms = assertAndCapturePrimaryTerms(null); logger.info("--> closing first node, and indexing more data to the second node"); internalCluster().fullRestart(new RestartCallback() { @@ -448,7 +448,7 @@ public void testReusePeerRecovery() throws Exception { .put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE, EnableAllocationDecider.Allocation.NONE)) .get(); - Map primaryTerms = assertAndCapturePrimaryTerms(null); + Map primaryTerms = assertAndCapturePrimaryTerms(null); logger.info("--> full cluster restart"); internalCluster().fullRestart(); @@ -511,7 +511,7 @@ public void testRecoveryDifferentNodeOrderStartup() throws Exception { internalCluster().startNode(settingsBuilder().put("path.data", createTempDir()).build()); ensureGreen(); - Map primaryTerms = assertAndCapturePrimaryTerms(null); + Map primaryTerms = assertAndCapturePrimaryTerms(null); internalCluster().fullRestart(new RestartCallback() { From a13a424c76880ca434cf46de5ad890cf69e74310 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Wed, 21 Oct 2015 18:07:37 +0200 Subject: [PATCH 19/19] final feeback --- .../org/elasticsearch/cluster/metadata/IndexMetaData.java | 4 ++-- .../java/org/elasticsearch/cluster/routing/ShardRouting.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 36cedee1cdd5b..5c03ba8b163bd 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -615,7 +615,7 @@ public Builder(IndexMetaData indexMetaData) { this.state = indexMetaData.state; this.version = indexMetaData.version; this.settings = indexMetaData.getSettings(); - this.primaryTerms = Arrays.copyOf(indexMetaData.primaryTerms, indexMetaData.primaryTerms.length); + this.primaryTerms = indexMetaData.primaryTerms.clone(); this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings); this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases); this.customs = ImmutableOpenMap.builder(indexMetaData.customs); @@ -759,7 +759,7 @@ public Builder primaryTerm(int shardId, long primaryTerm) { } private void primaryTerms(long[] primaryTerms) { - this.primaryTerms = primaryTerms; + this.primaryTerms = primaryTerms.clone(); } private void initializePrimaryTerms() { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index 28f7137a7102b..b7ebb8cbb28e8 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -684,8 +684,7 @@ public int hashCode() { result = 31 * result + (currentNodeId != null ? currentNodeId.hashCode() : 0); result = 31 * result + (relocatingNodeId != null ? relocatingNodeId.hashCode() : 0); result = 31 * result + (primary ? 1 : 0); - result = 31 * result + (int) (primaryTerm ^ (primaryTerm >>> 32)); - ; + result = 31 * result + Long.hashCode(primaryTerm); result = 31 * result + (state != null ? state.hashCode() : 0); result = 31 * result + (int) (version ^ (version >>> 32)); result = 31 * result + (restoreSource != null ? restoreSource.hashCode() : 0);