From 7807e3bc9193e668274093611aa13f63454ff821 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 2 Nov 2018 11:13:12 +0100 Subject: [PATCH 1/2] [CCR] Adjust list of dynamic index settings that should be replicated and added a test that verifies whether builtin dynamic index settings are classified as replicated or non replicated (whitelisted). --- .../action/TransportResumeFollowAction.java | 62 +++++++++++++++++-- .../TransportResumeFollowActionTests.java | 32 ++++++++++ 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java index d088d0f551e98..f0e62382ecea4 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java @@ -13,7 +13,9 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; @@ -25,8 +27,11 @@ import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexingSlowLog; +import org.elasticsearch.index.MergePolicyConfig; +import org.elasticsearch.index.MergeSchedulerConfig; import org.elasticsearch.index.SearchSlowLog; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; +import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesRequestCache; @@ -320,21 +325,37 @@ static String[] extractLeaderShardHistoryUUIDs(Map ccrIndexMetaD return historyUUIDs.split(","); } - private static final Set> WHITE_LISTED_SETTINGS; + /** + * These are settings that are not replicated to the follower index and + * therefor these settings are not validated whether they have the same + * value between leader and follower index. + * + * These dynamic settings don't affect how documents are indexed (affect index time text analysis) and / or + * are inconvenient if they were replicated (e.g. changing number of replicas). + */ + static final Set> WHITE_LISTED_SETTINGS; static { final Set> whiteListedSettings = new HashSet<>(); whiteListedSettings.add(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING); whiteListedSettings.add(IndexMetaData.INDEX_AUTO_EXPAND_REPLICAS_SETTING); - whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING); whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING); whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING); + whiteListedSettings.add(IndexMetaData.INDEX_READ_ONLY_SETTING); + whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_READ_SETTING); + whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING); + whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_METADATA_SETTING); + whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING); + whiteListedSettings.add(IndexMetaData.INDEX_PRIORITY_SETTING); + whiteListedSettings.add(IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS); + whiteListedSettings.add(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING); whiteListedSettings.add(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING); whiteListedSettings.add(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING); + whiteListedSettings.add(MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY); + whiteListedSettings.add(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING); - whiteListedSettings.add(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING); whiteListedSettings.add(IndexSettings.MAX_RESULT_WINDOW_SETTING); whiteListedSettings.add(IndexSettings.INDEX_WARMER_ENABLED_SETTING); whiteListedSettings.add(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING); @@ -346,6 +367,26 @@ static String[] extractLeaderShardHistoryUUIDs(Map ccrIndexMetaD whiteListedSettings.add(IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD); whiteListedSettings.add(IndexSettings.ALLOW_UNMAPPED); whiteListedSettings.add(IndexSettings.INDEX_SEARCH_IDLE_AFTER); + whiteListedSettings.add(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING); + whiteListedSettings.add(IndexSettings.MAX_SCRIPT_FIELDS_SETTING); + whiteListedSettings.add(IndexSettings.MAX_REGEX_LENGTH_SETTING); + whiteListedSettings.add(IndexSettings.MAX_TERMS_COUNT_SETTING); + whiteListedSettings.add(IndexSettings.MAX_ANALYZED_OFFSET_SETTING); + whiteListedSettings.add(IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING); + whiteListedSettings.add(IndexSettings.MAX_TOKEN_COUNT_SETTING); + whiteListedSettings.add(IndexSettings.MAX_SLICES_PER_SCROLL); + whiteListedSettings.add(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING); + whiteListedSettings.add(IndexSettings.DEFAULT_PIPELINE); + whiteListedSettings.add(IndexSettings.INDEX_SEARCH_THROTTLED); + whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING); + whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING); + whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING); + whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING); + whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING); + whiteListedSettings.add(IndexSettings.INDEX_GC_DELETES_SETTING); + whiteListedSettings.add(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD); + + whiteListedSettings.add(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING); whiteListedSettings.add(BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING); whiteListedSettings.add(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING); @@ -365,7 +406,20 @@ static String[] extractLeaderShardHistoryUUIDs(Map ccrIndexMetaD whiteListedSettings.add(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING); whiteListedSettings.add(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING); - whiteListedSettings.add(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING); + whiteListedSettings.add(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING); + whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING); + whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING); + whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING); + whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING); + whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING); + whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING); + whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING); + whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING); + + whiteListedSettings.add(MergeSchedulerConfig.AUTO_THROTTLE_SETTING); + whiteListedSettings.add(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING); + whiteListedSettings.add(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING); + whiteListedSettings.add(EngineConfig.INDEX_CODEC_SETTING); WHITE_LISTED_SETTINGS = Collections.unmodifiableSet(whiteListedSettings); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 5a4b41e3f456a..33deb2964bb28 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -9,9 +9,12 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData.State; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ccr.Ccr; @@ -21,11 +24,14 @@ import java.io.IOException; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import static java.util.Collections.singletonMap; import static org.elasticsearch.xpack.ccr.action.TransportResumeFollowAction.validate; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; public class TransportResumeFollowActionTests extends ESTestCase { @@ -202,6 +208,32 @@ public void testValidation() throws IOException { validate(request, leaderIMD, followIMD, UUIDs, mapperService); } } + + public void testDynamicIndexSettingsAreClassified() { + // We should be conscious which dynamic settings are replicated from leader to follower index. + // This is the list of settings that should be replicated: + Set> replicatedSettings = new HashSet<>(); + + // These fields need to be replicated otherwise documents that can be indexed in the leader index cannot + // be indexed in the follower index: + replicatedSettings.add(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING); + replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING); + replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING); + replicatedSettings.add(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING); + replicatedSettings.add(MapperService.INDEX_MAPPER_DYNAMIC_SETTING); + replicatedSettings.add(IndexSettings.MAX_NGRAM_DIFF_SETTING); + replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING); + replicatedSettings.add(EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS); + + for (Setting setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) { + if (setting.isDynamic()) { + boolean notReplicated = TransportResumeFollowAction.WHITE_LISTED_SETTINGS.contains(setting); + boolean replicated = replicatedSettings.contains(setting); + assertThat("setting [" + setting.getKey() + "] is not classified as replicated or not replicated", + notReplicated || replicated, is(true)); + } + } + } private static IndexMetaData createIMD(String index, int numberOfShards, From 6a6f2249c98a35d8b3bec6241d72646be57fe554 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 7 Nov 2018 21:55:26 -0500 Subject: [PATCH 2/2] Use xor --- .../xpack/ccr/action/TransportResumeFollowActionTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 33deb2964bb28..a6ef54b3403f4 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -229,8 +229,8 @@ public void testDynamicIndexSettingsAreClassified() { if (setting.isDynamic()) { boolean notReplicated = TransportResumeFollowAction.WHITE_LISTED_SETTINGS.contains(setting); boolean replicated = replicatedSettings.contains(setting); - assertThat("setting [" + setting.getKey() + "] is not classified as replicated or not replicated", - notReplicated || replicated, is(true)); + assertThat("setting [" + setting.getKey() + "] is not classified as replicated xor not replicated", + notReplicated ^ replicated, is(true)); } } }