[CCR] Adjust list of dynamic index settings that should be replicated (#35195)

Adjust list of dynamic index settings that should be replicated
and added a test that verifies whether builtin dynamic index settings
are classified as replicated or non replicated (whitelisted).

Author: martijnvg

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java

@@ -15,8 +15,10 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -28,8 +30,11 @@
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexingSlowLog;
+import org.elasticsearch.index.MergePolicyConfig;
+import org.elasticsearch.index.MergeSchedulerConfig;
 import org.elasticsearch.index.SearchSlowLog;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
+import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesRequestCache;
@@ -335,21 +340,37 @@ static String[] extractLeaderShardHistoryUUIDs(Map<String, String> ccrIndexMetaD
         return historyUUIDs.split(",");
     }
 
-    private static final Set<Setting<?>> WHITE_LISTED_SETTINGS;
+    /**
+     * These are settings that are not replicated to the follower index and
+     * therefor these settings are not validated whether they have the same
+     * value between leader and follower index.
+     *
+     * These dynamic settings don't affect how documents are indexed (affect index time text analysis) and / or
+     * are inconvenient if they were replicated (e.g. changing number of replicas).
+     */
+    static final Set<Setting<?>> WHITE_LISTED_SETTINGS;
 
     static {
         final Set<Setting<?>> whiteListedSettings = new HashSet<>();
         whiteListedSettings.add(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING);
         whiteListedSettings.add(IndexMetaData.INDEX_AUTO_EXPAND_REPLICAS_SETTING);
-
         whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING);
         whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING);
         whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING);
+        whiteListedSettings.add(IndexMetaData.INDEX_READ_ONLY_SETTING);
+        whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_READ_SETTING);
+        whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING);
+        whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_METADATA_SETTING);
+        whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING);
+        whiteListedSettings.add(IndexMetaData.INDEX_PRIORITY_SETTING);
+        whiteListedSettings.add(IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS);
+
         whiteListedSettings.add(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING);
         whiteListedSettings.add(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING);
         whiteListedSettings.add(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING);
+        whiteListedSettings.add(MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY);
+        whiteListedSettings.add(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING);
 
-        whiteListedSettings.add(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING);
         whiteListedSettings.add(IndexSettings.MAX_RESULT_WINDOW_SETTING);
         whiteListedSettings.add(IndexSettings.INDEX_WARMER_ENABLED_SETTING);
         whiteListedSettings.add(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING);
@@ -361,6 +382,26 @@ static String[] extractLeaderShardHistoryUUIDs(Map<String, String> ccrIndexMetaD
         whiteListedSettings.add(IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD);
         whiteListedSettings.add(IndexSettings.ALLOW_UNMAPPED);
         whiteListedSettings.add(IndexSettings.INDEX_SEARCH_IDLE_AFTER);
+        whiteListedSettings.add(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING);
+        whiteListedSettings.add(IndexSettings.MAX_SCRIPT_FIELDS_SETTING);
+        whiteListedSettings.add(IndexSettings.MAX_REGEX_LENGTH_SETTING);
+        whiteListedSettings.add(IndexSettings.MAX_TERMS_COUNT_SETTING);
+        whiteListedSettings.add(IndexSettings.MAX_ANALYZED_OFFSET_SETTING);
+        whiteListedSettings.add(IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING);
+        whiteListedSettings.add(IndexSettings.MAX_TOKEN_COUNT_SETTING);
+        whiteListedSettings.add(IndexSettings.MAX_SLICES_PER_SCROLL);
+        whiteListedSettings.add(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING);
+        whiteListedSettings.add(IndexSettings.DEFAULT_PIPELINE);
+        whiteListedSettings.add(IndexSettings.INDEX_SEARCH_THROTTLED);
+        whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING);
+        whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING);
+        whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING);
+        whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING);
+        whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING);
+        whiteListedSettings.add(IndexSettings.INDEX_GC_DELETES_SETTING);
+        whiteListedSettings.add(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD);
+
+        whiteListedSettings.add(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING);
         whiteListedSettings.add(BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING);
 
         whiteListedSettings.add(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING);
@@ -380,7 +421,20 @@ static String[] extractLeaderShardHistoryUUIDs(Map<String, String> ccrIndexMetaD
         whiteListedSettings.add(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING);
         whiteListedSettings.add(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING);
 
-        whiteListedSettings.add(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING);
+        whiteListedSettings.add(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING);
+        whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING);
+        whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING);
+        whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING);
+        whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING);
+        whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING);
+        whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING);
+        whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING);
+        whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING);
+
+        whiteListedSettings.add(MergeSchedulerConfig.AUTO_THROTTLE_SETTING);
+        whiteListedSettings.add(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING);
+        whiteListedSettings.add(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING);
+        whiteListedSettings.add(EngineConfig.INDEX_CODEC_SETTING);
 
         WHITE_LISTED_SETTINGS = Collections.unmodifiableSet(whiteListedSettings);
     }

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java

@@ -9,9 +9,12 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData.State;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.ccr.Ccr;
@@ -21,11 +24,14 @@
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import static java.util.Collections.singletonMap;
 import static org.elasticsearch.xpack.ccr.action.TransportResumeFollowAction.validate;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 
 public class TransportResumeFollowActionTests extends ESTestCase {
 
@@ -202,6 +208,32 @@ public void testValidation() throws IOException {
             validate(request, leaderIMD, followIMD, UUIDs, mapperService);
         }
     }
+    
+    public void testDynamicIndexSettingsAreClassified() {
+        // We should be conscious which dynamic settings are replicated from leader to follower index.
+        // This is the list of settings that should be replicated:
+        Set<Setting<?>> replicatedSettings = new HashSet<>();
+
+        // These fields need to be replicated otherwise documents that can be indexed in the leader index cannot
+        // be indexed in the follower index:
+        replicatedSettings.add(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
+        replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING);
+        replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING);
+        replicatedSettings.add(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING);
+        replicatedSettings.add(MapperService.INDEX_MAPPER_DYNAMIC_SETTING);
+        replicatedSettings.add(IndexSettings.MAX_NGRAM_DIFF_SETTING);
+        replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING);
+        replicatedSettings.add(EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS);
+
+        for (Setting<?> setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) {
+            if (setting.isDynamic()) {
+                boolean notReplicated = TransportResumeFollowAction.WHITE_LISTED_SETTINGS.contains(setting);
+                boolean replicated = replicatedSettings.contains(setting);
+                assertThat("setting [" + setting.getKey() + "] is not classified as replicated xor not replicated",
+                    notReplicated ^ replicated, is(true));
+            }
+        }
+    }
 
     private static IndexMetaData createIMD(String index,
                                            int numberOfShards,