Add limit to total number of fields in mapping. #17357

This is to prevent mapping explosion when dynamic keys such as UUID are used as field names. index.mapping.total_fields.limit specifies the total number of fields an index can have. An exception will be thrown when the limit is reached. The default limit is 1000. Value 0 means no limit. This setting is runtime adjustable

Closes #11443

Author: yanjunh

core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -128,6 +128,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         PercolatorQueryCache.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING,
         MapperService.INDEX_MAPPER_DYNAMIC_SETTING,
         MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING,
+        MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
         BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
         IndexModule.INDEX_STORE_TYPE_SETTING,
         IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING,

core/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -84,6 +84,8 @@ public enum MergeReason {
     public static final String DEFAULT_MAPPING = "_default_";
     public static final Setting<Long> INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING =
         Setting.longSetting("index.mapping.nested_fields.limit", 50L, 0, Property.Dynamic, Property.IndexScope);
+    public static final Setting<Long> INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING =
+        Setting.longSetting("index.mapping.total_fields.limit", 1000L, 0, Property.Dynamic, Property.IndexScope);
     public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true;
     public static final Setting<Boolean> INDEX_MAPPER_DYNAMIC_SETTING =
         Setting.boolSetting("index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT, Property.IndexScope);
@@ -289,6 +291,7 @@ private synchronized DocumentMapper merge(DocumentMapper mapper, MergeReason rea
             // deserializing cluster state that was sent by the master node,
             // this check will be skipped.
             checkNestedFieldsLimit(fullPathObjectMappers);
+            checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size());
         }
 
         Set<String> parentTypes = this.parentTypes;
@@ -403,11 +406,18 @@ private void checkNestedFieldsLimit(Map<String, ObjectMapper> fullPathObjectMapp
                 actualNestedFields++;
             }
         }
-        if (allowedNestedFields >= 0 && actualNestedFields > allowedNestedFields) {
+        if (actualNestedFields > allowedNestedFields) {
             throw new IllegalArgumentException("Limit of nested fields [" + allowedNestedFields + "] in index [" + index().getName() + "] has been exceeded");
         }
     }
 
+    private void checkTotalFieldsLimit(long totalMappers) {
+        long allowedTotalFields = indexSettings.getValue(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
+        if (allowedTotalFields < totalMappers) {
+            throw new IllegalArgumentException("Limit of total fields [" + allowedTotalFields + "] in index [" + index().getName() + "] has been exceeded");
+        }
+    }
+
     public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException {
         String defaultMappingSource;
         if (PercolatorFieldMapper.TYPE_NAME.equals(mappingType)) {

core/src/test/java/org/elasticsearch/cluster/SimpleClusterStateIT.java

@@ -31,6 +31,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.CollectionAssertions;
 import org.junit.Before;
@@ -145,7 +146,9 @@ public void testLargeClusterStatePublishing() throws Exception {
         int numberOfShards = scaledRandomIntBetween(1, cluster().numDataNodes());
         // if the create index is ack'ed, then all nodes have successfully processed the cluster state
         assertAcked(client().admin().indices().prepareCreate("test")
-                .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards,
+                        IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0,
+                        MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), Long.MAX_VALUE)
                 .addMapping("type", mapping)
                 .setTimeout("60s").get());
         ensureGreen(); // wait for green state, so its both green, and there are no more pending events

core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java

@@ -30,15 +30,21 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.junit.Rule;
 import org.junit.rules.ExpectedException;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.function.Function;
 import java.util.concurrent.ExecutionException;
 
 import static org.hamcrest.CoreMatchers.containsString;
@@ -135,4 +141,24 @@ public void testIndexIntoDefaultMapping() throws Throwable {
         assertFalse(indexService.mapperService().hasMapping(MapperService.DEFAULT_MAPPING));
     }
 
+    public void testTotalFieldsExceedsLimit() throws Throwable {
+        Function<String, String> mapping = type -> {
+            try {
+                return XContentFactory.jsonBuilder().startObject().startObject(type).startObject("properties")
+                    .startObject("field1").field("type", "string")
+                    .endObject().endObject().endObject().endObject().string();
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        };
+        createIndex("test1").mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
+        //set total number of fields to 1 to trigger an exception
+        try {
+            createIndex("test2", Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1).build())
+                .mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
+            fail("Expected IllegalArgumentException");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("Limit of total fields [1] in index [test2] has been exceeded"));
+        }
+    }
 }

docs/reference/mapping/dynamic/field-mapping.asciidoc

@@ -30,6 +30,12 @@ detected.  All other datatypes must be mapped explicitly.
 Besides the options listed below, dynamic field mapping rules can be further
 customised with <<dynamic-templates,`dynamic_templates`>>.
 
+[[total-fields-limit]]
+==== Total fields limit
+
+To avoid mapping explosion, Index has a default limit of 1000 total number of fields.
+The default setting can be updated with `index.mapping.total_fields.limit`.
+
 [[date-detection]]
 ==== Date detection