From 65819aa3cb952de26371176c8a6a13c5f8cf1828 Mon Sep 17 00:00:00 2001 From: Yanjun Huang Date: Sun, 13 Sep 2015 14:28:20 -0700 Subject: [PATCH 1/2] add max_number_of_fields to Object mapping --- .../MaxNumberOfFieldsMappingException.java | 35 ++++++++++++ .../index/mapper/object/ObjectMapper.java | 54 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 src/main/java/org/elasticsearch/index/mapper/MaxNumberOfFieldsMappingException.java diff --git a/src/main/java/org/elasticsearch/index/mapper/MaxNumberOfFieldsMappingException.java b/src/main/java/org/elasticsearch/index/mapper/MaxNumberOfFieldsMappingException.java new file mode 100644 index 0000000000000..4b25f96e17b62 --- /dev/null +++ b/src/main/java/org/elasticsearch/index/mapper/MaxNumberOfFieldsMappingException.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.elasticsearch.rest.RestStatus; + +/** + */ +public class MaxNumberOfFieldsMappingException extends MapperParsingException { + + public MaxNumberOfFieldsMappingException(String path, String fieldName) { + super("[" + fieldName + "] exceeds the max number of fields configured for [" + path + "]"); + } + + @Override + public RestStatus status() { + return RestStatus.BAD_REQUEST; + } +} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java b/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java index e3f4dc3cc4fbd..7bb0b2efbbe70 100644 --- a/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java @@ -48,6 +48,7 @@ import static com.google.common.collect.Lists.newArrayList; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue; import static org.elasticsearch.index.mapper.MapperBuilders.*; import static org.elasticsearch.index.mapper.core.TypeParsers.parsePathType; @@ -117,6 +118,8 @@ public static class Builder extends M protected Boolean includeInAll; + protected Integer maxNumberOfFields; + protected final List mappersBuilders = newArrayList(); public Builder(String name) { @@ -149,6 +152,11 @@ public T includeInAll(boolean includeInAll) { return builder; } + public T maxNumberOfFields(int maxNumberOfFields) { + this.maxNumberOfFields = maxNumberOfFields; + return builder; + } + public T add(Mapper.Builder builder) { mappersBuilders.add(builder); return this.builder; @@ -170,6 +178,7 @@ public Y build(BuilderContext context) { ObjectMapper objectMapper = createMapper(name, context.path().fullPathAsText(name), enabled, nested, dynamic, pathType, mappers, context.indexSettings()); objectMapper.includeInAllIfNotSet(includeInAll); + objectMapper.maxNumberOfFieldsIfNotSet(maxNumberOfFields); return (Y) objectMapper; } @@ -217,6 +226,9 @@ protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, O } else if (fieldName.equals("include_in_all")) { builder.includeInAll(nodeBooleanValue(fieldNode)); return true; + } else if (fieldName.equals("max_number_of_fields")) { + builder.maxNumberOfFields(nodeIntegerValue(fieldNode)); + return true; } return false; } @@ -318,6 +330,8 @@ protected Builder createBuilder(String name) { private Boolean includeInAll; + private Integer maxNumberOfFields; + private volatile CopyOnWriteHashMap mappers; private final Object mutex = new Object(); @@ -382,6 +396,31 @@ public void unsetIncludeInAll() { } } + public void maxNumberOfFields(Integer maxNumberOfFields) { + if (maxNumberOfFields == null) { + return; + } + this.maxNumberOfFields = maxNumberOfFields; + // when called from outside, apply this on all the inner mappers + for (Mapper mapper : mappers.values()) { + if (mapper instanceof ObjectMapper) { + ((ObjectMapper) mapper).maxNumberOfFields(maxNumberOfFields); + } + } + } + + public void maxNumberOfFieldsIfNotSet(Integer maxNumberOfFields) { + if (this.maxNumberOfFields == null) { + this.maxNumberOfFields = maxNumberOfFields; + } + // when called from outside, apply this on all the inner mappers + for (Mapper mapper : mappers.values()) { + if (mapper instanceof ObjectMapper) { + ((ObjectMapper) mapper).maxNumberOfFieldsIfNotSet(maxNumberOfFields); + } + } + } + public Nested nested() { return this.nested; } @@ -394,6 +433,9 @@ public ObjectMapper putMapper(Mapper mapper) { if (mapper instanceof AllFieldMapper.IncludeInAll) { ((AllFieldMapper.IncludeInAll) mapper).includeInAllIfNotSet(includeInAll); } + if (mapper instanceof ObjectMapper) { + ((ObjectMapper) mapper).maxNumberOfFieldsIfNotSet(maxNumberOfFields); + } synchronized (mutex) { mappers = mappers.copyAndPut(mapper.name(), mapper); } @@ -563,6 +605,9 @@ private void serializeObject(final ParseContext context, String currentFieldName // we sync here just so we won't add it twice. Its not the end of the world // to sync here since next operations will get it before synchronized (mutex) { + if (maxNumberOfFields != null && maxNumberOfFields > 0 && mappers.size() >= maxNumberOfFields) { + throw new MaxNumberOfFieldsMappingException(fullPath, currentFieldName); + } objectMapper = mappers.get(currentFieldName); if (objectMapper == null) { // remove the current field name from path, since template search and the object builder add it as well... @@ -615,6 +660,9 @@ private void serializeArray(ParseContext context, String lastFieldName) throws I // we sync here just so we won't add it twice. Its not the end of the world // to sync here since next operations will get it before synchronized (mutex) { + if (maxNumberOfFields != null && maxNumberOfFields > 0 && mappers.size() > maxNumberOfFields) { + throw new MaxNumberOfFieldsMappingException(fullPath, arrayFieldName); + } mapper = mappers.get(arrayFieldName); if (mapper == null) { Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, "object"); @@ -724,6 +772,9 @@ public void parseDynamicValue(final ParseContext context, String currentFieldNam // its not the end of the world, since we add it to the mappers once we create it // so next time we won't even get here for this field synchronized (mutex) { + if (maxNumberOfFields != null && maxNumberOfFields > 0 && mappers.size() >= maxNumberOfFields) { + throw new MaxNumberOfFieldsMappingException(fullPath, currentFieldName); + } Mapper mapper = mappers.get(currentFieldName); if (mapper == null) { BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); @@ -986,6 +1037,9 @@ public void toXContent(XContentBuilder builder, Params params, ToXContent custom if (includeInAll != null) { builder.field("include_in_all", includeInAll); } + if (maxNumberOfFields != null) { + builder.field("max_number_of_fields", maxNumberOfFields); + } if (custom != null) { custom.toXContent(builder, params); From 4288648af5dadd958a6fce8ce78a0df11c5312c3 Mon Sep 17 00:00:00 2001 From: Yanjun Huang Date: Fri, 2 Oct 2015 16:34:53 -0700 Subject: [PATCH 2/2] use index.subfields settings to limit the number of direct subfields of an object --- .../index/mapper/DocumentMapper.java | 16 +++- .../index/mapper/DocumentMapperParser.java | 6 +- .../index/mapper/MapperBuilders.java | 4 +- .../index/mapper/MapperService.java | 40 ++++++++- .../MaxNumberOfFieldsMappingException.java | 35 -------- .../index/mapper/ParseContext.java | 16 ++++ .../index/mapper/object/ObjectMapper.java | 87 ++++++++----------- .../settings/IndexDynamicSettingsModule.java | 3 + .../mapper/multifield/MultiFieldTests.java | 2 +- .../mapper/simple/SimpleMapperTests.java | 4 +- 10 files changed, 115 insertions(+), 98 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/index/mapper/MaxNumberOfFieldsMappingException.java diff --git a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index b72f1c88fd2d8..1d657d72bbf2d 100644 --- a/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -154,15 +154,18 @@ public static class Builder { private final Settings indexSettings; + private final MapperService mapperService; + private final RootObjectMapper rootObjectMapper; private ImmutableMap meta = ImmutableMap.of(); private final Mapper.BuilderContext builderContext; - public Builder(String index, Settings indexSettings, RootObjectMapper.Builder builder) { + public Builder(String index, Settings indexSettings, MapperService mapperService, RootObjectMapper.Builder builder) { this.index = index; this.indexSettings = indexSettings; + this.mapperService = mapperService; this.builderContext = new Mapper.BuilderContext(indexSettings, new ContentPath(1)); this.rootObjectMapper = builder.build(builderContext); IdFieldMapper idFieldMapper = new IdFieldMapper(); @@ -245,7 +248,7 @@ public Builder transform(ScriptService scriptService, String script, ScriptType public DocumentMapper build(DocumentMapperParser docMapperParser) { Preconditions.checkNotNull(rootObjectMapper, "Mapper builder must have the root object mapper set"); - return new DocumentMapper(index, indexSettings, docMapperParser, rootObjectMapper, meta, + return new DocumentMapper(index, indexSettings, mapperService, docMapperParser, rootObjectMapper, meta, indexAnalyzer, searchAnalyzer, searchQuoteAnalyzer, rootMappers, sourceTransforms); } } @@ -264,6 +267,8 @@ protected ParseContext.InternalParseContext initialValue() { private final Settings indexSettings; + private final MapperService mapperService; + private final String type; private final StringAndBytesText typeText; @@ -300,13 +305,14 @@ protected ParseContext.InternalParseContext initialValue() { private final List sourceTransforms; - public DocumentMapper(String index, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, + public DocumentMapper(String index, @Nullable Settings indexSettings, MapperService mapperService, DocumentMapperParser docMapperParser, RootObjectMapper rootObjectMapper, ImmutableMap meta, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer, Map, RootMapper> rootMappers, List sourceTransforms) { this.index = index; this.indexSettings = indexSettings; + this.mapperService = mapperService; this.type = rootObjectMapper.name(); this.typeText = new StringAndBytesText(this.type); this.docMapperParser = docMapperParser; @@ -481,6 +487,10 @@ public ImmutableMap objectMappers() { return this.objectMappers; } + public MapperService mapperService() { + return this.mapperService; + } + public ParsedDocument parse(BytesReference source) throws MapperParsingException { return parse(SourceToParse.source(source)); } diff --git a/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index bfbc57042734c..252f025971f00 100644 --- a/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -67,6 +67,7 @@ public class DocumentMapperParser extends AbstractIndexComponent { private final DocValuesFormatService docValuesFormatService; private final SimilarityLookupService similarityLookupService; private final ScriptService scriptService; + private final MapperService mapperService; private final RootObjectMapper.TypeParser rootObjectTypeParser = new RootObjectMapper.TypeParser(); @@ -76,7 +77,7 @@ public class DocumentMapperParser extends AbstractIndexComponent { private volatile ImmutableMap typeParsers; private volatile ImmutableMap rootTypeParsers; - public DocumentMapperParser(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService, + public DocumentMapperParser(Index index, @IndexSettings Settings indexSettings, MapperService mapperService, AnalysisService analysisService, PostingsFormatService postingsFormatService, DocValuesFormatService docValuesFormatService, SimilarityLookupService similarityLookupService, ScriptService scriptService) { super(index, indexSettings); @@ -85,6 +86,7 @@ public DocumentMapperParser(Index index, @IndexSettings Settings indexSettings, this.docValuesFormatService = docValuesFormatService; this.similarityLookupService = similarityLookupService; this.scriptService = scriptService; + this.mapperService = mapperService; MapBuilder typeParsersBuilder = new MapBuilder() .put(ByteFieldMapper.CONTENT_TYPE, new ByteFieldMapper.TypeParser()) .put(ShortFieldMapper.CONTENT_TYPE, new ShortFieldMapper.TypeParser()) @@ -208,7 +210,7 @@ private DocumentMapper parse(String type, Map mapping, String de Mapper.TypeParser.ParserContext parserContext = parserContext(); // parse RootObjectMapper - DocumentMapper.Builder docBuilder = doc(index.name(), indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext)); + DocumentMapper.Builder docBuilder = doc(index.name(), indexSettings, mapperService, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext)); Iterator> iterator = mapping.entrySet().iterator(); // parse DocumentMapper while(iterator.hasNext()) { diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java b/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java index 600b6d5510606..a779c8a05a9e9 100644 --- a/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java @@ -37,8 +37,8 @@ private MapperBuilders() { } - public static DocumentMapper.Builder doc(String index, Settings settings, RootObjectMapper.Builder objectBuilder) { - return new DocumentMapper.Builder(index, settings, objectBuilder); + public static DocumentMapper.Builder doc(String index, Settings settings, MapperService mapperService, RootObjectMapper.Builder objectBuilder) { + return new DocumentMapper.Builder(index, settings, mapperService, objectBuilder); } public static SourceFieldMapper.Builder source() { diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 2609d08ed2cc1..4a79f13764d77 100755 --- a/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.search.AndFilter; import org.elasticsearch.common.lucene.search.NotFilter; import org.elasticsearch.common.lucene.search.XBooleanFilter; @@ -58,6 +59,7 @@ import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.search.nested.NonNestedDocsFilter; import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.settings.IndexSettingsService; import org.elasticsearch.index.similarity.SimilarityLookupService; import org.elasticsearch.indices.InvalidTypeNameException; import org.elasticsearch.indices.TypeMissingException; @@ -119,15 +121,37 @@ public class MapperService extends AbstractIndexComponent { private volatile ImmutableMap> unmappedFieldMappers = ImmutableMap.of(); + public static final String SETTING_SUBFIELDS_LIMIT = "index.subfields.limit"; + public static final String SETTING_SUBFIELDS_DYNAMIC_AT_LIMIT = "index.subfields.dynamic_at_limit"; + + private final IndexSettingsService indexSettingsService; + + private final ApplySettings applySettings = new ApplySettings(); + + private int subfieldsLimit; + private String subfieldsDynamicAtLimit; + + class ApplySettings implements IndexSettingsService.Listener { + @Override + public void onRefreshSettings(Settings settings) { + subfieldsLimit = settings.getAsInt(SETTING_SUBFIELDS_LIMIT, 0); + subfieldsDynamicAtLimit = settings.get(SETTING_SUBFIELDS_DYNAMIC_AT_LIMIT, "strict"); + } + } + @Inject - public MapperService(Index index, @IndexSettings Settings indexSettings, Environment environment, AnalysisService analysisService, IndexFieldDataService fieldDataService, + public MapperService(Index index, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService, Environment environment, AnalysisService analysisService, IndexFieldDataService fieldDataService, PostingsFormatService postingsFormatService, DocValuesFormatService docValuesFormatService, SimilarityLookupService similarityLookupService, ScriptService scriptService) { super(index, indexSettings); + this.indexSettingsService = indexSettingsService; + subfieldsLimit = indexSettings.getAsInt(SETTING_SUBFIELDS_LIMIT, 0); + subfieldsDynamicAtLimit = indexSettings.get(SETTING_SUBFIELDS_DYNAMIC_AT_LIMIT, "strict"); + this.indexSettingsService.addListener(applySettings); this.analysisService = analysisService; this.fieldDataService = fieldDataService; this.fieldMappers = new FieldMappersLookup(); - this.documentParser = new DocumentMapperParser(index, indexSettings, analysisService, postingsFormatService, docValuesFormatService, similarityLookupService, scriptService); + this.documentParser = new DocumentMapperParser(index, indexSettings, this, analysisService, postingsFormatService, docValuesFormatService, similarityLookupService, scriptService); this.searchAnalyzer = new SmartIndexNameSearchAnalyzer(analysisService.defaultSearchAnalyzer()); this.searchQuoteAnalyzer = new SmartIndexNameSearchQuoteAnalyzer(analysisService.defaultSearchQuoteAnalyzer()); @@ -244,6 +268,18 @@ public boolean hasNested() { return this.hasNested; } + public int subfieldsLimit() { + return this.subfieldsLimit; + } + + public String subfieldsDynamicAtLimit() { + return this.subfieldsDynamicAtLimit; + } + + public ESLogger logger() { + return logger; + } + /** * returns an immutable iterator over current document mappers. * diff --git a/src/main/java/org/elasticsearch/index/mapper/MaxNumberOfFieldsMappingException.java b/src/main/java/org/elasticsearch/index/mapper/MaxNumberOfFieldsMappingException.java deleted file mode 100644 index 4b25f96e17b62..0000000000000 --- a/src/main/java/org/elasticsearch/index/mapper/MaxNumberOfFieldsMappingException.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.mapper; - -import org.elasticsearch.rest.RestStatus; - -/** - */ -public class MaxNumberOfFieldsMappingException extends MapperParsingException { - - public MaxNumberOfFieldsMappingException(String path, String fieldName) { - super("[" + fieldName + "] exceeds the max number of fields configured for [" + path + "]"); - } - - @Override - public RestStatus status() { - return RestStatus.BAD_REQUEST; - } -} \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/src/main/java/org/elasticsearch/index/mapper/ParseContext.java index 6a4b6ad439715..897c2a733c846 100644 --- a/src/main/java/org/elasticsearch/index/mapper/ParseContext.java +++ b/src/main/java/org/elasticsearch/index/mapper/ParseContext.java @@ -500,6 +500,14 @@ public Settings indexSettings() { return this.indexSettings; } + public int subfieldsLimit() { + return this.docMapper.mapperService().subfieldsLimit(); + } + + public String subfieldsDynamicAtLimit() { + return this.docMapper.mapperService().subfieldsDynamicAtLimit(); + } + public String type() { return sourceToParse.type(); } @@ -709,6 +717,14 @@ public boolean isWithinMultiFields() { @Nullable public abstract Settings indexSettings(); + public int subfieldsLimit() { + return 0; + } + + public String subfieldsDynamicAtLimit() { + return "strict"; + } + public abstract String type(); public abstract SourceToParse sourceToParse(); diff --git a/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java b/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java index 7bb0b2efbbe70..5a457b20c8a86 100644 --- a/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java @@ -118,8 +118,6 @@ public static class Builder extends M protected Boolean includeInAll; - protected Integer maxNumberOfFields; - protected final List mappersBuilders = newArrayList(); public Builder(String name) { @@ -152,11 +150,6 @@ public T includeInAll(boolean includeInAll) { return builder; } - public T maxNumberOfFields(int maxNumberOfFields) { - this.maxNumberOfFields = maxNumberOfFields; - return builder; - } - public T add(Mapper.Builder builder) { mappersBuilders.add(builder); return this.builder; @@ -178,7 +171,6 @@ public Y build(BuilderContext context) { ObjectMapper objectMapper = createMapper(name, context.path().fullPathAsText(name), enabled, nested, dynamic, pathType, mappers, context.indexSettings()); objectMapper.includeInAllIfNotSet(includeInAll); - objectMapper.maxNumberOfFieldsIfNotSet(maxNumberOfFields); return (Y) objectMapper; } @@ -226,9 +218,6 @@ protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, O } else if (fieldName.equals("include_in_all")) { builder.includeInAll(nodeBooleanValue(fieldNode)); return true; - } else if (fieldName.equals("max_number_of_fields")) { - builder.maxNumberOfFields(nodeIntegerValue(fieldNode)); - return true; } return false; } @@ -330,8 +319,6 @@ protected Builder createBuilder(String name) { private Boolean includeInAll; - private Integer maxNumberOfFields; - private volatile CopyOnWriteHashMap mappers; private final Object mutex = new Object(); @@ -396,31 +383,6 @@ public void unsetIncludeInAll() { } } - public void maxNumberOfFields(Integer maxNumberOfFields) { - if (maxNumberOfFields == null) { - return; - } - this.maxNumberOfFields = maxNumberOfFields; - // when called from outside, apply this on all the inner mappers - for (Mapper mapper : mappers.values()) { - if (mapper instanceof ObjectMapper) { - ((ObjectMapper) mapper).maxNumberOfFields(maxNumberOfFields); - } - } - } - - public void maxNumberOfFieldsIfNotSet(Integer maxNumberOfFields) { - if (this.maxNumberOfFields == null) { - this.maxNumberOfFields = maxNumberOfFields; - } - // when called from outside, apply this on all the inner mappers - for (Mapper mapper : mappers.values()) { - if (mapper instanceof ObjectMapper) { - ((ObjectMapper) mapper).maxNumberOfFieldsIfNotSet(maxNumberOfFields); - } - } - } - public Nested nested() { return this.nested; } @@ -433,9 +395,6 @@ public ObjectMapper putMapper(Mapper mapper) { if (mapper instanceof AllFieldMapper.IncludeInAll) { ((AllFieldMapper.IncludeInAll) mapper).includeInAllIfNotSet(includeInAll); } - if (mapper instanceof ObjectMapper) { - ((ObjectMapper) mapper).maxNumberOfFieldsIfNotSet(maxNumberOfFields); - } synchronized (mutex) { mappers = mappers.copyAndPut(mapper.name(), mapper); } @@ -605,8 +564,18 @@ private void serializeObject(final ParseContext context, String currentFieldName // we sync here just so we won't add it twice. Its not the end of the world // to sync here since next operations will get it before synchronized (mutex) { - if (maxNumberOfFields != null && maxNumberOfFields > 0 && mappers.size() >= maxNumberOfFields) { - throw new MaxNumberOfFieldsMappingException(fullPath, currentFieldName); + int maxNumberOfFields = context.subfieldsLimit(); + if (maxNumberOfFields > 0 && mappers.size() >= maxNumberOfFields) { + String dynamicAtLimit = context.subfieldsDynamicAtLimit(); + if (dynamicAtLimit.equalsIgnoreCase("strict")) { + throw new StrictDynamicMappingException(fullPath, currentFieldName); + } else { + context.docMapper().mapperService().logger().warn("[" + currentFieldName + + "] exceeds the max number of fields configured for [" + fullPath + "]"); + if (!nodeBooleanValue(dynamicAtLimit)) { + return; + } + } } objectMapper = mappers.get(currentFieldName); if (objectMapper == null) { @@ -649,7 +618,6 @@ private void serializeArray(ParseContext context, String lastFieldName) throws I serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } else { - Dynamic dynamic = this.dynamic; if (dynamic == null) { dynamic = context.root().dynamic(); @@ -660,8 +628,18 @@ private void serializeArray(ParseContext context, String lastFieldName) throws I // we sync here just so we won't add it twice. Its not the end of the world // to sync here since next operations will get it before synchronized (mutex) { - if (maxNumberOfFields != null && maxNumberOfFields > 0 && mappers.size() > maxNumberOfFields) { - throw new MaxNumberOfFieldsMappingException(fullPath, arrayFieldName); + int maxNumberOfFields = context.subfieldsLimit(); + if (maxNumberOfFields > 0 && mappers.size() >= maxNumberOfFields) { + String dynamicAtLimit = context.subfieldsDynamicAtLimit(); + if (dynamicAtLimit.equalsIgnoreCase("strict")) { + throw new StrictDynamicMappingException(fullPath, arrayFieldName); + } else { + context.docMapper().mapperService().logger().warn("[" + arrayFieldName + + "] exceeds the max number of fields configured for [" + fullPath + "]"); + if (!nodeBooleanValue(dynamicAtLimit)) { + return; + } + } } mapper = mappers.get(arrayFieldName); if (mapper == null) { @@ -772,8 +750,18 @@ public void parseDynamicValue(final ParseContext context, String currentFieldNam // its not the end of the world, since we add it to the mappers once we create it // so next time we won't even get here for this field synchronized (mutex) { - if (maxNumberOfFields != null && maxNumberOfFields > 0 && mappers.size() >= maxNumberOfFields) { - throw new MaxNumberOfFieldsMappingException(fullPath, currentFieldName); + int maxNumberOfFields = context.subfieldsLimit(); + if (maxNumberOfFields > 0 && mappers.size() >= maxNumberOfFields) { + String dynamicAtLimit = context.subfieldsDynamicAtLimit(); + if (dynamicAtLimit.equalsIgnoreCase("strict")) { + throw new StrictDynamicMappingException(fullPath, currentFieldName); + } else { + context.docMapper().mapperService().logger().warn("[" + currentFieldName + + "] exceeds the max number of fields configured for [" + fullPath + "]"); + if (!nodeBooleanValue(dynamicAtLimit)) { + return; + } + } } Mapper mapper = mappers.get(currentFieldName); if (mapper == null) { @@ -1037,9 +1025,6 @@ public void toXContent(XContentBuilder builder, Params params, ToXContent custom if (includeInAll != null) { builder.field("include_in_all", includeInAll); } - if (maxNumberOfFields != null) { - builder.field("max_number_of_fields", maxNumberOfFields); - } if (custom != null) { custom.toXContent(builder, params); diff --git a/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java b/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java index e105debe087e0..e4496c144bae3 100644 --- a/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java +++ b/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java @@ -31,6 +31,7 @@ import org.elasticsearch.gateway.local.LocalGatewayAllocator; import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.indexing.slowlog.ShardSlowLogIndexingService; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.merge.policy.LogByteSizeMergePolicyProvider; import org.elasticsearch.index.merge.policy.LogDocMergePolicyProvider; import org.elasticsearch.index.merge.policy.TieredMergePolicyProvider; @@ -124,6 +125,8 @@ public IndexDynamicSettingsModule() { indexDynamicSettings.addDynamicSetting(IndicesWarmer.INDEX_WARMER_ENABLED); indexDynamicSettings.addDynamicSetting(IndicesQueryCache.INDEX_CACHE_QUERY_ENABLED, Validator.BOOLEAN); indexDynamicSettings.addDynamicSetting(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, Validator.TIME); + indexDynamicSettings.addDynamicSetting(MapperService.SETTING_SUBFIELDS_LIMIT, Validator.NON_NEGATIVE_INTEGER); + indexDynamicSettings.addDynamicSetting(MapperService.SETTING_SUBFIELDS_DYNAMIC_AT_LIMIT); } public void addDynamicSettings(String... settings) { diff --git a/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java b/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java index 17274ac454450..34938cbbb459e 100644 --- a/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java @@ -141,7 +141,7 @@ public void testBuildThenParse() throws Exception { Settings settings = indexService.settingsService().getSettings(); DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser(); - DocumentMapper builderDocMapper = doc("test", settings, rootObject("person").add( + DocumentMapper builderDocMapper = doc("test", settings, indexService.mapperService(), rootObject("person").add( stringField("name").store(true) .addMultiField(stringField("indexed").index(true).tokenized(true)) .addMultiField(stringField("not_indexed").index(false).store(true)) diff --git a/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java b/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java index 756e044116cdf..11e6384fb18bd 100644 --- a/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java @@ -45,7 +45,7 @@ public void testSimpleMapper() throws Exception { IndexService indexService = createIndex("test"); Settings settings = indexService.settingsService().getSettings(); DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser(); - DocumentMapper docMapper = doc("test", settings, + DocumentMapper docMapper = doc("test", settings, indexService.mapperService(), rootObject("person") .add(object("name").add(stringField("first").store(true).index(false))) ).build(mapperParser); @@ -124,7 +124,7 @@ public void testNoDocumentSent() throws Exception { IndexService indexService = createIndex("test"); Settings settings = indexService.settingsService().getSettings(); DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser(); - DocumentMapper docMapper = doc("test", settings, + DocumentMapper docMapper = doc("test", settings, indexService.mapperService(), rootObject("person") .add(object("name").add(stringField("first").store(true).index(false))) ).build(mapperParser);