From 34977e4bd8617e42ecf9edd3b6765be6eef2d97c Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 20 Oct 2020 14:00:59 +0100 Subject: [PATCH 01/19] Centralize index analyzer management --- .../index/mapper/RankFeatureFieldMapper.java | 8 +- .../index/mapper/RankFeaturesFieldMapper.java | 8 +- .../index/mapper/ScaledFloatFieldMapper.java | 6 ++ .../mapper/SearchAsYouTypeFieldMapper.java | 73 +++++++++-------- .../index/mapper/TokenCountFieldMapper.java | 6 ++ .../SearchAsYouTypeFieldMapperTests.java | 28 ++----- .../mapper/SearchAsYouTypeFieldTypeTests.java | 1 - .../join/mapper/ParentIdFieldMapper.java | 8 +- .../join/mapper/ParentJoinFieldMapper.java | 8 +- .../percolator/PercolatorFieldMapper.java | 6 ++ .../ICUCollationKeywordFieldMapper.java | 8 +- .../AnnotatedTextFieldMapper.java | 46 ++++------- .../mapper/murmur3/Murmur3FieldMapper.java | 6 ++ .../analyze/TransportAnalyzeAction.java | 19 +++-- .../index/analysis/FieldNameAnalyzer.java | 16 ++++ .../mapper/AbstractGeometryFieldMapper.java | 7 ++ .../index/mapper/BinaryFieldMapper.java | 7 ++ .../index/mapper/BooleanFieldMapper.java | 8 ++ .../index/mapper/CompletionFieldMapper.java | 82 +++++++------------ .../index/mapper/DateFieldMapper.java | 7 ++ .../index/mapper/DocumentMapper.java | 2 +- .../index/mapper/FieldMapper.java | 44 +--------- .../index/mapper/IdFieldMapper.java | 8 +- .../index/mapper/IpFieldMapper.java | 7 ++ .../index/mapper/KeywordFieldMapper.java | 16 +++- .../mapper/LegacyGeoShapeFieldMapper.java | 8 ++ .../index/mapper/MappedFieldType.java | 8 -- .../index/mapper/MapperService.java | 29 +++---- .../index/mapper/MappingLookup.java | 19 ++--- .../index/mapper/MetadataFieldMapper.java | 7 ++ .../index/mapper/NumberFieldMapper.java | 9 +- .../index/mapper/RangeFieldMapper.java | 9 +- .../index/mapper/RoutingFieldMapper.java | 8 +- .../index/mapper/TextFieldMapper.java | 64 ++++++++------- .../index/mapper/TextParams.java | 28 +++++-- .../index/termvectors/TermVectorsService.java | 10 +-- .../SignificantTextAggregatorFactory.java | 8 +- .../highlight/FastVectorHighlighter.java | 11 +-- .../highlight/FragmentBuilderHelper.java | 19 ++--- .../highlight/SimpleFragmentsBuilder.java | 12 ++- .../SourceScoreOrderFragmentsBuilder.java | 11 ++- .../SourceSimpleFragmentsBuilder.java | 3 +- .../MetadataRolloverServiceTests.java | 3 +- .../mapper/CompletionFieldMapperTests.java | 38 +++++---- .../mapper/DocumentFieldMapperTests.java | 32 ++++---- .../index/mapper/ExternalMapper.java | 7 ++ .../index/mapper/FakeStringFieldMapper.java | 8 +- .../FieldAliasMapperValidationTests.java | 19 ++--- .../index/mapper/ParametrizedMapperTests.java | 7 ++ .../index/mapper/TextFieldTypeTests.java | 2 - .../SignificantTermsAggregatorTests.java | 11 +-- .../terms/SignificantTextAggregatorTests.java | 17 ++-- .../index/mapper/MockFieldMapper.java | 7 ++ .../aggregations/AggregatorTestCase.java | 1 + .../mapper/HistogramFieldMapper.java | 8 ++ .../mapper/ConstantKeywordFieldMapper.java | 7 ++ .../mapper/FlatObjectFieldMapper.java | 9 +- .../unsignedlong/UnsignedLongFieldMapper.java | 6 ++ .../VersionStringFieldMapper.java | 8 +- .../mapper/RuntimeFieldMapper.java | 7 ++ .../mapper/DenseVectorFieldMapper.java | 7 ++ .../mapper/SparseVectorFieldMapper.java | 6 ++ .../wildcard/mapper/WildcardFieldMapper.java | 17 ++-- 63 files changed, 548 insertions(+), 377 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index a64e5ae190114..22c860d7a87ef 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.FeatureField; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -35,6 +36,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -91,7 +93,6 @@ public static final class RankFeatureFieldType extends MappedFieldType { public RankFeatureFieldType(String name, Map meta, boolean positiveScoreImpact) { super(name, true, false, false, TextSearchInfo.NONE, meta); this.positiveScoreImpact = positiveScoreImpact; - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -188,6 +189,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index 3831bf46b3344..f3170a3ddf7dd 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.FeatureField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Query; @@ -32,6 +33,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -69,7 +71,6 @@ public static final class RankFeaturesFieldType extends MappedFieldType { public RankFeaturesFieldType(String name, Map meta) { super(name, false, false, false, TextSearchInfo.NONE, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -161,4 +162,9 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index d370f22ce71f8..9f18af0d5c68e 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; @@ -53,6 +54,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; /** A {@link FieldMapper} for scaled floats. Values are internally multiplied @@ -312,6 +314,10 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault, coerceByDefault).init(this); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index e65445b37e20c..121301d0fdb94 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -47,7 +47,6 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.common.collect.Iterators; -import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; @@ -61,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType.hasGaps; @@ -95,18 +95,14 @@ public static class Defaults { public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer())); - private static SearchAsYouTypeFieldMapper toType(FieldMapper in) { - return (SearchAsYouTypeFieldMapper) in; - } - - private static SearchAsYouTypeFieldType ft(FieldMapper in) { - return toType(in).fieldType(); + private static Builder builder(FieldMapper in) { + return ((SearchAsYouTypeFieldMapper)in).builder; } public static class Builder extends ParametrizedFieldMapper.Builder { - private final Parameter index = Parameter.indexParam(m -> toType(m).index, true); - private final Parameter store = Parameter.storeParam(m -> toType(m).store, false); + private final Parameter index = Parameter.indexParam(m -> builder(m).index.get(), true); + private final Parameter store = Parameter.storeParam(m -> builder(m).store.get(), false); // This is only here because for some reason the initial impl of this always serialized // `doc_values=false`, even though it cannot be set; and so we need to continue @@ -120,7 +116,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { .alwaysSerialize(); private final Parameter maxShingleSize = Parameter.intParam("max_shingle_size", false, - m -> toType(m).maxShingleSize, Defaults.MAX_SHINGLE_SIZE) + m -> builder(m).maxShingleSize.get(), Defaults.MAX_SHINGLE_SIZE) .setValidator(v -> { if (v < MAX_SHINGLE_SIZE_LOWER_BOUND || v > MAX_SHINGLE_SIZE_UPPER_BOUND) { throw new MapperParsingException("[max_shingle_size] must be at least [" + MAX_SHINGLE_SIZE_LOWER_BOUND @@ -130,17 +126,17 @@ public static class Builder extends ParametrizedFieldMapper.Builder { .alwaysSerialize(); final TextParams.Analyzers analyzers; - final Parameter similarity = TextParams.similarity(m -> ft(m).getTextSearchInfo().getSimilarity()); + final Parameter similarity = TextParams.similarity(m -> builder(m).similarity.get()); - final Parameter indexOptions = TextParams.indexOptions(m -> toType(m).indexOptions); - final Parameter norms = TextParams.norms(true, m -> ft(m).getTextSearchInfo().hasNorms()); - final Parameter termVectors = TextParams.termVectors(m -> toType(m).termVectors); + final Parameter indexOptions = TextParams.indexOptions(m -> builder(m).indexOptions.get()); + final Parameter norms = TextParams.norms(true, m -> builder(m).norms.get()); + final Parameter termVectors = TextParams.termVectors(m -> builder(m).termVectors.get()); private final Parameter> meta = Parameter.metaParam(); public Builder(String name, Supplier defaultAnalyzer) { super(name); - this.analyzers = new TextParams.Analyzers(defaultAnalyzer); + this.analyzers = new TextParams.Analyzers(defaultAnalyzer, m -> builder(m).analyzers); } @Override @@ -164,7 +160,6 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, similarity.getValue(), analyzers.getSearchAnalyzer(), analyzers.getSearchQuoteAnalyzer(), meta.getValue()); - ft.setIndexAnalyzer(analyzers.getIndexAnalyzer()); // set up the prefix field FieldType prefixft = new FieldType(fieldType); @@ -182,8 +177,7 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { TextSearchInfo prefixSearchInfo = new TextSearchInfo(prefixft, similarity.getValue(), prefixSearchWrapper, searchAnalyzer); final PrefixFieldType prefixFieldType = new PrefixFieldType(fullName, prefixSearchInfo, Defaults.MIN_GRAM, Defaults.MAX_GRAM); - prefixFieldType.setIndexAnalyzer(new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, prefixIndexWrapper)); - final PrefixFieldMapper prefixFieldMapper = new PrefixFieldMapper(prefixft, prefixFieldType); + final PrefixFieldMapper prefixFieldMapper = new PrefixFieldMapper(prefixft, prefixFieldType, prefixIndexWrapper); // set up the shingle fields final ShingleFieldMapper[] shingleFieldMappers = new ShingleFieldMapper[maxShingleSize.getValue() - 1]; @@ -203,10 +197,9 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { TextSearchInfo textSearchInfo = new TextSearchInfo(shingleft, similarity.getValue(), shingleSearchWrapper, shingleSearchQuoteWrapper); final ShingleFieldType shingleFieldType = new ShingleFieldType(fieldName, shingleSize, textSearchInfo); - shingleFieldType.setIndexAnalyzer(new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, shingleIndexWrapper)); shingleFieldType.setPrefixFieldType(prefixFieldType); shingleFieldTypes[i] = shingleFieldType; - shingleFieldMappers[i] = new ShingleFieldMapper(shingleft, shingleFieldType); + shingleFieldMappers[i] = new ShingleFieldMapper(shingleft, shingleFieldType, shingleIndexWrapper); } ft.setPrefixField(prefixFieldType); ft.setShingleFields(shingleFieldTypes); @@ -399,8 +392,11 @@ public Query existsQuery(QueryShardContext context) { static final class PrefixFieldMapper extends FieldMapper { - PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType) { + final Analyzer analyzer; + + PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType, Analyzer analyzer) { super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); + this.analyzer = analyzer; } @Override @@ -422,6 +418,11 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), analyzer); + } + @Override protected String contentType() { return "prefix"; @@ -435,8 +436,11 @@ public String toString() { static final class ShingleFieldMapper extends FieldMapper { - ShingleFieldMapper(FieldType fieldType, ShingleFieldType mappedFieldtype) { + final Analyzer analyzer; + + ShingleFieldMapper(FieldType fieldType, ShingleFieldType mappedFieldtype, Analyzer analyzer) { super(mappedFieldtype.name(), fieldType, mappedFieldtype, MultiFields.empty(), CopyTo.empty()); + this.analyzer = analyzer; } FieldType getLuceneFieldType() { @@ -448,6 +452,11 @@ public ShingleFieldType fieldType() { return (ShingleFieldType) super.fieldType(); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), analyzer); + } + @Override protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); @@ -540,14 +549,11 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew } } - private final boolean index; - private final boolean store; - private final String indexOptions; - private final String termVectors; - private final int maxShingleSize; private final PrefixFieldMapper prefixField; private final ShingleFieldMapper[] shingleFields; + private final Analyzer analyzer; + private final Builder builder; public SearchAsYouTypeFieldMapper(String simpleName, SearchAsYouTypeFieldType mappedFieldType, @@ -559,10 +565,8 @@ public SearchAsYouTypeFieldMapper(String simpleName, this.prefixField = prefixField; this.shingleFields = shingleFields; this.maxShingleSize = builder.maxShingleSize.getValue(); - this.index = builder.index.getValue(); - this.store = builder.store.getValue(); - this.indexOptions = builder.indexOptions.getValue(); - this.termVectors = builder.termVectors.getValue(); + this.analyzer = builder.analyzers.getIndexAnalyzer(); + this.builder = builder; } @Override @@ -587,9 +591,14 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), analyzer); + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), () -> fieldType().indexAnalyzer()).init(this); + return new Builder(simpleName(), builder.analyzers.indexAnalyzer::getDefaultValue).init(this); } public static String getShingleFieldName(String parentField, int shingleSize) { diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index ae21b3711ee19..e3744e6bcf82d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue; @@ -195,6 +196,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index ec2a1b3e43f87..2ebe860e7ae26 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -602,9 +602,7 @@ private static void assertSearchAsYouTypeFieldType(SearchAsYouTypeFieldType fiel PrefixFieldType prefixFieldType) { assertThat(fieldType.shingleFields.length, equalTo(maxShingleSize - 1)); - for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) { - assertThat(analyzer.name(), equalTo(analyzerName)); - } + assertThat(fieldType.getTextSearchInfo().getSearchAnalyzer().name(), equalTo(analyzerName)); int shingleSize = 2; for (ShingleFieldType shingleField : fieldType.shingleFields) { assertShingleFieldType(shingleField, shingleSize++, analyzerName, prefixFieldType); @@ -620,13 +618,12 @@ private static void assertShingleFieldType(ShingleFieldType fieldType, assertThat(fieldType.shingleSize, equalTo(shingleSize)); - for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) { - assertThat(analyzer.name(), equalTo(analyzerName)); - if (shingleSize > 1) { - final SearchAsYouTypeAnalyzer wrappedAnalyzer = (SearchAsYouTypeAnalyzer) analyzer.analyzer(); - assertThat(wrappedAnalyzer.shingleSize(), equalTo(shingleSize)); - assertThat(wrappedAnalyzer.indexPrefixes(), equalTo(false)); - } + NamedAnalyzer analyzer = fieldType.getTextSearchInfo().getSearchAnalyzer(); + assertThat(analyzer.name(), equalTo(analyzerName)); + if (shingleSize > 1) { + final SearchAsYouTypeAnalyzer wrappedAnalyzer = (SearchAsYouTypeAnalyzer) analyzer.analyzer(); + assertThat(wrappedAnalyzer.shingleSize(), equalTo(shingleSize)); + assertThat(wrappedAnalyzer.indexPrefixes(), equalTo(false)); } assertThat(fieldType.prefixFieldType, equalTo(prefixFieldType)); @@ -634,17 +631,10 @@ private static void assertShingleFieldType(ShingleFieldType fieldType, } private static void assertPrefixFieldType(PrefixFieldType fieldType, int shingleSize, String analyzerName) { - for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) { - assertThat(analyzer.name(), equalTo(analyzerName)); - } - - final SearchAsYouTypeAnalyzer wrappedIndexAnalyzer = (SearchAsYouTypeAnalyzer) fieldType.indexAnalyzer().analyzer(); + assertThat(fieldType.getTextSearchInfo().getSearchAnalyzer().name(), equalTo(analyzerName)); final SearchAsYouTypeAnalyzer wrappedSearchAnalyzer = (SearchAsYouTypeAnalyzer) fieldType.getTextSearchInfo().getSearchAnalyzer().analyzer(); - for (SearchAsYouTypeAnalyzer analyzer : asList(wrappedIndexAnalyzer, wrappedSearchAnalyzer)) { - assertThat(analyzer.shingleSize(), equalTo(shingleSize)); - } - assertThat(wrappedIndexAnalyzer.indexPrefixes(), equalTo(true)); + assertThat(wrappedSearchAnalyzer.shingleSize(), equalTo(shingleSize)); assertThat(wrappedSearchAnalyzer.indexPrefixes(), equalTo(false)); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java index dab80326afa18..31721fb77fe4a 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java @@ -114,7 +114,6 @@ public void testPrefixQuery() { public void testFetchSourceValue() throws IOException { SearchAsYouTypeFieldType fieldType = createFieldType(); - fieldType.setIndexAnalyzer(Lucene.STANDARD_ANALYZER); assertEquals(List.of("value"), fetchSourceValue(fieldType, "value")); assertEquals(List.of("42"), fetchSourceValue(fieldType, 42L)); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index d31f07923fd4b..397ff297c0529 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.join.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedDocValuesField; @@ -37,6 +38,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; +import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -60,7 +62,6 @@ static class Defaults { public static final class ParentIdFieldType extends StringFieldType { public ParentIdFieldType(String name, boolean eagerGlobalOrdinals) { super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); setEagerGlobalOrdinals(eagerGlobalOrdinals); } @@ -116,6 +117,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + @Override public Builder getMergeBuilder() { return null; // always constructed by ParentJoinFieldMapper, not through type parsers diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index e14f6203e1deb..d920e0c3df3d3 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.join.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedDocValuesField; @@ -53,6 +54,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -145,7 +147,6 @@ public static final class JoinFieldType extends StringFieldType { private JoinFieldType(String name, Joiner joiner, Map meta) { super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.joiner = joiner; } @@ -207,6 +208,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + @Override protected ParentJoinFieldMapper clone() { return (ParentJoinFieldMapper) super.clone(); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index ba22a1ddd9e02..10a32387dc16e 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.percolator; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.BinaryRange; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; @@ -88,6 +89,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; @@ -472,6 +474,10 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + } + boolean isMapUnmappedFieldAsText() { return mapUnmappedFieldsAsText; } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index c8ddf6da9bc6a..76cf78edd43b4 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -23,6 +23,7 @@ import com.ibm.icu.text.RawCollationKey; import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; @@ -48,6 +49,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; public class ICUCollationKeywordFieldMapper extends ParametrizedFieldMapper { @@ -62,7 +64,6 @@ public static final class CollationFieldType extends StringFieldType { public CollationFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues, Collator collator, String nullValue, int ignoreAbove, Map meta) { super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.collator = collator; this.nullValue = nullValue; this.ignoreAbove = ignoreAbove; @@ -435,6 +436,11 @@ public CollationFieldType fieldType() { return (CollationFieldType) super.fieldType(); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 442cf76c5bb06..058abc56b83e0 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -35,7 +35,6 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -55,6 +54,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -79,6 +79,10 @@ private static Builder builder(FieldMapper in) { return ((AnnotatedTextFieldMapper)in).builder; } + private static NamedAnalyzer wrapAnalyzer(NamedAnalyzer in) { + return new NamedAnalyzer(in.name(), AnalyzerScope.INDEX, new AnnotationAnalyzerWrapper(in.analyzer())); + } + public static class Builder extends ParametrizedFieldMapper.Builder { private final Parameter store = Parameter.storeParam(m -> builder(m).store.getValue(), false); @@ -91,55 +95,32 @@ public static class Builder extends ParametrizedFieldMapper.Builder { final Parameter norms = TextParams.norms(true, m -> builder(m).norms.getValue()); final Parameter termVectors = TextParams.termVectors(m -> builder(m).termVectors.getValue()); - final Parameter positionIncrementGap = Parameter.intParam("position_increment_gap", false, - m -> builder(m).positionIncrementGap.getValue(), POSITION_INCREMENT_GAP_USE_ANALYZER) - .setValidator(v -> { - if (v != POSITION_INCREMENT_GAP_USE_ANALYZER && v < 0) { - throw new MapperParsingException("[positions_increment_gap] must be positive, got [" + v + "]"); - } - }); - private final Parameter> meta = Parameter.metaParam(); public Builder(String name, Supplier defaultAnalyzer) { super(name); - this.analyzers = new TextParams.Analyzers(defaultAnalyzer); + this.analyzers = new TextParams.Analyzers(defaultAnalyzer, m -> builder(m).analyzers); } @Override protected List> getParameters() { return Arrays.asList(store, indexOptions, norms, termVectors, similarity, - analyzers.indexAnalyzer, analyzers.searchAnalyzer, analyzers.searchQuoteAnalyzer, positionIncrementGap, + analyzers.indexAnalyzer, analyzers.searchAnalyzer, analyzers.searchQuoteAnalyzer, + analyzers.positionIncrementGap, meta); } - private NamedAnalyzer wrapAnalyzer(NamedAnalyzer in, int positionIncrementGap) { - return new NamedAnalyzer(in.name(), AnalyzerScope.INDEX, - new AnnotationAnalyzerWrapper(in.analyzer()), positionIncrementGap); - } - private AnnotatedTextFieldType buildFieldType(FieldType fieldType, BuilderContext context) { - int posGap; - if (positionIncrementGap.get() == POSITION_INCREMENT_GAP_USE_ANALYZER) { - posGap = TextFieldMapper.Defaults.POSITION_INCREMENT_GAP; - } else { - if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { - throw new IllegalArgumentException("Cannot set position_increment_gap on field [" + name() - + "] without positions enabled"); - } - posGap = positionIncrementGap.get(); - } TextSearchInfo tsi = new TextSearchInfo( fieldType, similarity.get(), - wrapAnalyzer(analyzers.getSearchAnalyzer(), posGap), - wrapAnalyzer(analyzers.getSearchQuoteAnalyzer(), posGap)); + wrapAnalyzer(analyzers.getSearchAnalyzer()), + wrapAnalyzer(analyzers.getSearchQuoteAnalyzer())); AnnotatedTextFieldType ft = new AnnotatedTextFieldType( buildFullName(context), store.getValue(), tsi, meta.getValue()); - ft.setIndexAnalyzer(wrapAnalyzer(analyzers.getIndexAnalyzer(), posGap)); return ft; } @@ -521,6 +502,7 @@ public String typeName() { } private final FieldType fieldType; + private final Analyzer analyzer; private final Builder builder; protected AnnotatedTextFieldMapper(String simpleName, FieldType fieldType, AnnotatedTextFieldType mappedFieldType, @@ -529,6 +511,12 @@ protected AnnotatedTextFieldMapper(String simpleName, FieldType fieldType, Annot assert fieldType.tokenized(); this.fieldType = fieldType; this.builder = builder; + this.analyzer = wrapAnalyzer(builder.analyzers.getIndexAnalyzer()); + } + + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), analyzer); } @Override diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 291f488647ba4..08b85162dadd2 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper.murmur3; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; @@ -43,6 +44,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; public class Murmur3FieldMapper extends ParametrizedFieldMapper { @@ -132,6 +134,10 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + } + @Override protected void parseCreateField(ParseContext context) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index ce772d9c4f789..4340d04fc90b4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -180,7 +180,7 @@ private static Analyzer getAnalyzer(AnalyzeAction.Request request, AnalysisRegis MappedFieldType fieldType = indexService.mapperService().fieldType(request.field()); if (fieldType != null) { if (fieldType instanceof StringFieldType) { - return fieldType.indexAnalyzer(); + return indexService.mapperService().indexAnalyzer(); } else { throw new IllegalArgumentException("Can't process field [" + request.field() + "], Analysis requests are only supported on tokenized fields"); @@ -220,10 +220,8 @@ private static List simpleAnalyze(AnalyzeAction.Requ List tokens = new ArrayList<>(); int lastPosition = -1; int lastOffset = 0; - // Note that we always pass "" as the field to the various Analyzer methods, because - // the analyzers we use here are all field-specific and so ignore this parameter for (String text : request.text()) { - try (TokenStream stream = analyzer.tokenStream("", text)) { + try (TokenStream stream = analyzer.tokenStream(request.field(), text)) { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); @@ -244,8 +242,8 @@ private static List simpleAnalyze(AnalyzeAction.Requ lastOffset += offset.endOffset(); lastPosition += posIncr.getPositionIncrement(); - lastPosition += analyzer.getPositionIncrementGap(""); - lastOffset += analyzer.getOffsetGap(""); + lastPosition += analyzer.getPositionIncrementGap(request.field()); + lastOffset += analyzer.getOffsetGap(request.field()); } catch (IOException e) { throw new ElasticsearchException("failed to analyze", e); } @@ -345,13 +343,16 @@ private static AnalyzeAction.DetailAnalyzeResponse detailAnalyze(AnalyzeAction.R if (analyzer instanceof NamedAnalyzer) { name = ((NamedAnalyzer) analyzer).name(); } else { - name = analyzer.getClass().getName(); + name = request.field(); } TokenListCreator tokenListCreator = new TokenListCreator(maxTokenCount); for (String text : request.text()) { - tokenListCreator.analyze(analyzer.tokenStream("", text), includeAttributes, analyzer.getPositionIncrementGap(""), - analyzer.getOffsetGap("")); + tokenListCreator.analyze( + analyzer.tokenStream(request.field(), text), + includeAttributes, + analyzer.getPositionIncrementGap(request.field()), + analyzer.getOffsetGap(request.field())); } detailResponse = new AnalyzeAction.DetailAnalyzeResponse(new AnalyzeAction.AnalyzeTokenList(name, tokenListCreator.getArrayTokens())); diff --git a/server/src/main/java/org/elasticsearch/index/analysis/FieldNameAnalyzer.java b/server/src/main/java/org/elasticsearch/index/analysis/FieldNameAnalyzer.java index dbb355ea51ca5..955d213dbaa55 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/FieldNameAnalyzer.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/FieldNameAnalyzer.java @@ -48,4 +48,20 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { // Fields need to be explicitly added throw new IllegalArgumentException("Field [" + fieldName + "] has no associated analyzer"); } + + public boolean containsBrokenAnalysis(String field) { + Analyzer analyzer = getWrappedAnalyzer(field); + if (analyzer instanceof NamedAnalyzer) { + analyzer = ((NamedAnalyzer) analyzer).analyzer(); + } + if (analyzer instanceof AnalyzerComponentsProvider) { + final TokenFilterFactory[] tokenFilters = ((AnalyzerComponentsProvider) analyzer).getComponents().getTokenFilters(); + for (TokenFilterFactory tokenFilterFactory : tokenFilters) { + if (tokenFilterFactory.breaksFastVectorHighlighter()) { + return true; + } + } + } + return false; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index e0d0c46708df6..0f05c1fbfd856 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -44,6 +45,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Function; /** @@ -289,6 +291,11 @@ public AbstractGeometryFieldType fieldType() { return (AbstractGeometryFieldType) mappedFieldType; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + + } + @Override protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index eb3bf22929a7f..ab07c0f555732 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.hppc.ObjectArrayList; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.StoredField; import org.apache.lucene.search.Query; import org.apache.lucene.store.ByteArrayDataOutput; @@ -42,6 +43,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; public class BinaryFieldMapper extends ParametrizedFieldMapper { @@ -193,6 +195,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + + } + public static class CustomBinaryDocValuesField extends CustomDocValuesField { private final ObjectArrayList bytesList; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 1c34517426b34..de254741309c0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -29,6 +30,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -43,6 +45,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -266,6 +269,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 30d1f44b01b9e..e3b0987133fca 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -54,6 +55,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.BiConsumer; /** * Mapper for completion field. The field values are indexed as a weighted FST for @@ -86,7 +88,7 @@ public class CompletionFieldMapper extends ParametrizedFieldMapper { @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), defaultAnalyzer, indexVersionCreated).init(this); + return new Builder(simpleName(), builder.defaultAnalyzer, builder.indexVersionCreated).init(this); } public static class Defaults { @@ -111,8 +113,8 @@ public static class Fields { public static final String CONTENT_FIELD_NAME_CONTEXTS = "contexts"; } - private static CompletionFieldMapper toType(FieldMapper in) { - return (CompletionFieldMapper) in; + private static Builder builder(FieldMapper in) { + return ((CompletionFieldMapper)in).builder; } /** @@ -123,13 +125,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Parameter analyzer; private final Parameter searchAnalyzer; private final Parameter preserveSeparators = Parameter.boolParam("preserve_separators", false, - m -> toType(m).preserveSeparators, Defaults.DEFAULT_PRESERVE_SEPARATORS) + m -> builder(m).preserveSeparators.get(), Defaults.DEFAULT_PRESERVE_SEPARATORS) .alwaysSerialize(); private final Parameter preservePosInc = Parameter.boolParam("preserve_position_increments", false, - m -> toType(m).preservePosInc, Defaults.DEFAULT_POSITION_INCREMENTS) + m -> builder(m).preservePosInc.get(), Defaults.DEFAULT_POSITION_INCREMENTS) .alwaysSerialize(); private final Parameter contexts = new Parameter<>("contexts", false, () -> null, - (n, c, o) -> ContextMappings.load(o, c.indexVersionCreated()), m -> toType(m).contexts) + (n, c, o) -> ContextMappings.load(o, c.indexVersionCreated()), m -> builder(m).contexts.get()) .setSerializer((b, n, c) -> { if (c == null) { return; @@ -139,7 +141,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { b.endArray(); }, Objects::toString); private final Parameter maxInputLength = Parameter.intParam("max_input_length", true, - m -> toType(m).maxInputLength, Defaults.DEFAULT_MAX_INPUT_LENGTH) + m -> builder(m).maxInputLength.get(), Defaults.DEFAULT_MAX_INPUT_LENGTH) .addDeprecatedName("max_input_len") .setValidator(Builder::validateInputLength) .alwaysSerialize(); @@ -157,10 +159,10 @@ public Builder(String name, NamedAnalyzer defaultAnalyzer, Version indexVersionC super(name); this.defaultAnalyzer = defaultAnalyzer; this.indexVersionCreated = indexVersionCreated; - this.analyzer = Parameter.analyzerParam("analyzer", false, m -> toType(m).analyzer, () -> defaultAnalyzer) + this.analyzer = Parameter.analyzerParam("analyzer", false, m -> builder(m).analyzer.get(), () -> defaultAnalyzer) .alwaysSerialize(); this.searchAnalyzer - = Parameter.analyzerParam("search_analyzer", true, m -> toType(m).searchAnalyzer, analyzer::getValue); + = Parameter.analyzerParam("search_analyzer", true, m -> builder(m).searchAnalyzer.get(), analyzer::getValue); } private static void validateInputLength(int maxInputLength) { @@ -183,11 +185,8 @@ public CompletionFieldMapper build(BuilderContext context) { CompletionFieldType ft = new CompletionFieldType(buildFullName(context), completionAnalyzer, meta.getValue()); ft.setContextMappings(contexts.getValue()); - ft.setPreservePositionIncrements(preservePosInc.getValue()); - ft.setPreserveSep(preserveSeparators.getValue()); - ft.setIndexAnalyzer(analyzer.getValue()); - return new CompletionFieldMapper(name, ft, defaultAnalyzer, - multiFieldsBuilder.build(this, context), copyTo.build(), indexVersionCreated, this); + return new CompletionFieldMapper(name, ft, + multiFieldsBuilder.build(this, context), copyTo.build(), this); } private void checkCompletionContextsLimit(BuilderContext context) { @@ -217,37 +216,16 @@ public static final class CompletionFieldType extends TermBasedFieldType { private static PostingsFormat postingsFormat; - private boolean preserveSep = Defaults.DEFAULT_PRESERVE_SEPARATORS; - private boolean preservePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS; private ContextMappings contextMappings = null; public CompletionFieldType(String name, NamedAnalyzer searchAnalyzer, Map meta) { super(name, true, false, false, new TextSearchInfo(Defaults.FIELD_TYPE, null, searchAnalyzer, searchAnalyzer), meta); } - public void setPreserveSep(boolean preserveSep) { - this.preserveSep = preserveSep; - } - - public void setPreservePositionIncrements(boolean preservePositionIncrements) { - this.preservePositionIncrements = preservePositionIncrements; - } - public void setContextMappings(ContextMappings contextMappings) { this.contextMappings = contextMappings; } - @Override - public NamedAnalyzer indexAnalyzer() { - final NamedAnalyzer indexAnalyzer = super.indexAnalyzer(); - if (indexAnalyzer != null && !(indexAnalyzer.analyzer() instanceof CompletionAnalyzer)) { - return new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, - new CompletionAnalyzer(indexAnalyzer, preserveSep, preservePositionIncrements)); - - } - return indexAnalyzer; - } - /** * @return true if there are one or more context mappings defined * for this field type @@ -326,25 +304,16 @@ protected List parseSourceValue(Object value) { } private final int maxInputLength; - private final boolean preserveSeparators; - private final boolean preservePosInc; - private final NamedAnalyzer defaultAnalyzer; + private final Builder builder; private final NamedAnalyzer analyzer; - private final NamedAnalyzer searchAnalyzer; - private final ContextMappings contexts; - private final Version indexVersionCreated; - public CompletionFieldMapper(String simpleName, MappedFieldType mappedFieldType, NamedAnalyzer defaultAnalyzer, - MultiFields multiFields, CopyTo copyTo, Version indexVersionCreated, Builder builder) { + public CompletionFieldMapper(String simpleName, MappedFieldType mappedFieldType, + MultiFields multiFields, CopyTo copyTo, Builder builder) { super(simpleName, mappedFieldType, multiFields, copyTo); - this.defaultAnalyzer = defaultAnalyzer; + this.builder = builder; this.maxInputLength = builder.maxInputLength.getValue(); - this.preserveSeparators = builder.preserveSeparators.getValue(); - this.preservePosInc = builder.preservePosInc.getValue(); - this.analyzer = builder.analyzer.getValue(); - this.searchAnalyzer = builder.searchAnalyzer.getValue(); - this.contexts = builder.contexts.getValue(); - this.indexVersionCreated = indexVersionCreated; + this.analyzer = new NamedAnalyzer(builder.analyzer.get().name(), AnalyzerScope.INDEX, + new CompletionAnalyzer(builder.analyzer.get(), builder.preserveSeparators.get(), builder.preservePosInc.get())); } @Override @@ -357,8 +326,13 @@ public boolean parsesArrayValue() { return true; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), analyzer); + } + int getMaxInputLength() { - return maxInputLength; + return builder.maxInputLength.get(); } /** @@ -402,7 +376,7 @@ public void parse(ParseContext context) throws IOException { } // truncate input if (input.length() > maxInputLength) { - int len = Math.min(maxInputLength, input.length()); + int len = maxInputLength; if (Character.isHighSurrogate(input.charAt(len - 1))) { assert input.length() >= len + 1 && Character.isLowSurrogate(input.charAt(len)); len += 1; @@ -508,7 +482,7 @@ private void parse(ParseContext parseContext, Token token, ContextMappings contextMappings = fieldType().getContextMappings(); XContentParser.Token currentToken = parser.currentToken(); if (currentToken == XContentParser.Token.START_OBJECT) { - ContextMapping contextMapping = null; + ContextMapping contextMapping = null; String fieldName = null; while ((currentToken = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (currentToken == XContentParser.Token.FIELD_NAME) { @@ -568,7 +542,7 @@ protected String contentType() { public void doValidate(MappingLookup mappers) { if (fieldType().hasContextMappings()) { for (ContextMapping contextMapping : fieldType().getContextMappings()) { - contextMapping.validateReferences(indexVersionCreated, s -> mappers.fieldTypes().get(s)); + contextMapping.validateReferences(builder.indexVersionCreated, s -> mappers.fieldTypes().get(s)); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 10f918149cb5d..d0888d0605f6b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; @@ -58,6 +59,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; @@ -625,6 +627,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + + } + public boolean getIgnoreMalformed() { return ignoreMalformed; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 9d4a57150486f..977192b717972 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -136,7 +136,7 @@ private DocumentMapper(IndexSettings indexSettings, this.documentParser = documentParser; this.indexSettings = indexSettings; this.indexAnalyzers = indexAnalyzers; - this.fieldMappers = MappingLookup.fromMapping(this.mapping, indexAnalyzers.getDefaultIndexAnalyzer()); + this.fieldMappers = MappingLookup.fromMapping(this.mapping); try { mappingSource = new CompressedXContent(this, XContentType.JSON, ToXContent.EMPTY_PARAMS); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index cb14192aeee80..e1d0de516ace1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -45,6 +46,7 @@ import java.util.Map; import java.util.Objects; import java.util.TreeMap; +import java.util.function.BiConsumer; import java.util.stream.StreamSupport; public abstract class FieldMapper extends Mapper implements Cloneable { @@ -400,17 +402,6 @@ private void mergeSharedOptions(FieldMapper mergeWith, List conflicts) { if (fieldType.storeTermVectorPayloads() != other.storeTermVectorPayloads()) { conflicts.add("mapper [" + name() + "] has different [store_term_vector_payloads] values"); } - - // null and "default"-named index analyzers both mean the default is used - if (mappedFieldType.indexAnalyzer() == null || "default".equals(mappedFieldType.indexAnalyzer().name())) { - if (otherm.indexAnalyzer() != null && "default".equals(otherm.indexAnalyzer().name()) == false) { - conflicts.add("mapper [" + name() + "] has different [analyzer]"); - } - } else if (otherm.indexAnalyzer() == null || "default".equals(otherm.indexAnalyzer().name())) { - conflicts.add("mapper [" + name() + "] has different [analyzer]"); - } else if (mappedFieldType.indexAnalyzer().name().equals(otherm.indexAnalyzer().name()) == false) { - conflicts.add("mapper [" + name() + "] has different [analyzer]"); - } } /** @@ -460,35 +451,6 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, } } - protected final void doXContentAnalyzers(XContentBuilder builder, boolean includeDefaults) throws IOException { - if (fieldType.tokenized() == false) { - return; - } - if (fieldType().indexAnalyzer() == null) { - if (includeDefaults) { - builder.field("analyzer", "default"); - } - } else { - boolean hasDefaultIndexAnalyzer = fieldType().indexAnalyzer().name().equals("default"); - final String searchAnalyzerName = fieldType().getTextSearchInfo().getSearchAnalyzer() == null - ? "default" : fieldType().getTextSearchInfo().getSearchAnalyzer().name(); - final String searchQuoteAnalyzerName = fieldType().getTextSearchInfo().getSearchQuoteAnalyzer() == null - ? searchAnalyzerName : fieldType().getTextSearchInfo().getSearchQuoteAnalyzer().name(); - boolean hasDifferentSearchAnalyzer = searchAnalyzerName.equals(fieldType().indexAnalyzer().name()) == false; - boolean hasDifferentSearchQuoteAnalyzer - = Objects.equals(searchAnalyzerName, searchQuoteAnalyzerName) == false; - if (includeDefaults || hasDefaultIndexAnalyzer == false || hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) { - builder.field("analyzer", fieldType().indexAnalyzer().name()); - if (includeDefaults || hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) { - builder.field("search_analyzer", searchAnalyzerName); - if (includeDefaults || hasDifferentSearchQuoteAnalyzer) { - builder.field("search_quote_analyzer", searchQuoteAnalyzerName); - } - } - } - } - } - protected static String indexOptionToString(IndexOptions indexOption) { switch (indexOption) { case DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS: @@ -506,6 +468,8 @@ protected static String indexOptionToString(IndexOptions indexOption) { protected abstract String contentType(); + public abstract void registerIndexAnalyzer(BiConsumer analyzerRegistry); + public static class MultiFields implements Iterable { public static MultiFields empty() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index e6c5dbd8364d1..7a6264076eee1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -54,6 +55,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -104,7 +106,6 @@ static final class IdFieldType extends TermBasedFieldType { IdFieldType(Supplier fieldDataEnabled) { super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); this.fieldDataEnabled = fieldDataEnabled; - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -208,6 +209,11 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu } } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + private static LeafFieldData wrap(LeafFieldData in) { return new LeafFieldData() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index a2e788ba93a1d..bcb5072bf3c68 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; @@ -47,6 +48,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Supplier; @@ -443,4 +445,9 @@ protected void parseCreateField(ParseContext context) throws IOException { public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).init(this); } + + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 81440ccf9703d..dce19de9e58cd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Field; @@ -42,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -178,6 +180,7 @@ public static final class KeywordFieldType extends StringFieldType { private final int ignoreAbove; private final String nullValue; + private final NamedAnalyzer normalizer; public KeywordFieldType(String name, FieldType fieldType, NamedAnalyzer normalizer, NamedAnalyzer searchAnalyzer, Builder builder) { @@ -188,14 +191,14 @@ public KeywordFieldType(String name, FieldType fieldType, new TextSearchInfo(fieldType, builder.similarity.getValue(), searchAnalyzer, searchAnalyzer), builder.meta.getValue()); setEagerGlobalOrdinals(builder.eagerGlobalOrdinals.getValue()); - setIndexAnalyzer(normalizer); + this.normalizer = normalizer; this.ignoreAbove = builder.ignoreAbove.getValue(); this.nullValue = builder.nullValue.getValue(); } public KeywordFieldType(String name, boolean isSearchable, boolean hasDocValues, Map meta) { super(name, isSearchable, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; } @@ -209,12 +212,14 @@ public KeywordFieldType(String name, FieldType fieldType) { false, false, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap()); + this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); + this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; } @@ -225,7 +230,7 @@ public String typeName() { } NamedAnalyzer normalizer() { - return indexAnalyzer(); + return normalizer; } @Override @@ -345,6 +350,11 @@ public KeywordFieldType fieldType() { return (KeywordFieldType) super.fieldType(); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), fieldType().normalizer()); + } + @Override protected void parseCreateField(ParseContext context) throws IOException { String value; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index 707b6f0f82714..edcc0acc8a25f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; @@ -41,6 +42,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; import org.elasticsearch.common.geo.parsers.ShapeParser; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; @@ -57,6 +59,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; /** * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s. @@ -443,6 +446,11 @@ public GeoShapeFieldType fieldType() { return (GeoShapeFieldType) super.fieldType(); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + @Override protected void addStoredFields(ParseContext context, Shape geometry) { // noop: we do not store geo_shapes; and will not store legacy geo_shape types diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 218a2a5e9bf94..9e3f6d384ebdf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -123,14 +123,6 @@ public boolean hasDocValues() { return docValues; } - public NamedAnalyzer indexAnalyzer() { - return indexAnalyzer; - } - - public void setIndexAnalyzer(NamedAnalyzer analyzer) { - this.indexAnalyzer = analyzer; - } - /** * Returns the collapse type of the field * CollapseType.NONE means the field can'be used for collapsing. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 18b78a03d27d6..5ffd90591d17f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -60,7 +60,6 @@ import java.util.Map; import java.util.Set; import java.util.function.BooleanSupplier; -import java.util.function.Function; import java.util.function.Supplier; public class MapperService extends AbstractIndexComponent implements Closeable { @@ -86,7 +85,7 @@ public enum MergeReason { * if a shard was moved to a different node or for administrative * purposes. */ - MAPPING_RECOVERY; + MAPPING_RECOVERY } public static final String SINGLE_MAPPING_NAME = "_doc"; @@ -122,7 +121,7 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, this.documentParser = new DocumentParser(xContentRegistry); this.documentMapperParser = new DocumentMapperParser(indexSettings, this, similarityService, mapperRegistry, queryShardContextSupplier, scriptService); - this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), MappedFieldType::indexAnalyzer); + this.indexAnalyzer = new MapperAnalyzerWrapper(); this.mapperRegistry = mapperRegistry; this.idFieldDataEnabled = idFieldDataEnabled; } @@ -421,6 +420,10 @@ public Analyzer indexAnalyzer() { return this.indexAnalyzer; } + public boolean containsBrokenAnalysis(String field) { + return this.indexAnalyzer.containsBrokenAnalysis(field); + } + /** * Returns true if fielddata is enabled for the {@link IdFieldMapper} field, false otherwise. */ @@ -460,25 +463,17 @@ public boolean isMetadataField(String field) { /** An analyzer wrapper that can lookup fields within the index mappings */ final class MapperAnalyzerWrapper extends DelegatingAnalyzerWrapper { - private final Analyzer defaultAnalyzer; - private final Function extractAnalyzer; - - MapperAnalyzerWrapper(Analyzer defaultAnalyzer, Function extractAnalyzer) { + MapperAnalyzerWrapper() { super(Analyzer.PER_FIELD_REUSE_STRATEGY); - this.defaultAnalyzer = defaultAnalyzer; - this.extractAnalyzer = extractAnalyzer; } @Override protected Analyzer getWrappedAnalyzer(String fieldName) { - MappedFieldType fieldType = fieldType(fieldName); - if (fieldType != null) { - Analyzer analyzer = extractAnalyzer.apply(fieldType); - if (analyzer != null) { - return analyzer; - } - } - return defaultAnalyzer; + return mapper.mappers().indexAnalyzer(); + } + + boolean containsBrokenAnalysis(String field) { + return mapper.mappers().indexAnalyzer().containsBrokenAnalysis(field); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 64cd4fc4cb9e2..abf16523aff8e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -42,14 +42,7 @@ public final class MappingLookup implements Iterable { private final int metadataFieldCount; private final FieldNameAnalyzer indexAnalyzer; - private static void put(Map analyzers, String key, Analyzer value, Analyzer defaultValue) { - if (value == null) { - value = defaultValue; - } - analyzers.put(key, value); - } - - public static MappingLookup fromMapping(Mapping mapping, Analyzer defaultIndex) { + public static MappingLookup fromMapping(Mapping mapping) { List newObjectMappers = new ArrayList<>(); List newFieldMappers = new ArrayList<>(); List newFieldAliasMappers = new ArrayList<>(); @@ -59,7 +52,7 @@ public static MappingLookup fromMapping(Mapping mapping, Analyzer defaultIndex) } } collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers); - return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length, defaultIndex); + return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length); } private static void collect(Mapper mapper, Collection objectMappers, @@ -86,8 +79,7 @@ private static void collect(Mapper mapper, Collection objectMapper public MappingLookup(Collection mappers, Collection objectMappers, Collection aliasMappers, - int metadataFieldCount, - Analyzer defaultIndex) { + int metadataFieldCount) { Map fieldMappers = new HashMap<>(); Map indexAnalyzers = new HashMap<>(); Map objects = new HashMap<>(); @@ -110,8 +102,7 @@ public MappingLookup(Collection mappers, if (fieldMappers.put(mapper.name(), mapper) != null) { throw new MapperParsingException("Field [" + mapper.name() + "] is defined more than once"); } - MappedFieldType fieldType = mapper.fieldType(); - put(indexAnalyzers, fieldType.name(), fieldType.indexAnalyzer(), defaultIndex); + mapper.registerIndexAnalyzer(indexAnalyzers::put); } this.metadataFieldCount = metadataFieldCount; @@ -149,7 +140,7 @@ public FieldTypeLookup fieldTypes() { * A smart analyzer used for indexing that takes into account specific analyzers configured * per {@link FieldMapper}. */ - public Analyzer indexAnalyzer() { + public FieldNameAnalyzer indexAnalyzer() { return this.indexAnalyzer; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index bbba108e2e803..2298ac1da6c22 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -19,12 +19,14 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; import java.io.IOException; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Function; @@ -158,6 +160,11 @@ protected void parseCreateField(ParseContext context) throws IOException { + " a document. Use the index API request parameters."); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + // no analyzers to register + } + /** * Called before {@link FieldMapper#parse(ParseContext)} on the {@link RootObjectMapper}. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index be99e75b6a526..a768e7e3cc083 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.exc.InputCoercionException; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; @@ -59,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -898,7 +900,6 @@ public NumberFieldType(String name, NumberType type, boolean isSearchable, boole this.type = Objects.requireNonNull(type); this.coerce = coerce; this.nullValue = nullValue; - this.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); // allows number fields in significant text aggs - do we need this? } NumberFieldType(String name, Builder builder) { @@ -1042,6 +1043,12 @@ public NumberFieldType fieldType() { return (NumberFieldType) super.fieldType(); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + // allows number fields in significant text aggs - do we need this? + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + @Override protected String contentType() { return fieldType().type.typeName(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index f8ba2973b2517..454251b28f8b2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; @@ -53,6 +54,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; @@ -158,7 +160,6 @@ public RangeFieldType(String name, RangeType type, boolean indexed, boolean stor this.rangeType = Objects.requireNonNull(type); dateTimeFormatter = null; dateMathParser = null; - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.coerce = coerce; } @@ -172,7 +173,6 @@ public RangeFieldType(String name, boolean indexed, boolean stored, boolean has this.rangeType = RangeType.DATE; this.dateTimeFormatter = Objects.requireNonNull(formatter); this.dateMathParser = dateTimeFormatter.toDateMathParser(); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.coerce = coerce; } @@ -458,6 +458,11 @@ public Object getTo() { } } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + static class BinaryRangesDocValuesField extends CustomDocValuesField { private final Set ranges; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 54bc2f2230e6a..8d2ed3808124b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -27,6 +28,7 @@ import java.util.Collections; import java.util.List; +import java.util.function.BiConsumer; public class RoutingFieldMapper extends MetadataFieldMapper { @@ -86,7 +88,6 @@ static final class RoutingFieldType extends StringFieldType { private RoutingFieldType() { super(NAME, true, true, false, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -125,4 +126,9 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index a3a4795df260a..a80e5e0d45bed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -64,7 +64,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; @@ -83,6 +82,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.BiConsumer; import java.util.function.IntPredicate; import java.util.function.Supplier; @@ -90,7 +90,6 @@ public class TextFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "text"; - private static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1; private static final String FAST_PHRASE_SUFFIX = "._index_phrase"; public static class Defaults { @@ -251,9 +250,6 @@ public static class Builder extends ParametrizedFieldMapper.Builder { final Parameter norms = TextParams.norms(true, m -> builder(m).norms.getValue()); final Parameter termVectors = TextParams.termVectors(m -> builder(m).termVectors.getValue()); - final Parameter positionIncrementGap = Parameter.intParam("position_increment_gap", false, - m -> builder(m).positionIncrementGap.getValue(), POSITION_INCREMENT_GAP_USE_ANALYZER); - final Parameter fieldData = Parameter.boolParam("fielddata", true, m -> builder(m).fieldData.getValue(), false); final Parameter freqFilter = new Parameter<>("fielddata_frequency_filter", true, @@ -277,7 +273,7 @@ public Builder(String name, Supplier defaultAnalyzer) { public Builder(String name, Version indexCreatedVersion, Supplier defaultAnalyzer) { super(name); this.indexCreatedVersion = indexCreatedVersion; - this.analyzers = new TextParams.Analyzers(defaultAnalyzer); + this.analyzers = new TextParams.Analyzers(defaultAnalyzer, m -> builder(m).analyzers); } public Builder index(boolean index) { @@ -309,28 +305,23 @@ public Builder addMultiField(Mapper.Builder builder) { protected List> getParameters() { return Arrays.asList(index, store, indexOptions, norms, termVectors, analyzers.indexAnalyzer, analyzers.searchAnalyzer, analyzers.searchQuoteAnalyzer, similarity, - positionIncrementGap, + analyzers.positionIncrementGap, fieldData, freqFilter, eagerGlobalOrdinals, indexPhrases, indexPrefixes, meta); } private TextFieldType buildFieldType(FieldType fieldType, BuilderContext context) { - NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer(); NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer(); NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer(); - if (positionIncrementGap.get() != POSITION_INCREMENT_GAP_USE_ANALYZER) { + if (analyzers.positionIncrementGap.get() != TextParams.POSITION_INCREMENT_GAP_USE_ANALYZER) { if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { throw new IllegalArgumentException("Cannot set position_increment_gap on field [" + name + "] without positions enabled"); } - indexAnalyzer = new NamedAnalyzer(indexAnalyzer, positionIncrementGap.get()); - searchAnalyzer = new NamedAnalyzer(searchAnalyzer, positionIncrementGap.get()); - searchQuoteAnalyzer = new NamedAnalyzer(searchQuoteAnalyzer, positionIncrementGap.get()); } TextSearchInfo tsi = new TextSearchInfo(fieldType, similarity.getValue(), searchAnalyzer, searchQuoteAnalyzer); TextFieldType ft = new TextFieldType(buildFullName(context), index.getValue(), store.getValue(), tsi, meta.getValue()); - ft.setIndexAnalyzer(indexAnalyzer); ft.setEagerGlobalOrdinals(eagerGlobalOrdinals.getValue()); if (fieldData.getValue()) { ft.setFielddata(true, freqFilter.getValue()); @@ -367,12 +358,12 @@ private PrefixFieldMapper buildPrefixMapper(BuilderContext context, FieldType fi pft.setStoreTermVectorOffsets(true); } PrefixFieldType prefixFieldType = new PrefixFieldType(tft, fullName + "._index_prefix", indexPrefixes.get()); - prefixFieldType.setAnalyzer(analyzers.getIndexAnalyzer()); tft.setPrefixFieldType(prefixFieldType); - return new PrefixFieldMapper(pft, prefixFieldType); + return new PrefixFieldMapper(pft, prefixFieldType, + new PrefixWrappedAnalyzer(analyzers.getIndexAnalyzer(), prefixFieldType.minChars, prefixFieldType.maxChars)); } - private PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType parent) { + private PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType parent, NamedAnalyzer parentAnalyzer) { if (indexPhrases.get() == false) { return null; } @@ -384,7 +375,7 @@ private PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType p } FieldType phraseFieldType = new FieldType(fieldType); parent.setIndexPhrases(); - return new PhraseFieldMapper(phraseFieldType, new PhraseFieldType(parent)); + return new PhraseFieldMapper(phraseFieldType, new PhraseFieldType(parent), parentAnalyzer); } @Override @@ -392,7 +383,7 @@ public TextFieldMapper build(BuilderContext context) { FieldType fieldType = TextParams.buildFieldType(index, store, indexOptions, norms, termVectors); TextFieldType tft = buildFieldType(fieldType, context); return new TextFieldMapper(name, fieldType, tft, - buildPrefixMapper(context, fieldType, tft), buildPhraseMapper(fieldType, tft), + buildPrefixMapper(context, fieldType, tft), buildPhraseMapper(fieldType, tft, analyzers.getIndexAnalyzer()), multiFieldsBuilder.build(this, context), copyTo.build(), this); } } @@ -451,14 +442,9 @@ static final class PhraseFieldType extends StringFieldType { PhraseFieldType(TextFieldType parent) { super(parent.name() + FAST_PHRASE_SUFFIX, true, false, false, parent.getTextSearchInfo(), Collections.emptyMap()); - setAnalyzer(parent.indexAnalyzer().name(), parent.indexAnalyzer().analyzer()); this.parent = parent; } - void setAnalyzer(String name, Analyzer delegate) { - setIndexAnalyzer(new NamedAnalyzer(name, AnalyzerScope.INDEX, new PhraseWrappedAnalyzer(delegate))); - } - @Override public String typeName() { return "phrase"; @@ -501,11 +487,6 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc return SourceValueFetcher.toString(name(), mapperService, format); } - void setAnalyzer(NamedAnalyzer delegate) { - setIndexAnalyzer(new NamedAnalyzer(delegate.name(), AnalyzerScope.INDEX, - new PrefixWrappedAnalyzer(delegate.analyzer(), minChars, maxChars))); - } - boolean accept(int length) { return length >= minChars - 1 && length <= maxChars; } @@ -569,8 +550,11 @@ public Query existsQuery(QueryShardContext context) { private static final class PhraseFieldMapper extends FieldMapper { - PhraseFieldMapper(FieldType fieldType, PhraseFieldType mappedFieldType) { + final Analyzer analyzer; + + PhraseFieldMapper(FieldType fieldType, PhraseFieldType mappedFieldType, NamedAnalyzer analyzer) { super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); + this.analyzer = new PhraseWrappedAnalyzer(analyzer.analyzer()); } @Override @@ -587,12 +571,20 @@ protected void mergeOptions(FieldMapper other, List conflicts) { protected String contentType() { return "phrase"; } + + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), analyzer); + } } private static final class PrefixFieldMapper extends FieldMapper { - protected PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType) { + final Analyzer analyzer; + + protected PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType, Analyzer analyzer) { super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); + this.analyzer = analyzer; } void addField(ParseContext context, String value) { @@ -609,6 +601,11 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), analyzer); + } + @Override protected String contentType() { return "prefix"; @@ -912,6 +909,11 @@ public TextFieldType fieldType() { return (TextFieldType) super.fieldType(); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), builder.analyzers.getIndexAnalyzer()); + } + public static Query createPhraseQuery(TokenStream stream, String field, int slop, boolean enablePositionIncrements) throws IOException { MultiPhraseQuery.Builder mpqb = new MultiPhraseQuery.Builder(); mpqb.setSlop(slop); @@ -1045,7 +1047,7 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, this.builder.analyzers.searchQuoteAnalyzer.toXContent(builder, includeDefaults); this.builder.similarity.toXContent(builder, includeDefaults); this.builder.eagerGlobalOrdinals.toXContent(builder, includeDefaults); - this.builder.positionIncrementGap.toXContent(builder, includeDefaults); + this.builder.analyzers.positionIncrementGap.toXContent(builder, includeDefaults); this.builder.fieldData.toXContent(builder, includeDefaults); this.builder.freqFilter.toXContent(builder, includeDefaults); this.builder.indexPrefixes.toXContent(builder, includeDefaults); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java b/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java index a22fd0c23e785..f110e3c8402ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java @@ -35,16 +35,20 @@ */ public final class TextParams { + public static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1; + private TextParams() {} public static final class Analyzers { public final Parameter indexAnalyzer; public final Parameter searchAnalyzer; public final Parameter searchQuoteAnalyzer; + public final Parameter positionIncrementGap; - public Analyzers(Supplier defaultAnalyzer) { + public Analyzers(Supplier defaultAnalyzer, + Function analyzerInitFunction) { this.indexAnalyzer = Parameter.analyzerParam("analyzer", false, - m -> m.fieldType().indexAnalyzer(), defaultAnalyzer) + m -> analyzerInitFunction.apply(m).indexAnalyzer.get(), defaultAnalyzer) .setSerializerCheck((id, ic, a) -> id || ic || Objects.equals(a, getSearchAnalyzer()) == false || Objects.equals(a, getSearchQuoteAnalyzer()) == false) .setValidator(a -> a.checkAllowedInMode(AnalysisMode.INDEX_TIME)); @@ -57,18 +61,32 @@ public Analyzers(Supplier defaultAnalyzer) { = Parameter.analyzerParam("search_quote_analyzer", true, m -> m.fieldType().getTextSearchInfo().getSearchQuoteAnalyzer(), searchAnalyzer::getValue) .setValidator(a -> a.checkAllowedInMode(AnalysisMode.SEARCH_TIME)); + this.positionIncrementGap = Parameter.intParam("position_increment_gap", false, + m -> analyzerInitFunction.apply(m).positionIncrementGap.get(), POSITION_INCREMENT_GAP_USE_ANALYZER) + .setValidator(v -> { + if (v != POSITION_INCREMENT_GAP_USE_ANALYZER && v < 0) { + throw new MapperParsingException("[position_increment_gap] must be positive, got [" + v + "]"); + } + }); } public NamedAnalyzer getIndexAnalyzer() { - return indexAnalyzer.getValue(); + return wrapAnalyzer(indexAnalyzer.getValue()); } public NamedAnalyzer getSearchAnalyzer() { - return searchAnalyzer.getValue(); + return wrapAnalyzer(searchAnalyzer.getValue()); } public NamedAnalyzer getSearchQuoteAnalyzer() { - return searchQuoteAnalyzer.getValue(); + return wrapAnalyzer(searchQuoteAnalyzer.getValue()); + } + + private NamedAnalyzer wrapAnalyzer(NamedAnalyzer a) { + if (positionIncrementGap.get() == POSITION_INCREMENT_GAP_USE_ANALYZER) { + return a; + } + return new NamedAnalyzer(a, positionIncrementGap.get()); } } diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index 02ffa1233d39e..99ee0d0c4c199 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -222,17 +222,11 @@ private static Fields addGeneratedTermVectors(IndexShard indexShard, Engine.GetR private static Analyzer getAnalyzerAtField(IndexShard indexShard, String field, @Nullable Map perFieldAnalyzer) { MapperService mapperService = indexShard.mapperService(); - Analyzer analyzer; if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) { - analyzer = mapperService.getIndexAnalyzers().get(perFieldAnalyzer.get(field)); + return mapperService.getIndexAnalyzers().get(perFieldAnalyzer.get(field)); } else { - MappedFieldType fieldType = mapperService.fieldType(field); - analyzer = fieldType.indexAnalyzer(); + return mapperService.indexAnalyzer(); } - if (analyzer == null) { - analyzer = mapperService.getIndexAnalyzers().getDefaultIndexAnalyzer(); - } - return analyzer; } private static Set getFieldsToGenerate(Map perAnalyzerField, Fields fieldsObject) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java index d2086eff04a53..b6402e41565f5 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.util.BytesRefHash; import org.elasticsearch.common.util.ObjectArray; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.Aggregator; @@ -84,7 +85,7 @@ public SignificantTextAggregatorFactory(String name, // Note that if the field is unmapped (its field type is null), we don't fail, // and just use the given field name as a placeholder. this.fieldType = context.getFieldType(fieldName); - if (fieldType != null && fieldType.indexAnalyzer() == null) { + if (fieldType != null && fieldType.getTextSearchInfo() == TextSearchInfo.NONE) { throw new IllegalArgumentException("Field [" + fieldType.name() + "] has no analyzer, but SignificantText " + "requires an analyzed field"); } @@ -124,6 +125,7 @@ protected Aggregator createInternal(SearchContext searchContext, Aggregator pare context.lookup().source(), context.bigArrays(), fieldType, + searchContext.mapperService().indexAnalyzer(), sourceFieldNames, filterDuplicateText ); @@ -150,6 +152,7 @@ private static class SignificantTextCollectorSource implements MapStringTermsAgg private final SourceLookup sourceLookup; private final BigArrays bigArrays; private final MappedFieldType fieldType; + private final Analyzer analyzer; private final String[] sourceFieldNames; private ObjectArray dupSequenceSpotters; @@ -157,12 +160,14 @@ private static class SignificantTextCollectorSource implements MapStringTermsAgg SourceLookup sourceLookup, BigArrays bigArrays, MappedFieldType fieldType, + Analyzer analyzer, String[] sourceFieldNames, boolean filterDuplicateText ) { this.sourceLookup = sourceLookup; this.bigArrays = bigArrays; this.fieldType = fieldType; + this.analyzer = analyzer; this.sourceFieldNames = sourceFieldNames; dupSequenceSpotters = filterDuplicateText ? bigArrays.newObjectArray(1) : null; } @@ -216,7 +221,6 @@ private void collectFromSource(int doc, long owningBucketOrd, DuplicateByteSeque return obj.toString(); }) .iterator(); - Analyzer analyzer = fieldType.indexAnalyzer(); while (itr.hasNext()) { TokenStream ts = analyzer.tokenStream(fieldType.name(), itr.next()); processTokenStream(doc, owningBucketOrd, ts, inDocTerms, spotter); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java index bc529c83bdfa6..39acc491a47d6 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java @@ -72,6 +72,7 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc FetchSubPhase.HitContext hitContext = fieldContext.hitContext; MappedFieldType fieldType = fieldContext.fieldType; boolean forceSource = fieldContext.forceSource; + boolean fixBrokenAnalysis = fieldContext.context.mapperService().containsBrokenAnalysis(fieldContext.fieldName); if (canHighlight(fieldType) == false) { throw new IllegalArgumentException("the field [" + fieldContext.fieldName + @@ -95,10 +96,10 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc fragListBuilder = new SingleFragListBuilder(); if (!forceSource && fieldType.isStored()) { - fragmentsBuilder = new SimpleFragmentsBuilder(fieldType, field.fieldOptions().preTags(), + fragmentsBuilder = new SimpleFragmentsBuilder(fieldType, fixBrokenAnalysis, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } else { - fragmentsBuilder = new SourceSimpleFragmentsBuilder(fieldType, hitContext.sourceLookup(), + fragmentsBuilder = new SourceSimpleFragmentsBuilder(fieldType, fixBrokenAnalysis, hitContext.sourceLookup(), field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } } else { @@ -109,16 +110,16 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc fragmentsBuilder = new ScoreOrderFragmentsBuilder(field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } else { - fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(fieldType, hitContext.sourceLookup(), + fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(fieldType, fixBrokenAnalysis, hitContext.sourceLookup(), field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } } else { if (!forceSource && fieldType.isStored()) { - fragmentsBuilder = new SimpleFragmentsBuilder(fieldType, field.fieldOptions().preTags(), + fragmentsBuilder = new SimpleFragmentsBuilder(fieldType, fixBrokenAnalysis, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } else { fragmentsBuilder = - new SourceSimpleFragmentsBuilder(fieldType, hitContext.sourceLookup(), + new SourceSimpleFragmentsBuilder(fieldType, fixBrokenAnalysis, hitContext.sourceLookup(), field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java index d896d3a9d922d..d485e25d3ce45 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FragmentBuilderHelper.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.fetch.subphase.highlight; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Field; import org.apache.lucene.search.vectorhighlight.FastVectorHighlighter; import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo; import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo; @@ -29,9 +28,7 @@ import org.elasticsearch.index.analysis.AnalyzerComponentsProvider; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.analysis.TokenFilterFactory; -import org.elasticsearch.index.mapper.MappedFieldType; -import java.util.Comparator; import java.util.List; /** @@ -47,23 +44,19 @@ private FragmentBuilderHelper() { * Fixes problems with broken analysis chains if positions and offsets are messed up that can lead to * {@link StringIndexOutOfBoundsException} in the {@link FastVectorHighlighter} */ - public static WeightedFragInfo fixWeightedFragInfo(MappedFieldType fieldType, Field[] values, WeightedFragInfo fragInfo) { + public static WeightedFragInfo fixWeightedFragInfo(WeightedFragInfo fragInfo) { assert fragInfo != null : "FragInfo must not be null"; - assert fieldType.name().equals(values[0].name()) : "Expected MappedFieldType for field " + values[0].name(); - if (!fragInfo.getSubInfos().isEmpty() && containsBrokenAnalysis(fieldType.indexAnalyzer())) { + if (!fragInfo.getSubInfos().isEmpty()) { /* This is a special case where broken analysis like WDF is used for term-vector creation at index-time * which can potentially mess up the offsets. To prevent a SAIIOBException we need to resort * the fragments based on their offsets rather than using solely the positions as it is done in * the FastVectorHighlighter. Yet, this is really a lucene problem and should be fixed in lucene rather * than in this hack... aka. "we are are working on in!" */ final List subInfos = fragInfo.getSubInfos(); - CollectionUtil.introSort(subInfos, new Comparator() { - @Override - public int compare(SubInfo o1, SubInfo o2) { - int startOffset = o1.getTermsOffsets().get(0).getStartOffset(); - int startOffset2 = o2.getTermsOffsets().get(0).getStartOffset(); - return FragmentBuilderHelper.compare(startOffset, startOffset2); - } + CollectionUtil.introSort(subInfos, (o1, o2) -> { + int startOffset = o1.getTermsOffsets().get(0).getStartOffset(); + int startOffset2 = o2.getTermsOffsets().get(0).getStartOffset(); + return compare(startOffset, startOffset2); }); return new WeightedFragInfo(Math.min(fragInfo.getSubInfos().get(0).getTermsOffsets().get(0).getStartOffset(), fragInfo.getStartOffset()), fragInfo.getEndOffset(), subInfos, fragInfo.getTotalBoost()); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SimpleFragmentsBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SimpleFragmentsBuilder.java index 851d916cde7e2..70afd6c365ab9 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SimpleFragmentsBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SimpleFragmentsBuilder.java @@ -29,20 +29,26 @@ * that corrects offsets for broken analysis chains. */ public class SimpleFragmentsBuilder extends org.apache.lucene.search.vectorhighlight.SimpleFragmentsBuilder { + protected final MappedFieldType fieldType; + private final boolean fixBrokenAnalysis; public SimpleFragmentsBuilder(MappedFieldType fieldType, + boolean fixBrokenAnalysis, String[] preTags, String[] postTags, BoundaryScanner boundaryScanner) { super(preTags, postTags, boundaryScanner); this.fieldType = fieldType; + this.fixBrokenAnalysis = fixBrokenAnalysis; } @Override protected String makeFragment( StringBuilder buffer, int[] index, Field[] values, WeightedFragInfo fragInfo, - String[] preTags, String[] postTags, Encoder encoder ){ - WeightedFragInfo weightedFragInfo = FragmentBuilderHelper.fixWeightedFragInfo(fieldType, values, fragInfo); - return super.makeFragment(buffer, index, values, weightedFragInfo, preTags, postTags, encoder); + String[] preTags, String[] postTags, Encoder encoder) { + if (fixBrokenAnalysis) { + fragInfo = FragmentBuilderHelper.fixWeightedFragInfo(fragInfo); + } + return super.makeFragment(buffer, index, values, fragInfo, preTags, postTags, encoder); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceScoreOrderFragmentsBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceScoreOrderFragmentsBuilder.java index dd8df92a8670c..aa62aaf9813f3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceScoreOrderFragmentsBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceScoreOrderFragmentsBuilder.java @@ -35,8 +35,10 @@ public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder private final MappedFieldType fieldType; private final SourceLookup sourceLookup; + private final boolean fixBrokenAnalysis; public SourceScoreOrderFragmentsBuilder(MappedFieldType fieldType, + boolean fixBrokenAnalysis, SourceLookup sourceLookup, String[] preTags, String[] postTags, @@ -44,6 +46,7 @@ public SourceScoreOrderFragmentsBuilder(MappedFieldType fieldType, super(preTags, postTags, boundaryScanner); this.fieldType = fieldType; this.sourceLookup = sourceLookup; + this.fixBrokenAnalysis = fixBrokenAnalysis; } @Override @@ -59,8 +62,10 @@ protected Field[] getFields(IndexReader reader, int docId, String fieldName) thr @Override protected String makeFragment( StringBuilder buffer, int[] index, Field[] values, WeightedFragInfo fragInfo, - String[] preTags, String[] postTags, Encoder encoder ){ - WeightedFragInfo weightedFragInfo = FragmentBuilderHelper.fixWeightedFragInfo(fieldType, values, fragInfo); - return super.makeFragment(buffer, index, values, weightedFragInfo, preTags, postTags, encoder); + String[] preTags, String[] postTags, Encoder encoder) { + if (fixBrokenAnalysis) { + fragInfo = FragmentBuilderHelper.fixWeightedFragInfo(fragInfo); + } + return super.makeFragment(buffer, index, values, fragInfo, preTags, postTags, encoder); } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceSimpleFragmentsBuilder.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceSimpleFragmentsBuilder.java index 59a90c0749522..ec5542e1c407b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceSimpleFragmentsBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceSimpleFragmentsBuilder.java @@ -33,11 +33,12 @@ public class SourceSimpleFragmentsBuilder extends SimpleFragmentsBuilder { private final SourceLookup sourceLookup; public SourceSimpleFragmentsBuilder(MappedFieldType fieldType, + boolean fixBrokenAnalysis, SourceLookup sourceLookup, String[] preTags, String[] postTags, BoundaryScanner boundaryScanner) { - super(fieldType, preTags, postTags, boundaryScanner); + super(fieldType, fixBrokenAnalysis, preTags, postTags, boundaryScanner); this.sourceLookup = sourceLookup; } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index d9a853a4eb45e..675638f8ceba2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.action.admin.indices.rollover; -import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; @@ -564,7 +563,7 @@ public void testRolloverClusterStateForDataStream() throws Exception { when(mockedTimestampField.copyTo()).thenReturn(FieldMapper.CopyTo.empty()); when(mockedTimestampField.multiFields()).thenReturn(FieldMapper.MultiFields.empty()); MappingLookup mappingLookup = - new MappingLookup(List.of(mockedTimestampField, dateFieldMapper), List.of(), List.of(), 0, new StandardAnalyzer()); + new MappingLookup(List.of(mockedTimestampField, dateFieldMapper), List.of(), List.of(), 0); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); Environment env = mock(Environment.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 300ee18cfbc51..592d5c0c4b7d8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -126,17 +126,22 @@ public void testDefaultConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); MappedFieldType completionFieldType = ((CompletionFieldMapper) fieldMapper).fieldType(); - NamedAnalyzer indexAnalyzer = completionFieldType.indexAnalyzer(); - assertThat(indexAnalyzer.name(), equalTo("simple")); - assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); - CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); - assertThat(analyzer.preservePositionIncrements(), equalTo(true)); - assertThat(analyzer.preserveSep(), equalTo(true)); + boolean[] registered = new boolean[1]; + ((CompletionFieldMapper) fieldMapper).registerIndexAnalyzer((n, a) -> { + NamedAnalyzer indexAnalyzer = (NamedAnalyzer) a; + assertThat(indexAnalyzer.name(), equalTo("simple")); + assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(true)); + registered[0] = true; + }); + assertTrue("Analyzer not registered", registered[0]); NamedAnalyzer searchAnalyzer = completionFieldType.getTextSearchInfo().getSearchAnalyzer(); assertThat(searchAnalyzer.name(), equalTo("simple")); assertThat(searchAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); - analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); + CompletionAnalyzer analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); assertThat(analyzer.preservePositionIncrements(), equalTo(true)); assertThat(analyzer.preserveSep(), equalTo(true)); } @@ -155,17 +160,22 @@ public void testCompletionAnalyzerSettings() throws Exception { assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); MappedFieldType completionFieldType = ((CompletionFieldMapper) fieldMapper).fieldType(); - NamedAnalyzer indexAnalyzer = completionFieldType.indexAnalyzer(); - assertThat(indexAnalyzer.name(), equalTo("simple")); - assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); - CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); - assertThat(analyzer.preservePositionIncrements(), equalTo(true)); - assertThat(analyzer.preserveSep(), equalTo(false)); + boolean[] registered = new boolean[1]; + ((CompletionFieldMapper) fieldMapper).registerIndexAnalyzer((n, a) -> { + NamedAnalyzer indexAnalyzer = (NamedAnalyzer) a; + assertThat(indexAnalyzer.name(), equalTo("simple")); + assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(false)); + registered[0] = true; + }); + assertTrue("Analyzer not registered", registered[0]); NamedAnalyzer searchAnalyzer = completionFieldType.getTextSearchInfo().getSearchAnalyzer(); assertThat(searchAnalyzer.name(), equalTo("standard")); assertThat(searchAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); - analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); + CompletionAnalyzer analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); assertThat(analyzer.preservePositionIncrements(), equalTo(true)); assertThat(analyzer.preserveSep(), equalTo(false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 4c90806e941fe..67d9f5b8d38cc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -24,14 +24,13 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.index.analysis.AnalyzerScope; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.StringReader; import java.util.Arrays; import java.util.Collections; +import java.util.function.BiConsumer; public class DocumentFieldMapperTests extends LuceneTestCase { @@ -47,10 +46,10 @@ private static class FakeAnalyzer extends Analyzer { protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = new Tokenizer() { boolean incremented = false; - CharTermAttribute term = addAttribute(CharTermAttribute.class); + final CharTermAttribute term = addAttribute(CharTermAttribute.class); @Override - public boolean incrementToken() throws IOException { + public boolean incrementToken() { if (incremented) { return false; } @@ -83,14 +82,22 @@ public String typeName() { static class FakeFieldMapper extends ParametrizedFieldMapper { - FakeFieldMapper(FakeFieldType fieldType) { + final String indexedValue; + + FakeFieldMapper(FakeFieldType fieldType, String indexedValue) { super(fieldType.name(), fieldType, MultiFields.empty(), CopyTo.empty()); + this.indexedValue = indexedValue; } @Override protected void parseCreateField(ParseContext context) { } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), new FakeAnalyzer(indexedValue)); + } + @Override protected String contentType() { return null; @@ -104,23 +111,20 @@ public Builder getMergeBuilder() { public void testAnalyzers() throws IOException { FakeFieldType fieldType1 = new FakeFieldType("field1"); - fieldType1.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new FakeAnalyzer("index"))); - FieldMapper fieldMapper1 = new FakeFieldMapper(fieldType1); + FieldMapper fieldMapper1 = new FakeFieldMapper(fieldType1, "index1"); FakeFieldType fieldType2 = new FakeFieldType("field2"); - FieldMapper fieldMapper2 = new FakeFieldMapper(fieldType2); - - Analyzer defaultIndex = new FakeAnalyzer("default_index"); + FieldMapper fieldMapper2 = new FakeFieldMapper(fieldType2, "index2"); MappingLookup mappingLookup = new MappingLookup( Arrays.asList(fieldMapper1, fieldMapper2), Collections.emptyList(), Collections.emptyList(), - 0, defaultIndex); - - assertAnalyzes(mappingLookup.indexAnalyzer(), "field1", "index"); + 0); - assertAnalyzes(mappingLookup.indexAnalyzer(), "field2", "default_index"); + assertAnalyzes(mappingLookup.indexAnalyzer(), "field1", "index1"); + assertAnalyzes(mappingLookup.indexAnalyzer(), "field2", "index2"); + expectThrows(IllegalArgumentException.class, () -> mappingLookup.indexAnalyzer().tokenStream("field3", "blah")); } private void assertAnalyzes(Analyzer analyzer, String field, String output) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index c9ed5268c4fe9..7d411dc2baffe 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.builders.PointBuilder; @@ -33,6 +34,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.function.BiConsumer; /** * This mapper add a new sub fields @@ -177,6 +179,11 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), generatedValue, mapperName); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + + } + @Override protected String contentType() { return mapperName; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 398e5da9c46b8..b153f9ed8b859 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -28,6 +29,7 @@ import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.function.BiConsumer; // Like a String mapper but with very few options. We just use it to test if highlighting on a custom string mapped field works as expected. public class FakeStringFieldMapper extends ParametrizedFieldMapper { @@ -67,7 +69,6 @@ public static final class FakeStringFieldType extends StringFieldType { private FakeStringFieldType(String name, boolean stored, TextSearchInfo textSearchInfo) { super(name, true, stored, true, textSearchInfo, Collections.emptyMap()); - setIndexAnalyzer(Lucene.STANDARD_ANALYZER); } @Override @@ -86,6 +87,11 @@ protected FakeStringFieldMapper(MappedFieldType mappedFieldType, super(mappedFieldType.name(), mappedFieldType, multiFields, copyTo); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.STANDARD_ANALYZER); + } + @Override protected void parseCreateField(ParseContext context) throws IOException { String value; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index a25ebf4fe2407..0351d8dd7b75f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; @@ -43,7 +42,7 @@ public void testDuplicateFieldAliasAndObject() { new MappingLookup( Collections.emptyList(), singletonList(objectMapper), - singletonList(aliasMapper), 0, Lucene.STANDARD_ANALYZER)); + singletonList(aliasMapper), 0)); assertEquals("Alias [some.path] is defined both as an object and an alias", e.getMessage()); } @@ -56,7 +55,7 @@ public void testDuplicateFieldAliasAndConcreteField() { new MappingLookup( Arrays.asList(field, invalidField), emptyList(), - singletonList(invalidAlias), 0, Lucene.STANDARD_ANALYZER)); + singletonList(invalidAlias), 0)); assertEquals("Alias [invalid] is defined both as an alias and a concrete field", e.getMessage()); } @@ -69,7 +68,7 @@ public void testAliasThatRefersToAlias() { MappingLookup mappers = new MappingLookup( singletonList(field), emptyList(), - Arrays.asList(alias, invalidAlias), 0, Lucene.STANDARD_ANALYZER); + Arrays.asList(alias, invalidAlias), 0); alias.validate(mappers); MapperParsingException e = expectThrows(MapperParsingException.class, () -> { @@ -87,7 +86,7 @@ public void testAliasThatRefersToItself() { MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias), 0, null); + singletonList(invalidAlias), 0); invalidAlias.validate(mappers); }); @@ -102,7 +101,7 @@ public void testAliasWithNonExistentPath() { MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias), 0, Lucene.STANDARD_ANALYZER); + singletonList(invalidAlias), 0); invalidAlias.validate(mappers); }); @@ -118,7 +117,7 @@ public void testFieldAliasWithNestedScope() { singletonList(createFieldMapper("nested", "field")), singletonList(objectMapper), singletonList(aliasMapper), - 0, Lucene.STANDARD_ANALYZER); + 0); aliasMapper.validate(mappers); } @@ -130,7 +129,7 @@ public void testFieldAliasWithDifferentObjectScopes() { List.of(createFieldMapper("object1", "field")), List.of(createObjectMapper("object1"), createObjectMapper("object2")), singletonList(aliasMapper), - 0, Lucene.STANDARD_ANALYZER); + 0); aliasMapper.validate(mappers); } @@ -143,7 +142,7 @@ public void testFieldAliasWithNestedTarget() { singletonList(createFieldMapper("nested", "field")), Collections.singletonList(objectMapper), singletonList(aliasMapper), - 0, Lucene.STANDARD_ANALYZER); + 0); aliasMapper.validate(mappers); }); @@ -161,7 +160,7 @@ public void testFieldAliasWithDifferentNestedScopes() { singletonList(createFieldMapper("nested1", "field")), List.of(createNestedObjectMapper("nested1"), createNestedObjectMapper("nested2")), singletonList(aliasMapper), - 0, Lucene.STANDARD_ANALYZER); + 0); aliasMapper.validate(mappers); }); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 244dbef598d97..af0f6997bdd44 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.Version; import org.elasticsearch.common.Strings; @@ -42,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.BiConsumer; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; @@ -191,6 +193,11 @@ protected void parseCreateField(ParseContext context) { } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + + } + @Override protected String contentType() { return "test_mapper"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 8ba1f23dcfe5f..2b30c2fac063e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -36,7 +36,6 @@ import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.lucene.BytesRefs; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; @@ -165,7 +164,6 @@ public void testIndexPrefixes() { public void testFetchSourceValue() throws IOException { TextFieldType fieldType = createFieldType(); - fieldType.setIndexAnalyzer(Lucene.STANDARD_ANALYZER); assertEquals(List.of("value"), fetchSourceValue(fieldType, "value")); assertEquals(List.of("42"), fetchSourceValue(fieldType, 42L)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java index e992eab92bf04..1c8b469b907f2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTermsAggregatorTests.java @@ -37,8 +37,6 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.analysis.AnalyzerScope; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -108,9 +106,8 @@ protected Map getFieldAliases(MappedFieldType... fieldT public void testSignificance() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); textFieldType.setFielddata(true); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment @@ -253,9 +250,8 @@ public void testNumericSignificance() throws IOException { public void testUnmapped() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); textFieldType.setFielddata(true); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { @@ -321,9 +317,8 @@ public void testRangeField() throws IOException { public void testFieldAlias() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); textFieldType.setFielddata(true); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java index 543c9429cdd36..6f9599e0289c6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java @@ -34,8 +34,6 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.analysis.AnalyzerScope; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -80,7 +78,10 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy protected List getSupportedValuesSourceTypes() { // TODO it is likely accidental that SigText supports anything other than Bytes, and then only text fields return List.of(CoreValuesSourceType.NUMERIC, + CoreValuesSourceType.BOOLEAN, CoreValuesSourceType.BYTES, + CoreValuesSourceType.DATE, + CoreValuesSourceType.IP, CoreValuesSourceType.RANGE, CoreValuesSourceType.GEOPOINT); } @@ -98,9 +99,8 @@ protected List unsupportedMappedFieldTypes() { */ public void testSignificance() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { @@ -146,9 +146,8 @@ public void testSignificance() throws IOException { public void testFieldAlias() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { @@ -197,9 +196,8 @@ public void testFieldAlias() throws IOException { public void testInsideTermsAgg() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { @@ -256,9 +254,8 @@ private void indexDocuments(IndexWriter writer) throws IOException { */ public void testSignificanceOnTextArrays() throws IOException { TextFieldType textFieldType = new TextFieldType("text"); - textFieldType.setIndexAnalyzer(new NamedAnalyzer("my_analyzer", AnalyzerScope.GLOBAL, new StandardAnalyzer())); - IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); + IndexWriterConfig indexWriterConfig = newIndexWriterConfig(new StandardAnalyzer()); indexWriterConfig.setMaxBufferedDocs(100); indexWriterConfig.setRAMBufferSizeMB(100); // flush on open to have a single segment try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, indexWriterConfig)) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index f43b34eb29af8..119356babe567 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -26,6 +27,7 @@ import java.util.Collections; import java.util.List; +import java.util.function.BiConsumer; // this sucks how much must be overridden just do get a dummy field mapper... public class MockFieldMapper extends ParametrizedFieldMapper { @@ -80,6 +82,11 @@ protected String contentType() { return null; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + + } + @Override protected void parseCreateField(ParseContext context) { } diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 743f1833a5131..b09e36bb349d2 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -311,6 +311,7 @@ public boolean shouldCache(Query query) { MapperService mapperService = mapperServiceMock(); when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.hasNested()).thenReturn(false); + when(mapperService.indexAnalyzer()).thenReturn(new StandardAnalyzer()); // for significant text when(searchContext.mapperService()).thenReturn(mapperService); IndexFieldDataService ifds = new IndexFieldDataService(indexSettings, new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index b9914c3f86abf..f2e4f295dc1e8 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -8,6 +8,7 @@ import com.carrotsearch.hppc.DoubleArrayList; import com.carrotsearch.hppc.IntArrayList; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; import org.apache.lucene.index.BinaryDocValues; @@ -20,6 +21,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentSubParser; @@ -51,6 +53,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -224,6 +227,11 @@ public Query termQuery(Object value, QueryShardContext context) { } } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + @Override public void parse(ParseContext context) throws IOException { if (context.externalValueSet()) { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 54b8418441829..15ed757f756b0 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.constantkeyword.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; @@ -45,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -236,6 +238,11 @@ public ConstantKeywordFieldType fieldType() { return (ConstantKeywordFieldType) super.fieldType(); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + + } + @Override protected void parseCreateField(ParseContext context) throws IOException { String value; diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 9b1dd9c95e8ca..c0ea60f686940 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.flattened.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.OrdinalMap; @@ -54,6 +55,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -167,7 +169,6 @@ public KeyedFlatObjectFieldType(String name, boolean indexed, boolean hasDocValu super(name, indexed, false, hasDocValues, splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.key = key; } @@ -379,7 +380,6 @@ public RootFlatObjectFieldType(String name, boolean indexed, boolean hasDocValue super(name, indexed, false, hasDocValues, splitQueriesOnWhitespace ? TextSearchInfo.WHITESPACE_MATCH_ONLY : TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.splitQueriesOnWhitespace = splitQueriesOnWhitespace; - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -452,6 +452,11 @@ public String keyedFieldName() { return mappedFieldType.name() + KEYED_FIELD_SUFFIX; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + @Override protected void parseCreateField(ParseContext context) throws IOException { if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 0ffc98edef592..a7a7289ca5ef9 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.exc.InputCoercionException; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -46,6 +47,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; @@ -488,6 +490,10 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault).init(this); diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 798b1c9fd2785..5b4f331f2cbe0 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.versionfield; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; @@ -58,6 +59,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; @@ -124,7 +126,6 @@ public static final class VersionStringFieldType extends TermBasedFieldType { private VersionStringFieldType(String name, FieldType fieldType, Map meta) { super(name, true, false, true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } @Override @@ -320,6 +321,11 @@ public VersionStringFieldType fieldType() { return (VersionStringFieldType) super.fieldType(); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + } + @Override protected String contentType() { return CONTENT_TYPE; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java index f4b757d505adb..11a60a49cf921 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.runtimefields.mapper; +import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.index.mapper.BooleanFieldMapper; @@ -24,6 +25,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.BiFunction; public final class RuntimeFieldMapper extends ParametrizedFieldMapper { @@ -61,6 +63,11 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { return new RuntimeFieldMapper.Builder(simpleName(), scriptCompiler).init(this); } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + + } + @Override protected void parseCreateField(ParseContext context) { // there is no lucene field diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 32ce1788ac1aa..655e9dd5c446b 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.vectors.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Query; @@ -35,6 +36,7 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -237,6 +239,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexCreatedVersion).init(this); diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 30df174a3dd5d..25a56dfa0c784 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.vectors.mapper; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.search.Query; import org.elasticsearch.Version; import org.elasticsearch.common.logging.DeprecationLogger; @@ -24,6 +25,7 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; +import java.util.function.BiConsumer; /** * A {@link FieldMapper} for indexing a sparse vector of floats. @@ -134,6 +136,10 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 57b2d0ff58b2d..fea3ab6da85bd 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -77,6 +77,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -250,14 +251,15 @@ public static final class WildcardFieldType extends MappedFieldType { private final String nullValue; private final int ignoreAbove; + private final Analyzer analyzer; private WildcardFieldType(String name, String nullValue, int ignoreAbove, Version version, Map meta) { super(name, true, false, true, Defaults.TEXT_SEARCH_INFO, meta); if (version.onOrAfter(Version.V_7_10_0)) { - setIndexAnalyzer(WILDCARD_ANALYZER_7_10); + this.analyzer = WILDCARD_ANALYZER_7_10; } else { - setIndexAnalyzer(WILDCARD_ANALYZER_7_9); + this.analyzer = WILDCARD_ANALYZER_7_9; } this.nullValue = nullValue; this.ignoreAbove = ignoreAbove; @@ -641,7 +643,7 @@ protected void getNgramTokens(Set tokens, String fragment) { return; } // Break fragment into multiple Ngrams - TokenStream tokenizer = indexAnalyzer().tokenStream(name(), fragment); + TokenStream tokenizer = analyzer.tokenStream(name(), fragment); CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class); int foundTokens = 0; try { @@ -660,7 +662,7 @@ protected void getNgramTokens(Set tokens, String fragment) { if (foundTokens == 0 && fragment.length() > 0) { // fragment must have been less than NGRAM_SIZE - add a placeholder which may be used in a prefix query e.g. ab* fragment = toLowerCase(fragment); - if (indexAnalyzer() == WILDCARD_ANALYZER_7_10) { + if (analyzer == WILDCARD_ANALYZER_7_10) { fragment = PunctuationFoldingFilter.normalize(fragment); } tokens.add(fragment); @@ -786,7 +788,7 @@ public Query fuzzyQuery( } } // Tokenize all content after the prefix - TokenStream tokenizer = indexAnalyzer().tokenStream(name(), postPrefixString); + TokenStream tokenizer = analyzer.tokenStream(name(), postPrefixString); CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class); ArrayList postPrefixTokens = new ArrayList<>(); String firstToken = null; @@ -1007,6 +1009,11 @@ protected String contentType() { return CONTENT_TYPE; } + @Override + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { + analyzerRegistry.accept(name(), fieldType().analyzer); + } + @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexVersionCreated).init(this); From d098bcdace36fcc663ede0d9e34cbbef748a870d Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 20 Oct 2020 14:39:12 +0100 Subject: [PATCH 02/19] precommit --- .../org/elasticsearch/percolator/PercolateQueryBuilder.java | 2 +- .../java/org/elasticsearch/index/mapper/IdFieldMapper.java | 3 ++- .../xpack/unsignedlong/UnsignedLongFieldMapper.java | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index a583c31831eb6..dbc4008da93ca 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -471,7 +471,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { docs.add(docMapper.parse(new SourceToParse(context.index().getName(), "_temp_id", document, documentXContentType))); } - FieldNameAnalyzer fieldNameAnalyzer = (FieldNameAnalyzer) docMapper.mappers().indexAnalyzer(); + FieldNameAnalyzer fieldNameAnalyzer = docMapper.mappers().indexAnalyzer(); // Need to this custom impl because FieldNameAnalyzer is strict and the percolator sometimes isn't when // 'index.percolator.map_unmapped_fields_as_string' is enabled: Analyzer analyzer = new DelegatingAnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index c6c5cf8ff4807..f97bd5e33ab9a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -201,7 +201,8 @@ public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested @Override public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, - Nested nested, SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) { + Nested nested, SortOrder sortOrder, DocValueFormat format, + int bucketSize, BucketedSort.ExtraData extra) { throw new UnsupportedOperationException("can't sort on the [" + CONTENT_TYPE + "] field"); } }; diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 0e046907451eb..c4ba1a1faf273 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -494,8 +494,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - } + public void registerIndexAnalyzer(BiConsumer analyzerRegistry) {} @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { From 4cccf095acf1c7fc9e1cbacf26600e144137fc66 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 20 Oct 2020 18:33:10 +0100 Subject: [PATCH 03/19] delegating analyzers --- .../index/mapper/TextFieldMapper.java | 37 ++++++++++++++----- .../index/mapper/TextFieldMapperTests.java | 11 ++++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index a80e5e0d45bed..2f35f287e5e9a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -359,11 +359,14 @@ private PrefixFieldMapper buildPrefixMapper(BuilderContext context, FieldType fi } PrefixFieldType prefixFieldType = new PrefixFieldType(tft, fullName + "._index_prefix", indexPrefixes.get()); tft.setPrefixFieldType(prefixFieldType); - return new PrefixFieldMapper(pft, prefixFieldType, - new PrefixWrappedAnalyzer(analyzers.getIndexAnalyzer(), prefixFieldType.minChars, prefixFieldType.maxChars)); + return new PrefixFieldMapper(pft, prefixFieldType, new PrefixWrappedAnalyzer( + analyzers.getIndexAnalyzer().analyzer(), + analyzers.positionIncrementGap.get(), + prefixFieldType.minChars, + prefixFieldType.maxChars)); } - private PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType parent, NamedAnalyzer parentAnalyzer) { + private PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType parent) { if (indexPhrases.get() == false) { return null; } @@ -375,7 +378,9 @@ private PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType p } FieldType phraseFieldType = new FieldType(fieldType); parent.setIndexPhrases(); - return new PhraseFieldMapper(phraseFieldType, new PhraseFieldType(parent), parentAnalyzer); + PhraseWrappedAnalyzer a + = new PhraseWrappedAnalyzer(analyzers.getIndexAnalyzer().analyzer(), analyzers.positionIncrementGap.get()); + return new PhraseFieldMapper(phraseFieldType, new PhraseFieldType(parent), a); } @Override @@ -383,7 +388,7 @@ public TextFieldMapper build(BuilderContext context) { FieldType fieldType = TextParams.buildFieldType(index, store, indexOptions, norms, termVectors); TextFieldType tft = buildFieldType(fieldType, context); return new TextFieldMapper(name, fieldType, tft, - buildPrefixMapper(context, fieldType, tft), buildPhraseMapper(fieldType, tft, analyzers.getIndexAnalyzer()), + buildPrefixMapper(context, fieldType, tft), buildPhraseMapper(fieldType, tft), multiFieldsBuilder.build(this, context), copyTo.build(), this); } } @@ -394,10 +399,17 @@ public TextFieldMapper build(BuilderContext context) { private static class PhraseWrappedAnalyzer extends AnalyzerWrapper { private final Analyzer delegate; + private final int posIncGap; - PhraseWrappedAnalyzer(Analyzer delegate) { + PhraseWrappedAnalyzer(Analyzer delegate, int posIncGap) { super(delegate.getReuseStrategy()); this.delegate = delegate; + this.posIncGap = posIncGap; + } + + @Override + public int getPositionIncrementGap(String fieldName) { + return posIncGap; } @Override @@ -415,11 +427,13 @@ private static class PrefixWrappedAnalyzer extends AnalyzerWrapper { private final int minChars; private final int maxChars; + private final int posIncGap; private final Analyzer delegate; - PrefixWrappedAnalyzer(Analyzer delegate, int minChars, int maxChars) { + PrefixWrappedAnalyzer(Analyzer delegate, int posIncGap, int minChars, int maxChars) { super(delegate.getReuseStrategy()); this.delegate = delegate; + this.posIncGap = posIncGap; this.minChars = minChars; this.maxChars = maxChars; } @@ -429,6 +443,11 @@ protected Analyzer getWrappedAnalyzer(String fieldName) { return delegate; } + @Override + public int getPositionIncrementGap(String fieldName) { + return posIncGap; + } + @Override protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) { TokenFilter filter = new EdgeNGramTokenFilter(components.getTokenStream(), minChars, maxChars, false); @@ -552,9 +571,9 @@ private static final class PhraseFieldMapper extends FieldMapper { final Analyzer analyzer; - PhraseFieldMapper(FieldType fieldType, PhraseFieldType mappedFieldType, NamedAnalyzer analyzer) { + PhraseFieldMapper(FieldType fieldType, PhraseFieldType mappedFieldType, PhraseWrappedAnalyzer analyzer) { super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); - this.analyzer = new PhraseWrappedAnalyzer(analyzer.analyzer()); + this.analyzer = analyzer; } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 1bc46afae8a5e..9ff9962c6e130 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -768,7 +768,7 @@ protected TokenStreamComponents createComponents(String fieldName) { public void testIndexPrefixMapping() throws IOException { { - DocumentMapper mapper = createDocumentMapper( + MapperService ms = createMapperService( fieldMapping( b -> b.field("type", "text") .field("analyzer", "standard") @@ -779,11 +779,12 @@ public void testIndexPrefixMapping() throws IOException { ) ); - assertThat(mapper.mappers().getMapper("field._index_prefix").toString(), containsString("prefixChars=2:10")); + assertThat(ms.documentMapper().mappers().getMapper("field._index_prefix").toString(), containsString("prefixChars=2:10")); - ParsedDocument doc = mapper.parse(source(b -> b.field("field", "Some English text that is going to be very useful"))); + ParsedDocument doc = ms.documentMapper().parse(source(b -> b.field("field", "Some English text that is going to be very useful"))); IndexableField[] fields = doc.rootDoc().getFields("field._index_prefix"); assertEquals(1, fields.length); + withLuceneIndex(ms, iw -> iw.addDocument(doc.rootDoc()), ir -> {}); // check we can index } { @@ -861,6 +862,10 @@ public void testFastPhrasePrefixes() throws IOException { } b.endObject(); })); + + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("synfield", "some text which we will index"))); + withLuceneIndex(mapperService, iw -> iw.addDocument(doc.rootDoc()), ir -> {}); // check indexing + QueryShardContext queryShardContext = createQueryShardContext(mapperService); { From c9437df65f23fa53c62c1b94748e9d1876cb1632 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 21 Oct 2020 09:37:41 +0100 Subject: [PATCH 04/19] precommit --- .../org/elasticsearch/index/mapper/TextFieldMapperTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 9ff9962c6e130..20d3cc25db1da 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -781,7 +781,8 @@ public void testIndexPrefixMapping() throws IOException { assertThat(ms.documentMapper().mappers().getMapper("field._index_prefix").toString(), containsString("prefixChars=2:10")); - ParsedDocument doc = ms.documentMapper().parse(source(b -> b.field("field", "Some English text that is going to be very useful"))); + ParsedDocument doc + = ms.documentMapper().parse(source(b -> b.field("field", "Some English text that is going to be very useful"))); IndexableField[] fields = doc.rootDoc().getFields("field._index_prefix"); assertEquals(1, fields.length); withLuceneIndex(ms, iw -> iw.addDocument(doc.rootDoc()), ir -> {}); // check we can index From 32382ad10226da332423054c78351e3d10ad688a Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 21 Oct 2020 11:12:39 +0100 Subject: [PATCH 05/19] position increments on annotations --- .../mapper/annotatedtext/AnnotatedTextFieldMapper.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 058abc56b83e0..e4cdb90a6f26c 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -73,14 +73,14 @@ public class AnnotatedTextFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "annotated_text"; - private static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1; private static Builder builder(FieldMapper in) { return ((AnnotatedTextFieldMapper)in).builder; } private static NamedAnalyzer wrapAnalyzer(NamedAnalyzer in) { - return new NamedAnalyzer(in.name(), AnalyzerScope.INDEX, new AnnotationAnalyzerWrapper(in.analyzer())); + return new NamedAnalyzer(in.name(), AnalyzerScope.INDEX, + new AnnotationAnalyzerWrapper(in.analyzer()), in.getPositionIncrementGap("")); } public static class Builder extends ParametrizedFieldMapper.Builder { @@ -130,6 +130,12 @@ public AnnotatedTextFieldMapper build(BuilderContext context) { if (fieldType.indexOptions() == IndexOptions.NONE ) { throw new IllegalArgumentException("[" + CONTENT_TYPE + "] fields must be indexed"); } + if (analyzers.positionIncrementGap.get() != TextParams.POSITION_INCREMENT_GAP_USE_ANALYZER) { + if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { + throw new IllegalArgumentException("Cannot set position_increment_gap on field [" + + name + "] without positions enabled"); + } + } return new AnnotatedTextFieldMapper( name, fieldType, buildFieldType(fieldType, context), multiFieldsBuilder.build(this, context), copyTo.build(), this); From f1b8095fdddd17457339486d13d1954e89ed8871 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 27 Oct 2020 15:29:11 +0000 Subject: [PATCH 06/19] Move to returning maps --- .../index/mapper/RankFeatureFieldMapper.java | 6 +-- .../index/mapper/RankFeaturesFieldMapper.java | 5 +-- .../index/mapper/ScaledFloatFieldMapper.java | 6 --- .../mapper/SearchAsYouTypeFieldMapper.java | 13 +++---- .../index/mapper/TokenCountFieldMapper.java | 6 --- .../join/mapper/ParentIdFieldMapper.java | 6 +-- .../join/mapper/ParentJoinFieldMapper.java | 5 +-- .../percolator/PercolatorFieldMapper.java | 6 --- .../ICUCollationKeywordFieldMapper.java | 5 +-- .../AnnotatedTextFieldMapper.java | 6 +-- .../mapper/murmur3/Murmur3FieldMapper.java | 6 --- .../mapper/AbstractGeometryFieldMapper.java | 7 ---- .../index/mapper/BinaryFieldMapper.java | 7 ---- .../index/mapper/BooleanFieldMapper.java | 5 +-- .../index/mapper/CompletionFieldMapper.java | 5 +-- .../index/mapper/DateFieldMapper.java | 7 ---- .../index/mapper/FieldMapper.java | 5 ++- .../index/mapper/IdFieldMapper.java | 6 +-- .../index/mapper/IpFieldMapper.java | 6 --- .../index/mapper/KeywordFieldMapper.java | 5 +-- .../mapper/LegacyGeoShapeFieldMapper.java | 5 +-- .../index/mapper/MappingLookup.java | 2 +- .../index/mapper/MetadataFieldMapper.java | 7 ---- .../index/mapper/NumberFieldMapper.java | 9 ----- .../index/mapper/RangeFieldMapper.java | 8 ---- .../index/mapper/RoutingFieldMapper.java | 6 +-- .../index/mapper/TextFieldMapper.java | 24 ++++++------ .../mapper/CompletionFieldMapperTests.java | 38 +++++++------------ .../mapper/DocumentFieldMapperTests.java | 6 +-- .../index/mapper/ExternalMapper.java | 7 ---- .../index/mapper/FakeStringFieldMapper.java | 6 +-- .../index/mapper/ParametrizedMapperTests.java | 7 ---- .../index/mapper/MockFieldMapper.java | 7 ---- .../mapper/HistogramFieldMapper.java | 8 ---- .../mapper/ConstantKeywordFieldMapper.java | 7 ---- .../mapper/FlattenedFieldMapper.java | 6 +-- .../unsignedlong/UnsignedLongFieldMapper.java | 5 --- .../VersionStringFieldMapper.java | 6 +-- .../mapper/RuntimeFieldMapper.java | 7 ---- .../mapper/DenseVectorFieldMapper.java | 7 ---- .../mapper/SparseVectorFieldMapper.java | 6 --- .../wildcard/mapper/WildcardFieldMapper.java | 6 +-- 42 files changed, 79 insertions(+), 234 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 22c860d7a87ef..f2b8d8a5f3413 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -34,9 +34,9 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -190,8 +190,8 @@ protected String contentType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index f3170a3ddf7dd..ae9150f8ed5c2 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -33,7 +33,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -163,8 +162,8 @@ protected String contentType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index e7920a75f653b..4896a8c2129c5 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; @@ -54,7 +53,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; /** A {@link FieldMapper} for scaled floats. Values are internally multiplied @@ -314,10 +312,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - } - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault, coerceByDefault).init(this); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index 121301d0fdb94..daa932fec5b51 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -60,7 +60,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType.hasGaps; @@ -419,8 +418,8 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), analyzer); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), analyzer); } @Override @@ -453,8 +452,8 @@ public ShingleFieldType fieldType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), analyzer); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), analyzer); } @Override @@ -592,8 +591,8 @@ protected String contentType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), analyzer); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), analyzer); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index e3744e6bcf82d..ae21b3711ee19 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -29,7 +29,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue; @@ -196,11 +195,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - - } - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 397ff297c0529..c19ddbcde219b 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -38,7 +38,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; -import java.util.function.BiConsumer; +import java.util.Map; import java.util.function.Supplier; /** @@ -118,8 +118,8 @@ protected String contentType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } @Override diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index d920e0c3df3d3..546b33a7b5f61 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -54,7 +54,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -209,8 +208,8 @@ protected String contentType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } @Override diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 10a32387dc16e..ba22a1ddd9e02 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.percolator; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.BinaryRange; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; @@ -89,7 +88,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; @@ -474,10 +472,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - } - boolean isMapUnmappedFieldAsText() { return mapUnmappedFieldsAsText; } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 76cf78edd43b4..88499dffb1b8a 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -49,7 +49,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; public class ICUCollationKeywordFieldMapper extends ParametrizedFieldMapper { @@ -437,8 +436,8 @@ public CollationFieldType fieldType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } @Override diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index e4cdb90a6f26c..fb71f4ba69624 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -50,11 +50,11 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -521,8 +521,8 @@ protected AnnotatedTextFieldMapper(String simpleName, FieldType fieldType, Annot } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), analyzer); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), analyzer); } @Override diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 08b85162dadd2..291f488647ba4 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper.murmur3; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; @@ -44,7 +43,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; public class Murmur3FieldMapper extends ParametrizedFieldMapper { @@ -134,10 +132,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - } - @Override protected void parseCreateField(ParseContext context) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 1e9896b6e4eff..af59dc64ec9fa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.elasticsearch.common.Explicit; @@ -38,7 +37,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Function; /** @@ -168,11 +166,6 @@ public AbstractGeometryFieldType fieldType() { return (AbstractGeometryFieldType) mappedFieldType; } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - - } - @Override protected void parseCreateField(ParseContext context) throws IOException { throw new UnsupportedOperationException("Parsing is implemented in parse(), this method should NEVER be called"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index ab07c0f555732..eb3bf22929a7f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.hppc.ObjectArrayList; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.StoredField; import org.apache.lucene.search.Query; import org.apache.lucene.store.ByteArrayDataOutput; @@ -43,7 +42,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; public class BinaryFieldMapper extends ParametrizedFieldMapper { @@ -195,11 +193,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - - } - public static class CustomBinaryDocValuesField extends CustomDocValuesField { private final ObjectArrayList bytesList; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index de254741309c0..5c523cf6f9a02 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -45,7 +45,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -270,8 +269,8 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index e3b0987133fca..a5fe404d5b605 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -55,7 +55,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.BiConsumer; /** * Mapper for completion field. The field values are indexed as a weighted FST for @@ -327,8 +326,8 @@ public boolean parsesArrayValue() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), analyzer); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), analyzer); } int getMaxInputLength() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 1b65e3a47be56..d6b594cbaa353 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; @@ -59,7 +58,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; @@ -628,11 +626,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - - } - public boolean getIgnoreMalformed() { return ignoreMalformed; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index e1d0de516ace1..8c432c0974c47 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -46,7 +46,6 @@ import java.util.Map; import java.util.Objects; import java.util.TreeMap; -import java.util.function.BiConsumer; import java.util.stream.StreamSupport; public abstract class FieldMapper extends Mapper implements Cloneable { @@ -468,7 +467,9 @@ protected static String indexOptionToString(IndexOptions indexOption) { protected abstract String contentType(); - public abstract void registerIndexAnalyzer(BiConsumer analyzerRegistry); + public Map indexAnalyzers() { + return Collections.emptyMap(); + } public static class MultiFields implements Iterable { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index f97bd5e33ab9a..76f41d9c3f23c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -55,7 +55,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.function.BiConsumer; +import java.util.Map; import java.util.function.BooleanSupplier; import java.util.function.Supplier; @@ -212,8 +212,8 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } private static LeafFieldData wrap(LeafFieldData in) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 73197ba842915..9dbf263690c47 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; @@ -48,7 +47,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Supplier; @@ -446,8 +444,4 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).init(this); } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index dce19de9e58cd..a27c58616fc03 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -43,7 +43,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -351,8 +350,8 @@ public KeywordFieldType fieldType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), fieldType().normalizer()); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), fieldType().normalizer); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index dc21a6bb6d7c3..4c9460ae382e9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -56,7 +56,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.BiConsumer; /** * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s. @@ -479,8 +478,8 @@ String strategy() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index abf16523aff8e..c9b0ce0d3d7a2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -102,7 +102,7 @@ public MappingLookup(Collection mappers, if (fieldMappers.put(mapper.name(), mapper) != null) { throw new MapperParsingException("Field [" + mapper.name() + "] is defined more than once"); } - mapper.registerIndexAnalyzer(indexAnalyzers::put); + indexAnalyzers.putAll(mapper.indexAnalyzers()); } this.metadataFieldCount = metadataFieldCount; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 2298ac1da6c22..bbba108e2e803 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -19,14 +19,12 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; import java.io.IOException; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Function; @@ -160,11 +158,6 @@ protected void parseCreateField(ParseContext context) throws IOException { + " a document. Use the index API request parameters."); } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - // no analyzers to register - } - /** * Called before {@link FieldMapper#parse(ParseContext)} on the {@link RootObjectMapper}. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index a5069f1289c07..f9cd7e6ff9ce1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.exc.InputCoercionException; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; @@ -38,7 +37,6 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Numbers; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -60,7 +58,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -1043,12 +1040,6 @@ public NumberFieldType fieldType() { return (NumberFieldType) super.fieldType(); } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - // allows number fields in significant text aggs - do we need this? - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); - } - @Override protected String contentType() { return fieldType().type.typeName(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 3ff868414bf6c..4be82e4194130 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -19,14 +19,12 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -54,7 +52,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; @@ -458,11 +455,6 @@ public Object getTo() { } } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); - } - static class BinaryRangesDocValuesField extends CustomDocValuesField { private final Set ranges; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 8d2ed3808124b..19e051fadbd7c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -28,7 +28,7 @@ import java.util.Collections; import java.util.List; -import java.util.function.BiConsumer; +import java.util.Map; public class RoutingFieldMapper extends MetadataFieldMapper { @@ -127,8 +127,8 @@ protected String contentType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 2f35f287e5e9a..1be05ac98b6fa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -78,11 +78,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.BiConsumer; import java.util.function.IntPredicate; import java.util.function.Supplier; @@ -590,11 +590,6 @@ protected void mergeOptions(FieldMapper other, List conflicts) { protected String contentType() { return "phrase"; } - - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), analyzer); - } } private static final class PrefixFieldMapper extends FieldMapper { @@ -620,11 +615,6 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), analyzer); - } - @Override protected String contentType() { return "prefix"; @@ -929,8 +919,16 @@ public TextFieldType fieldType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), builder.analyzers.getIndexAnalyzer()); + public Map indexAnalyzers() { + Map analyzers = new HashMap<>(); + analyzers.put(name(), builder.analyzers.getIndexAnalyzer()); + if (phraseFieldMapper != null) { + analyzers.put(phraseFieldMapper.name(), phraseFieldMapper.analyzer); + } + if (prefixFieldMapper != null) { + analyzers.put(prefixFieldMapper.name(), prefixFieldMapper.analyzer); + } + return analyzers; } public static Query createPhraseQuery(TokenStream stream, String field, int slop, boolean enablePositionIncrements) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 592d5c0c4b7d8..379ea2079b756 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -126,22 +126,17 @@ public void testDefaultConfiguration() throws IOException { assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); MappedFieldType completionFieldType = ((CompletionFieldMapper) fieldMapper).fieldType(); - boolean[] registered = new boolean[1]; - ((CompletionFieldMapper) fieldMapper).registerIndexAnalyzer((n, a) -> { - NamedAnalyzer indexAnalyzer = (NamedAnalyzer) a; - assertThat(indexAnalyzer.name(), equalTo("simple")); - assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); - CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); - assertThat(analyzer.preservePositionIncrements(), equalTo(true)); - assertThat(analyzer.preserveSep(), equalTo(true)); - registered[0] = true; - }); - assertTrue("Analyzer not registered", registered[0]); + NamedAnalyzer indexAnalyzer = (NamedAnalyzer) ((CompletionFieldMapper) fieldMapper).indexAnalyzers().values().iterator().next(); + assertThat(indexAnalyzer.name(), equalTo("simple")); + assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(true)); NamedAnalyzer searchAnalyzer = completionFieldType.getTextSearchInfo().getSearchAnalyzer(); assertThat(searchAnalyzer.name(), equalTo("simple")); assertThat(searchAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); - CompletionAnalyzer analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); + analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); assertThat(analyzer.preservePositionIncrements(), equalTo(true)); assertThat(analyzer.preserveSep(), equalTo(true)); } @@ -160,22 +155,17 @@ public void testCompletionAnalyzerSettings() throws Exception { assertThat(fieldMapper, instanceOf(CompletionFieldMapper.class)); MappedFieldType completionFieldType = ((CompletionFieldMapper) fieldMapper).fieldType(); - boolean[] registered = new boolean[1]; - ((CompletionFieldMapper) fieldMapper).registerIndexAnalyzer((n, a) -> { - NamedAnalyzer indexAnalyzer = (NamedAnalyzer) a; - assertThat(indexAnalyzer.name(), equalTo("simple")); - assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); - CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); - assertThat(analyzer.preservePositionIncrements(), equalTo(true)); - assertThat(analyzer.preserveSep(), equalTo(false)); - registered[0] = true; - }); - assertTrue("Analyzer not registered", registered[0]); + NamedAnalyzer indexAnalyzer = (NamedAnalyzer) ((CompletionFieldMapper) fieldMapper).indexAnalyzers().values().iterator().next(); + assertThat(indexAnalyzer.name(), equalTo("simple")); + assertThat(indexAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); + CompletionAnalyzer analyzer = (CompletionAnalyzer) indexAnalyzer.analyzer(); + assertThat(analyzer.preservePositionIncrements(), equalTo(true)); + assertThat(analyzer.preserveSep(), equalTo(false)); NamedAnalyzer searchAnalyzer = completionFieldType.getTextSearchInfo().getSearchAnalyzer(); assertThat(searchAnalyzer.name(), equalTo("standard")); assertThat(searchAnalyzer.analyzer(), instanceOf(CompletionAnalyzer.class)); - CompletionAnalyzer analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); + analyzer = (CompletionAnalyzer) searchAnalyzer.analyzer(); assertThat(analyzer.preservePositionIncrements(), equalTo(true)); assertThat(analyzer.preserveSep(), equalTo(false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 67d9f5b8d38cc..3bf287a8e2746 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -30,7 +30,7 @@ import java.io.StringReader; import java.util.Arrays; import java.util.Collections; -import java.util.function.BiConsumer; +import java.util.Map; public class DocumentFieldMapperTests extends LuceneTestCase { @@ -94,8 +94,8 @@ protected void parseCreateField(ParseContext context) { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), new FakeAnalyzer(indexedValue)); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), new FakeAnalyzer(indexedValue)); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 1fa0e53452410..d408f38ee7549 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.builders.PointBuilder; @@ -34,7 +33,6 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.function.BiConsumer; /** * This mapper add a new sub fields @@ -179,11 +177,6 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), generatedValue, mapperName); } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - - } - @Override protected String contentType() { return mapperName; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index b153f9ed8b859..da3aa7dd60a16 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -29,7 +29,7 @@ import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.function.BiConsumer; +import java.util.Map; // Like a String mapper but with very few options. We just use it to test if highlighting on a custom string mapped field works as expected. public class FakeStringFieldMapper extends ParametrizedFieldMapper { @@ -88,8 +88,8 @@ protected FakeStringFieldMapper(MappedFieldType mappedFieldType, } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.STANDARD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.STANDARD_ANALYZER); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 27e6f2f3011d6..4fb708398058a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.Version; import org.elasticsearch.common.Strings; @@ -43,7 +42,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.BiConsumer; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; @@ -193,11 +191,6 @@ protected void parseCreateField(ParseContext context) { } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - - } - @Override protected String contentType() { return "test_mapper"; diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 119356babe567..f43b34eb29af8 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -27,7 +26,6 @@ import java.util.Collections; import java.util.List; -import java.util.function.BiConsumer; // this sucks how much must be overridden just do get a dummy field mapper... public class MockFieldMapper extends ParametrizedFieldMapper { @@ -82,11 +80,6 @@ protected String contentType() { return null; } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - - } - @Override protected void parseCreateField(ParseContext context) { } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index f2e4f295dc1e8..b9914c3f86abf 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -8,7 +8,6 @@ import com.carrotsearch.hppc.DoubleArrayList; import com.carrotsearch.hppc.IntArrayList; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; import org.apache.lucene.index.BinaryDocValues; @@ -21,7 +20,6 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentSubParser; @@ -53,7 +51,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -227,11 +224,6 @@ public Query termQuery(Object value, QueryShardContext context) { } } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); - } - @Override public void parse(ParseContext context) throws IOException { if (context.externalValueSet()) { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 15ed757f756b0..54b8418441829 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.constantkeyword.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; @@ -46,7 +45,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -238,11 +236,6 @@ public ConstantKeywordFieldType fieldType() { return (ConstantKeywordFieldType) super.fieldType(); } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - - } - @Override protected void parseCreateField(ParseContext context) throws IOException { String value; diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java index f38bbacb246a8..8a4c3e2f6decd 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java @@ -53,9 +53,9 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -453,8 +453,8 @@ public String keyedFieldName() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } @Override diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index ead706201a963..6e8a219a2befa 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.exc.InputCoercionException; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -47,7 +46,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; @@ -493,9 +491,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) {} - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault).init(this); diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 5b4f331f2cbe0..6dd71837709e6 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -56,10 +56,10 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; @@ -322,8 +322,8 @@ public VersionStringFieldType fieldType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), Lucene.KEYWORD_ANALYZER); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java index 5a9989ee0b901..630ca4df72f5e 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.runtimefields.mapper; -import org.apache.lucene.analysis.Analyzer; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.index.mapper.BooleanFieldMapper; @@ -26,7 +25,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.BiFunction; public final class RuntimeFieldMapper extends ParametrizedFieldMapper { @@ -64,11 +62,6 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { return new RuntimeFieldMapper.Builder(simpleName(), scriptCompiler).init(this); } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - - } - @Override protected void parseCreateField(ParseContext context) { // there is no lucene field diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 655e9dd5c446b..32ce1788ac1aa 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.vectors.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Query; @@ -36,7 +35,6 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; import java.util.function.Supplier; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; @@ -239,11 +237,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - - } - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexCreatedVersion).init(this); diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 25a56dfa0c784..30df174a3dd5d 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.vectors.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.search.Query; import org.elasticsearch.Version; import org.elasticsearch.common.logging.DeprecationLogger; @@ -25,7 +24,6 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; -import java.util.function.BiConsumer; /** * A {@link FieldMapper} for indexing a sparse vector of floats. @@ -136,10 +134,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - } - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index fea3ab6da85bd..26f039d1ccc55 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -72,12 +72,12 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.BiConsumer; import java.util.function.Supplier; /** @@ -1010,8 +1010,8 @@ protected String contentType() { } @Override - public void registerIndexAnalyzer(BiConsumer analyzerRegistry) { - analyzerRegistry.accept(name(), fieldType().analyzer); + public Map indexAnalyzers() { + return Collections.singletonMap(name(), fieldType().analyzer); } @Override From bb81e255fc362f3afb43df1666bf0bf495fdcc04 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 27 Oct 2020 15:48:45 +0000 Subject: [PATCH 07/19] imports --- .../bucket/terms/SignificantTextAggregatorTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java index eb91b8d0330a7..cb5b35387ffa2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java @@ -35,7 +35,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.BinaryFieldMapper; -import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; From 4f3da1d43c7615e7ae2c51704a66966485a8791e Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 29 Oct 2020 11:42:30 +0000 Subject: [PATCH 08/19] Make everything a NamedAnalyzer --- .../index/mapper/RankFeatureFieldMapper.java | 4 +- .../index/mapper/RankFeaturesFieldMapper.java | 4 +- .../mapper/SearchAsYouTypeFieldMapper.java | 25 +++++--- .../SearchAsYouTypeFieldMapperTests.java | 63 ++++++++++++------- .../join/mapper/ParentIdFieldMapper.java | 4 +- .../join/mapper/ParentJoinFieldMapper.java | 4 +- .../ICUCollationKeywordFieldMapper.java | 4 +- .../AnnotatedTextFieldMapper.java | 4 +- .../index/mapper/BooleanFieldMapper.java | 4 +- .../index/mapper/CompletionFieldMapper.java | 3 +- .../index/mapper/FieldMapper.java | 3 +- .../index/mapper/IdFieldMapper.java | 4 +- .../index/mapper/KeywordFieldMapper.java | 3 +- .../mapper/LegacyGeoShapeFieldMapper.java | 4 +- .../index/mapper/RoutingFieldMapper.java | 4 +- .../index/mapper/TextFieldMapper.java | 16 +++-- .../mapper/DocumentFieldMapperTests.java | 8 ++- .../index/mapper/FakeStringFieldMapper.java | 4 +- .../mapper/FlattenedFieldMapper.java | 4 +- .../VersionStringFieldMapper.java | 4 +- .../wildcard/mapper/WildcardFieldMapper.java | 4 +- 21 files changed, 103 insertions(+), 74 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index f2b8d8a5f3413..4dd2a81455705 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.FeatureField; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -28,6 +27,7 @@ import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SearchLookup; @@ -190,7 +190,7 @@ protected String contentType() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index ae9150f8ed5c2..def94b7548536 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -19,12 +19,12 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.FeatureField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Query; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SearchLookup; @@ -162,7 +162,7 @@ protected String contentType() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index daa932fec5b51..88cd6b5679f79 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -47,6 +47,7 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; @@ -176,7 +177,9 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { TextSearchInfo prefixSearchInfo = new TextSearchInfo(prefixft, similarity.getValue(), prefixSearchWrapper, searchAnalyzer); final PrefixFieldType prefixFieldType = new PrefixFieldType(fullName, prefixSearchInfo, Defaults.MIN_GRAM, Defaults.MAX_GRAM); - final PrefixFieldMapper prefixFieldMapper = new PrefixFieldMapper(prefixft, prefixFieldType, prefixIndexWrapper); + final NamedAnalyzer prefixAnalyzer + = new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, prefixIndexWrapper); + final PrefixFieldMapper prefixFieldMapper = new PrefixFieldMapper(prefixft, prefixFieldType, prefixAnalyzer); // set up the shingle fields final ShingleFieldMapper[] shingleFieldMappers = new ShingleFieldMapper[maxShingleSize.getValue() - 1]; @@ -198,7 +201,9 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { final ShingleFieldType shingleFieldType = new ShingleFieldType(fieldName, shingleSize, textSearchInfo); shingleFieldType.setPrefixFieldType(prefixFieldType); shingleFieldTypes[i] = shingleFieldType; - shingleFieldMappers[i] = new ShingleFieldMapper(shingleft, shingleFieldType, shingleIndexWrapper); + NamedAnalyzer shingleAnalyzer + = new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, shingleIndexWrapper); + shingleFieldMappers[i] = new ShingleFieldMapper(shingleft, shingleFieldType, shingleAnalyzer); } ft.setPrefixField(prefixFieldType); ft.setShingleFields(shingleFieldTypes); @@ -391,9 +396,9 @@ public Query existsQuery(QueryShardContext context) { static final class PrefixFieldMapper extends FieldMapper { - final Analyzer analyzer; + final NamedAnalyzer analyzer; - PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType, Analyzer analyzer) { + PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType, NamedAnalyzer analyzer) { super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); this.analyzer = analyzer; } @@ -418,7 +423,7 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), analyzer); } @@ -435,9 +440,9 @@ public String toString() { static final class ShingleFieldMapper extends FieldMapper { - final Analyzer analyzer; + final NamedAnalyzer analyzer; - ShingleFieldMapper(FieldType fieldType, ShingleFieldType mappedFieldtype, Analyzer analyzer) { + ShingleFieldMapper(FieldType fieldType, ShingleFieldType mappedFieldtype, NamedAnalyzer analyzer) { super(mappedFieldtype.name(), fieldType, mappedFieldtype, MultiFields.empty(), CopyTo.empty()); this.analyzer = analyzer; } @@ -452,7 +457,7 @@ public ShingleFieldType fieldType() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), analyzer); } @@ -551,7 +556,7 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew private final int maxShingleSize; private final PrefixFieldMapper prefixField; private final ShingleFieldMapper[] shingleFields; - private final Analyzer analyzer; + private final NamedAnalyzer analyzer; private final Builder builder; public SearchAsYouTypeFieldMapper(String simpleName, @@ -591,7 +596,7 @@ protected String contentType() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), analyzer); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 2ebe860e7ae26..b0786bce80f3f 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -178,12 +178,12 @@ public void testDefaultConfiguration() throws IOException { assertRootFieldMapper(rootMapper, 3, "default"); PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(defaultMapper, "field._index_prefix"); - assertPrefixFieldType(prefixFieldMapper.fieldType(), 3, "default"); + assertPrefixFieldType(prefixFieldMapper, 3, "default"); assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._2gram").fieldType(), 2, "default", prefixFieldMapper.fieldType()); + getShingleFieldMapper(defaultMapper, "field._2gram"), 2, "default", prefixFieldMapper.fieldType()); assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._3gram").fieldType(), 3, "default", prefixFieldMapper.fieldType()); + getShingleFieldMapper(defaultMapper, "field._3gram"), 3, "default", prefixFieldMapper.fieldType()); } public void testConfiguration() throws IOException { @@ -199,14 +199,14 @@ public void testConfiguration() throws IOException { assertRootFieldMapper(rootMapper, maxShingleSize, analyzerName); PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(defaultMapper, "field._index_prefix"); - assertPrefixFieldType(prefixFieldMapper.fieldType(), maxShingleSize, analyzerName); + assertPrefixFieldType(prefixFieldMapper, maxShingleSize, analyzerName); assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._2gram").fieldType(), 2, analyzerName, prefixFieldMapper.fieldType()); + getShingleFieldMapper(defaultMapper, "field._2gram"), 2, analyzerName, prefixFieldMapper.fieldType()); assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._3gram").fieldType(), 3, analyzerName, prefixFieldMapper.fieldType()); + getShingleFieldMapper(defaultMapper, "field._3gram"), 3, analyzerName, prefixFieldMapper.fieldType()); assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._4gram").fieldType(), 4, analyzerName, prefixFieldMapper.fieldType()); + getShingleFieldMapper(defaultMapper, "field._4gram"), 4, analyzerName, prefixFieldMapper.fieldType()); } public void testSimpleMerge() throws IOException { @@ -580,61 +580,78 @@ private static void assertRootFieldMapper(SearchAsYouTypeFieldMapper mapper, assertThat(mapper.maxShingleSize(), equalTo(maxShingleSize)); assertThat(mapper.fieldType(), notNullValue()); - assertSearchAsYouTypeFieldType(mapper.fieldType(), maxShingleSize, analyzerName, mapper.prefixField().fieldType()); + assertSearchAsYouTypeFieldType(mapper, mapper.fieldType(), maxShingleSize, analyzerName, mapper.prefixField().fieldType()); assertThat(mapper.prefixField(), notNullValue()); assertThat(mapper.prefixField().fieldType().parentField, equalTo(mapper.name())); - assertPrefixFieldType(mapper.prefixField().fieldType(), maxShingleSize, analyzerName); + assertPrefixFieldType(mapper.prefixField(), maxShingleSize, analyzerName); for (int shingleSize = 2; shingleSize <= maxShingleSize; shingleSize++) { final ShingleFieldMapper shingleFieldMapper = mapper.shingleFields()[shingleSize - 2]; assertThat(shingleFieldMapper, notNullValue()); - assertShingleFieldType(shingleFieldMapper.fieldType(), shingleSize, analyzerName, mapper.prefixField().fieldType()); + assertShingleFieldType(shingleFieldMapper, shingleSize, analyzerName, mapper.prefixField().fieldType()); } final int numberOfShingleSubfields = (maxShingleSize - 2) + 1; assertThat(mapper.shingleFields().length, equalTo(numberOfShingleSubfields)); } - private static void assertSearchAsYouTypeFieldType(SearchAsYouTypeFieldType fieldType, int maxShingleSize, + private static void assertSearchAsYouTypeFieldType(SearchAsYouTypeFieldMapper mapper, + SearchAsYouTypeFieldType fieldType, + int maxShingleSize, String analyzerName, PrefixFieldType prefixFieldType) { assertThat(fieldType.shingleFields.length, equalTo(maxShingleSize - 1)); - assertThat(fieldType.getTextSearchInfo().getSearchAnalyzer().name(), equalTo(analyzerName)); + NamedAnalyzer indexAnalyzer = (NamedAnalyzer) mapper.indexAnalyzers().values().iterator().next(); + for (NamedAnalyzer analyzer : asList(indexAnalyzer, fieldType.getTextSearchInfo().getSearchAnalyzer())) { + assertThat(analyzer.name(), equalTo(analyzerName)); + } int shingleSize = 2; - for (ShingleFieldType shingleField : fieldType.shingleFields) { + for (ShingleFieldMapper shingleField : mapper.shingleFields()) { assertShingleFieldType(shingleField, shingleSize++, analyzerName, prefixFieldType); } assertThat(fieldType.prefixField, equalTo(prefixFieldType)); } - private static void assertShingleFieldType(ShingleFieldType fieldType, + private static void assertShingleFieldType(ShingleFieldMapper mapper, int shingleSize, String analyzerName, PrefixFieldType prefixFieldType) { + ShingleFieldType fieldType = mapper.fieldType(); assertThat(fieldType.shingleSize, equalTo(shingleSize)); - NamedAnalyzer analyzer = fieldType.getTextSearchInfo().getSearchAnalyzer(); - assertThat(analyzer.name(), equalTo(analyzerName)); - if (shingleSize > 1) { - final SearchAsYouTypeAnalyzer wrappedAnalyzer = (SearchAsYouTypeAnalyzer) analyzer.analyzer(); - assertThat(wrappedAnalyzer.shingleSize(), equalTo(shingleSize)); - assertThat(wrappedAnalyzer.indexPrefixes(), equalTo(false)); + NamedAnalyzer indexAnalyer = mapper.indexAnalyzers().values().iterator().next(); + for (NamedAnalyzer analyzer : asList(indexAnalyer, fieldType.getTextSearchInfo().getSearchAnalyzer())) { + assertThat(analyzer.name(), equalTo(analyzerName)); + if (shingleSize > 1) { + final SearchAsYouTypeAnalyzer wrappedAnalyzer = (SearchAsYouTypeAnalyzer) analyzer.analyzer(); + assertThat(wrappedAnalyzer.shingleSize(), equalTo(shingleSize)); + assertThat(wrappedAnalyzer.indexPrefixes(), equalTo(false)); + } } assertThat(fieldType.prefixFieldType, equalTo(prefixFieldType)); } - private static void assertPrefixFieldType(PrefixFieldType fieldType, int shingleSize, String analyzerName) { - assertThat(fieldType.getTextSearchInfo().getSearchAnalyzer().name(), equalTo(analyzerName)); + private static void assertPrefixFieldType(PrefixFieldMapper mapper, int shingleSize, String analyzerName) { + PrefixFieldType fieldType = mapper.fieldType(); + NamedAnalyzer indexAnalyzer = (NamedAnalyzer) mapper.indexAnalyzers().values().iterator().next(); + for (NamedAnalyzer analyzer : asList(indexAnalyzer, fieldType.getTextSearchInfo().getSearchAnalyzer())) { + assertThat(analyzer.name(), equalTo(analyzerName)); + } + + final SearchAsYouTypeAnalyzer wrappedIndexAnalyzer = (SearchAsYouTypeAnalyzer) indexAnalyzer.analyzer(); final SearchAsYouTypeAnalyzer wrappedSearchAnalyzer = (SearchAsYouTypeAnalyzer) fieldType.getTextSearchInfo().getSearchAnalyzer().analyzer(); - assertThat(wrappedSearchAnalyzer.shingleSize(), equalTo(shingleSize)); + for (SearchAsYouTypeAnalyzer analyzer : asList(wrappedIndexAnalyzer, wrappedSearchAnalyzer)) { + assertThat(analyzer.shingleSize(), equalTo(shingleSize)); + } + assertThat(wrappedIndexAnalyzer.indexPrefixes(), equalTo(true)); assertThat(wrappedSearchAnalyzer.indexPrefixes(), equalTo(false)); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index c19ddbcde219b..dcb258f8302b8 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -19,13 +19,13 @@ package org.elasticsearch.join.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.MapperService; @@ -118,7 +118,7 @@ protected String contentType() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 546b33a7b5f61..9fb1ffea64da8 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.join.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedDocValuesField; @@ -29,6 +28,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.ContentPath; @@ -208,7 +208,7 @@ protected String contentType() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 88499dffb1b8a..b9612d2b0a370 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -23,7 +23,6 @@ import com.ibm.icu.text.RawCollationKey; import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; @@ -37,6 +36,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.IndexableBinaryStringTools; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; @@ -436,7 +436,7 @@ public CollationFieldType fieldType() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index fb71f4ba69624..0ca4ea76dfc22 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -508,7 +508,7 @@ public String typeName() { } private final FieldType fieldType; - private final Analyzer analyzer; + private final NamedAnalyzer analyzer; private final Builder builder; protected AnnotatedTextFieldMapper(String simpleName, FieldType fieldType, AnnotatedTextFieldType mappedFieldType, @@ -521,7 +521,7 @@ protected AnnotatedTextFieldMapper(String simpleName, FieldType fieldType, Annot } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), analyzer); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 5c523cf6f9a02..4a1545b0a415e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedNumericDocValuesField; @@ -33,6 +32,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -269,7 +269,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index a5fe404d5b605..0e0a5c08d91ae 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -326,7 +325,7 @@ public boolean parsesArrayValue() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), analyzer); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 8c432c0974c47..7ed3910ccf6d9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,7 +21,6 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -467,7 +466,7 @@ protected static String indexOptionToString(IndexOptions indexOption) { protected abstract String contentType(); - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.emptyMap(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 76f41d9c3f23c..33b253b8dfac6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -32,6 +31,7 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -212,7 +212,7 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index a27c58616fc03..c3182c5168530 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Field; @@ -350,7 +349,7 @@ public KeywordFieldType fieldType() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), fieldType().normalizer); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index 4c9460ae382e9..53fa9f9a59715 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; @@ -44,6 +43,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.LegacyGeoShapeQueryProcessor; import org.elasticsearch.index.query.QueryShardContext; import org.locationtech.spatial4j.shape.Shape; @@ -478,7 +478,7 @@ String strategy() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 19e051fadbd7c..3ce29d3f7529a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -19,11 +19,11 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; @@ -127,7 +127,7 @@ protected String contentType() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 1be05ac98b6fa..b02f6ba51a145 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -64,6 +64,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; @@ -919,14 +920,19 @@ public TextFieldType fieldType() { } @Override - public Map indexAnalyzers() { - Map analyzers = new HashMap<>(); - analyzers.put(name(), builder.analyzers.getIndexAnalyzer()); + public Map indexAnalyzers() { + Map analyzers = new HashMap<>(); + NamedAnalyzer main = builder.analyzers.getIndexAnalyzer(); + analyzers.put(name(), main); if (phraseFieldMapper != null) { - analyzers.put(phraseFieldMapper.name(), phraseFieldMapper.analyzer); + analyzers.put( + phraseFieldMapper.name(), + new NamedAnalyzer(main.name() + "_phrase", AnalyzerScope.INDEX, phraseFieldMapper.analyzer)); } if (prefixFieldMapper != null) { - analyzers.put(prefixFieldMapper.name(), prefixFieldMapper.analyzer); + analyzers.put( + prefixFieldMapper.name(), + new NamedAnalyzer(main.name() + "_prefix", AnalyzerScope.INDEX, prefixFieldMapper.analyzer)); } return analyzers; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 3bf287a8e2746..2437fd20f35d9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -24,6 +24,8 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.index.analysis.AnalyzerScope; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -94,8 +96,10 @@ protected void parseCreateField(ParseContext context) { } @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), new FakeAnalyzer(indexedValue)); + public Map indexAnalyzers() { + return Collections.singletonMap( + name(), + new NamedAnalyzer("fake", AnalyzerScope.INDEX, new FakeAnalyzer(indexedValue))); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index da3aa7dd60a16..44193b93b5b11 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -19,11 +19,11 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -88,7 +88,7 @@ protected FakeStringFieldMapper(MappedFieldType mappedFieldType, } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.STANDARD_ANALYZER); } diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java index 8a4c3e2f6decd..2a1b2fed0f17e 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.flattened.mapper; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.OrdinalMap; @@ -21,6 +20,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -453,7 +453,7 @@ public String keyedFieldName() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 6dd71837709e6..c4c1185336c75 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.versionfield; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; @@ -32,6 +31,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.FieldMapper; @@ -322,7 +322,7 @@ public VersionStringFieldType fieldType() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 26f039d1ccc55..f3816b3a5dba3 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -251,7 +251,7 @@ public static final class WildcardFieldType extends MappedFieldType { private final String nullValue; private final int ignoreAbove; - private final Analyzer analyzer; + private final NamedAnalyzer analyzer; private WildcardFieldType(String name, String nullValue, int ignoreAbove, Version version, Map meta) { @@ -1010,7 +1010,7 @@ protected String contentType() { } @Override - public Map indexAnalyzers() { + public Map indexAnalyzers() { return Collections.singletonMap(name(), fieldType().analyzer); } From 73044b58cab37ba8a4dea29e98332b8bd297eef0 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 29 Oct 2020 12:04:05 +0000 Subject: [PATCH 09/19] warnings --- .../index/mapper/SearchAsYouTypeFieldMapperTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index b0786bce80f3f..5445766e7672d 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -604,7 +604,7 @@ private static void assertSearchAsYouTypeFieldType(SearchAsYouTypeFieldMapper ma PrefixFieldType prefixFieldType) { assertThat(fieldType.shingleFields.length, equalTo(maxShingleSize - 1)); - NamedAnalyzer indexAnalyzer = (NamedAnalyzer) mapper.indexAnalyzers().values().iterator().next(); + NamedAnalyzer indexAnalyzer = mapper.indexAnalyzers().values().iterator().next(); for (NamedAnalyzer analyzer : asList(indexAnalyzer, fieldType.getTextSearchInfo().getSearchAnalyzer())) { assertThat(analyzer.name(), equalTo(analyzerName)); } @@ -640,7 +640,7 @@ private static void assertShingleFieldType(ShingleFieldMapper mapper, private static void assertPrefixFieldType(PrefixFieldMapper mapper, int shingleSize, String analyzerName) { PrefixFieldType fieldType = mapper.fieldType(); - NamedAnalyzer indexAnalyzer = (NamedAnalyzer) mapper.indexAnalyzers().values().iterator().next(); + NamedAnalyzer indexAnalyzer = mapper.indexAnalyzers().values().iterator().next(); for (NamedAnalyzer analyzer : asList(indexAnalyzer, fieldType.getTextSearchInfo().getSearchAnalyzer())) { assertThat(analyzer.name(), equalTo(analyzerName)); } From 25f3c0d4cc16fff576b1a6c71ad9437253f7e8f8 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 29 Oct 2020 14:53:56 +0000 Subject: [PATCH 10/19] sigtext --- .../bucket/terms/SignificantTextAggregatorFactory.java | 2 +- .../bucket/terms/SignificantTextAggregatorTests.java | 5 ++++- .../search/aggregations/AggregatorTestCase.java | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java index 73aa1a6bd23f6..c5618c0be9973 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorFactory.java @@ -131,7 +131,7 @@ protected Aggregator createInternal(SearchContext searchContext, Aggregator pare context.lookup().source(), context.bigArrays(), fieldType, - searchContext.mapperService().indexAnalyzer(), + searchContext.getQueryShardContext().getIndexAnalyzer(), sourceFieldNames, filterDuplicateText ); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java index cb5b35387ffa2..8233153c8a6ea 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java @@ -76,7 +76,10 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy @Override protected List getSupportedValuesSourceTypes() { - return List.of(CoreValuesSourceType.BYTES); + return List.of( + CoreValuesSourceType.BOOLEAN, + CoreValuesSourceType.BYTES + ); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 70a197f7619aa..0e4ebf82e88c7 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -280,6 +280,7 @@ public boolean shouldCache(Query query) { MapperService mapperService = mapperServiceMock(); when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.hasNested()).thenReturn(false); + when(mapperService.indexAnalyzer()).thenReturn(new StandardAnalyzer()); // for significant text QueryShardContext queryShardContext = queryShardContextMock(contextIndexSearcher, mapperService, indexSettings, circuitBreakerService, bigArrays); when(searchContext.getQueryShardContext()).thenReturn(queryShardContext); From c04c7d1bbd124b9dd74fbb9d6706edd14bc940d1 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 29 Oct 2020 15:19:04 +0000 Subject: [PATCH 11/19] Collapse ParametrizedFieldMapper into FieldMapper --- .../index/mapper/RankFeatureFieldMapper.java | 11 +- .../index/mapper/RankFeaturesFieldMapper.java | 13 +- .../index/mapper/ScaledFloatFieldMapper.java | 11 +- .../mapper/SearchAsYouTypeFieldMapper.java | 24 +- .../index/mapper/TokenCountFieldMapper.java | 6 +- .../SearchAsYouTypeFieldMapperTests.java | 69 +- .../join/mapper/MetaJoinFieldMapper.java | 5 - .../join/mapper/ParentIdFieldMapper.java | 9 +- .../join/mapper/ParentJoinFieldMapper.java | 12 +- .../percolator/PercolatorFieldMapper.java | 9 +- .../ICUCollationKeywordFieldMapper.java | 6 +- .../AnnotatedTextFieldMapper.java | 12 +- .../mapper/murmur3/Murmur3FieldMapper.java | 7 +- .../index/mapper/size/SizeFieldMapper.java | 3 +- .../mapper/AbstractGeometryFieldMapper.java | 2 +- .../index/mapper/BinaryFieldMapper.java | 6 +- .../index/mapper/BooleanFieldMapper.java | 6 +- .../index/mapper/CompletionFieldMapper.java | 6 +- .../index/mapper/DateFieldMapper.java | 11 +- .../index/mapper/DynamicKeyFieldMapper.java | 2 +- .../index/mapper/FieldMapper.java | 893 ++++++++++++------ .../index/mapper/FieldNamesFieldMapper.java | 2 +- .../index/mapper/GeoPointFieldMapper.java | 6 +- .../index/mapper/GeoShapeFieldMapper.java | 6 +- .../index/mapper/IpFieldMapper.java | 11 +- .../index/mapper/KeywordFieldMapper.java | 11 +- .../mapper/LegacyGeoShapeFieldMapper.java | 4 +- .../index/mapper/MetadataFieldMapper.java | 6 +- .../index/mapper/NumberFieldMapper.java | 11 +- .../index/mapper/ParametrizedFieldMapper.java | 697 -------------- .../index/mapper/RangeFieldMapper.java | 11 +- .../elasticsearch/index/mapper/RangeType.java | 2 +- .../index/mapper/RoutingFieldMapper.java | 2 +- .../index/mapper/SourceFieldMapper.java | 2 +- .../index/mapper/TextFieldMapper.java | 29 +- .../index/mapper/TextParams.java | 2 +- .../index/mapper/TypeParsers.java | 78 +- .../MetadataIndexTemplateServiceTests.java | 7 +- .../index/mapper/BinaryFieldMapperTests.java | 11 +- .../mapper/DocumentFieldMapperTests.java | 2 +- .../index/mapper/ExternalMapper.java | 6 +- .../index/mapper/FakeStringFieldMapper.java | 6 +- .../index/mapper/ParametrizedMapperTests.java | 8 +- .../index/mapper/TextFieldMapperTests.java | 50 +- .../index/mapper/MockFieldMapper.java | 6 +- .../mapper/HistogramFieldMapper.java | 7 +- .../DataStreamTimestampFieldMapper.java | 3 +- .../mapper/ConstantKeywordFieldMapper.java | 12 +- .../mapper/FlattenedFieldMapper.java | 11 +- .../unsignedlong/UnsignedLongFieldMapper.java | 12 +- .../VersionStringFieldMapper.java | 12 +- .../mapper/RuntimeFieldMapper.java | 7 +- .../GeoShapeWithDocValuesFieldMapper.java | 7 +- .../index/mapper/PointFieldMapper.java | 7 +- .../index/mapper/ShapeFieldMapper.java | 5 +- .../mapper/DenseVectorFieldMapper.java | 14 +- .../mapper/SparseVectorFieldMapper.java | 12 +- .../wildcard/mapper/WildcardFieldMapper.java | 12 +- 58 files changed, 854 insertions(+), 1366 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index a64e5ae190114..925e633d9dbab 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -40,7 +40,7 @@ /** * A {@link FieldMapper} that exposes Lucene's {@link FeatureField}. */ -public class RankFeatureFieldMapper extends ParametrizedFieldMapper { +public class RankFeatureFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "rank_feature"; @@ -59,7 +59,7 @@ private static RankFeatureFieldType ft(FieldMapper in) { return ((RankFeatureFieldMapper)in).fieldType(); } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter positiveScoreImpact = Parameter.boolParam("positive_score_impact", false, m -> ft(m).positiveScoreImpact, true); @@ -140,11 +140,6 @@ private RankFeatureFieldMapper(String simpleName, MappedFieldType mappedFieldTyp this.positiveScoreImpact = positiveScoreImpact; } - @Override - protected RankFeatureFieldMapper clone() { - return (RankFeatureFieldMapper) super.clone(); - } - @Override public RankFeatureFieldType fieldType() { return (RankFeatureFieldType) super.fieldType(); @@ -189,7 +184,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index 3831bf46b3344..9bf122bcb915d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.FeatureField; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Query; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentParser.Token; @@ -38,11 +37,11 @@ * A {@link FieldMapper} that exposes Lucene's {@link FeatureField} as a sparse * vector of features. */ -public class RankFeaturesFieldMapper extends ParametrizedFieldMapper { +public class RankFeaturesFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "rank_features"; - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); @@ -101,19 +100,13 @@ public Query termQuery(Object value, QueryShardContext context) { private RankFeaturesFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { super(simpleName, mappedFieldType, multiFields, copyTo); - assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0; } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } - @Override - protected RankFeaturesFieldMapper clone() { - return (RankFeaturesFieldMapper) super.clone(); - } - @Override public RankFeaturesFieldType fieldType() { return (RankFeaturesFieldType) super.fieldType(); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 4896a8c2129c5..13f222e99d88c 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -57,7 +57,7 @@ /** A {@link FieldMapper} for scaled floats. Values are internally multiplied * by a scaling factor and rounded to the closest long. */ -public class ScaledFloatFieldMapper extends ParametrizedFieldMapper { +public class ScaledFloatFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "scaled_float"; @@ -68,7 +68,7 @@ private static ScaledFloatFieldMapper toType(FieldMapper in) { return (ScaledFloatFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -313,15 +313,10 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault, coerceByDefault).init(this); } - @Override - protected ScaledFloatFieldMapper clone() { - return (ScaledFloatFieldMapper) super.clone(); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index e65445b37e20c..ba09983317b4c 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -79,7 +79,7 @@ * └── [ PrefixFieldMapper, PrefixFieldType, analysis wrapped with max_shingle_size-shingles and edge-ngrams ] * */ -public class SearchAsYouTypeFieldMapper extends ParametrizedFieldMapper { +public class SearchAsYouTypeFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "search_as_you_type"; private static final int MAX_SHINGLE_SIZE_LOWER_BOUND = 2; @@ -103,7 +103,7 @@ private static SearchAsYouTypeFieldType ft(FieldMapper in) { return toType(in).fieldType(); } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter index = Parameter.indexParam(m -> toType(m).index, true); private final Parameter store = Parameter.storeParam(m -> toType(m).store, false); @@ -399,8 +399,11 @@ public Query existsQuery(QueryShardContext context) { static final class PrefixFieldMapper extends FieldMapper { + final FieldType fieldType; + PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType) { - super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); + super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); + this.fieldType = fieldType; } @Override @@ -418,8 +421,8 @@ protected void parseCreateField(ParseContext context) { } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - + public Builder getMergeBuilder() { + return null; } @Override @@ -435,8 +438,11 @@ public String toString() { static final class ShingleFieldMapper extends FieldMapper { + private final FieldType fieldType; + ShingleFieldMapper(FieldType fieldType, ShingleFieldType mappedFieldtype) { - super(mappedFieldtype.name(), fieldType, mappedFieldtype, MultiFields.empty(), CopyTo.empty()); + super(mappedFieldtype.name(), mappedFieldtype, MultiFields.empty(), CopyTo.empty()); + this.fieldType = fieldType; } FieldType getLuceneFieldType() { @@ -454,8 +460,8 @@ protected void parseCreateField(ParseContext context) { } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - + public Builder getMergeBuilder() { + return null; } @Override @@ -588,7 +594,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), () -> fieldType().indexAnalyzer()).init(this); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index ae21b3711ee19..069ea93cd3a8f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -36,14 +36,14 @@ * A {@link FieldMapper} that takes a string and writes a count of the tokens in that string * to the index. In most ways the mapper acts just like an {@link NumberFieldMapper}. */ -public class TokenCountFieldMapper extends ParametrizedFieldMapper { +public class TokenCountFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "token_count"; private static TokenCountFieldMapper toType(FieldMapper in) { return (TokenCountFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter index = Parameter.indexParam(m -> toType(m).index, true); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -196,7 +196,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index ec2a1b3e43f87..3c42d0277181d 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -24,6 +24,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -270,53 +271,61 @@ private void assertMultiField(int shingleSize) throws IOException { } } + private static IndexableFieldType fieldType(ParsedDocument doc, String field) { + return doc.rootDoc().getField(field).fieldType(); + } + public void testIndexOptions() throws IOException { DocumentMapper mapper = createDocumentMapper( fieldMapping(b -> b.field("type", "search_as_you_type").field("index_options", "offsets")) ); - assertThat(getRootFieldMapper(mapper, "field").fieldType().fieldType.indexOptions(), + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); + + assertThat(fieldType(doc, "field").indexOptions(), equalTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)); Stream.of( - getPrefixFieldMapper(mapper, "field._index_prefix"), - getShingleFieldMapper(mapper, "field._2gram"), - getShingleFieldMapper(mapper, "field._3gram") - ).forEach(m -> assertThat("for " + m.name(), - m.fieldType.indexOptions(), equalTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS))); + fieldType(doc, "field._index_prefix"), + fieldType(doc, "field._2gram"), + fieldType(doc, "field._3gram") + ).forEach(ft -> assertThat(ft.indexOptions(), equalTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS))); } public void testStore() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type").field("store", true))); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); - assertTrue(getRootFieldMapper(mapper, "field").fieldType().fieldType.stored()); + assertTrue(fieldType(doc, "field").stored()); Stream.of( - getPrefixFieldMapper(mapper, "field._index_prefix"), - getShingleFieldMapper(mapper, "field._2gram"), - getShingleFieldMapper(mapper, "field._3gram") - ).forEach(m -> assertFalse("for " + m.name(), m.fieldType.stored())); + fieldType(doc, "field._index_prefix"), + fieldType(doc, "field._2gram"), + fieldType(doc, "field._3gram") + ).forEach(ft -> assertFalse(ft.stored())); } public void testIndex() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type").field("index", false))); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); Stream.of( - getRootFieldMapper(mapper, "field"), - getPrefixFieldMapper(mapper, "field._index_prefix"), - getShingleFieldMapper(mapper, "field._2gram"), - getShingleFieldMapper(mapper, "field._3gram") - ).forEach(m -> assertThat("for " + m.name(), m.fieldType.indexOptions(), equalTo(IndexOptions.NONE))); + fieldType(doc, "field"), + fieldType(doc, "field._index_prefix"), + fieldType(doc, "field._2gram"), + fieldType(doc, "field._3gram") + ).forEach(ft -> assertThat(ft.indexOptions(), equalTo(IndexOptions.NONE))); } public void testTermVectors() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type").field("term_vector", "yes"))); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); - assertTrue(getRootFieldMapper(mapper, "field").fieldType().fieldType.storeTermVectors()); + assertTrue(fieldType(doc, "field").storeTermVectors()); Stream.of( - getShingleFieldMapper(mapper, "field._2gram"), - getShingleFieldMapper(mapper, "field._3gram") - ).forEach(m -> assertTrue("for " + m.name(), m.fieldType.storeTermVectors())); + fieldType(doc, "field._2gram"), + fieldType(doc, "field._3gram") + ).forEach(ft -> assertTrue(ft.storeTermVectors())); PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(mapper, "field._index_prefix"); assertFalse(prefixFieldMapper.fieldType.storeTermVectors()); @@ -326,12 +335,13 @@ public void testNorms() throws IOException { // default setting { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); Stream.of( - getRootFieldMapper(mapper, "field"), - getShingleFieldMapper(mapper, "field._2gram"), - getShingleFieldMapper(mapper, "field._3gram") - ).forEach(m -> assertFalse("for " + m.name(), m.fieldType.omitNorms())); + fieldType(doc, "field"), + fieldType(doc, "field._2gram"), + fieldType(doc, "field._3gram") + ).forEach(ft -> assertFalse(ft.omitNorms())); PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(mapper, "field._index_prefix"); assertTrue(prefixFieldMapper.fieldType.omitNorms()); @@ -340,14 +350,15 @@ public void testNorms() throws IOException { // can disable norms on search_as_you_type fields { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type").field("norms", false))); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); - assertTrue(getRootFieldMapper(mapper, "field").fieldType().fieldType.omitNorms()); + assertTrue(fieldType(doc, "field").omitNorms()); Stream.of( - getPrefixFieldMapper(mapper, "field._index_prefix"), - getShingleFieldMapper(mapper, "field._2gram"), - getShingleFieldMapper(mapper, "field._3gram") - ).forEach(m -> assertTrue("for " + m.name(), m.fieldType.omitNorms())); + fieldType(doc, "field._index_prefix"), + fieldType(doc, "field._2gram"), + fieldType(doc, "field._3gram") + ).forEach(ft -> assertTrue(ft.omitNorms())); } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 52e0847e85dc3..2ee676a78110f 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -94,11 +94,6 @@ public MetaJoinFieldType fieldType() { return (MetaJoinFieldType) super.fieldType(); } - @Override - protected MetaJoinFieldMapper clone() { - return (MetaJoinFieldMapper) super.clone(); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { throw new IllegalStateException("Should never be called"); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index d31f07923fd4b..fc5df42311782 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -27,8 +27,8 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; @@ -43,7 +43,7 @@ * A field mapper used internally by the {@link ParentJoinFieldMapper} to index * the value that link documents in the index (parent _id or _id if the document is a parent). */ -public final class ParentIdFieldMapper extends ParametrizedFieldMapper { +public final class ParentIdFieldMapper extends FieldMapper { static final String CONTENT_TYPE = "parent"; static class Defaults { @@ -94,11 +94,6 @@ protected ParentIdFieldMapper(String name, boolean eagerGlobalOrdinals) { super(name, new ParentIdFieldType(name, eagerGlobalOrdinals), MultiFields.empty(), CopyTo.empty()); } - @Override - protected ParentIdFieldMapper clone() { - return (ParentIdFieldMapper) super.clone(); - } - @Override protected void parseCreateField(ParseContext context) { if (context.externalValueSet() == false) { diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index e14f6203e1deb..9b94c27bd7914 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -35,7 +35,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; @@ -62,7 +61,7 @@ * This field is only used to ensure that there is a single parent-join field defined in the mapping and * cannot be used to index or query any data. */ -public final class ParentJoinFieldMapper extends ParametrizedFieldMapper { +public final class ParentJoinFieldMapper extends FieldMapper { public static final String NAME = "join"; public static final String CONTENT_TYPE = "join"; @@ -96,7 +95,7 @@ private static ParentJoinFieldMapper toType(FieldMapper in) { return (ParentJoinFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter eagerGlobalOrdinals = Parameter.boolParam("eager_global_ordinals", true, m -> toType(m).eagerGlobalOrdinals, true); @@ -207,11 +206,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - protected ParentJoinFieldMapper clone() { - return (ParentJoinFieldMapper) super.clone(); - } - @Override public JoinFieldType fieldType() { return (JoinFieldType) super.fieldType(); @@ -312,7 +306,7 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index ba22a1ddd9e02..658e847d1f5e9 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -61,7 +61,6 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; @@ -92,7 +91,7 @@ import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; -public class PercolatorFieldMapper extends ParametrizedFieldMapper { +public class PercolatorFieldMapper extends FieldMapper { static final Setting INDEX_MAP_UNMAPPED_FIELDS_AS_TEXT_SETTING = Setting.boolSetting( "index.percolator.map_unmapped_fields_as_text", false, Setting.Property.IndexScope); @@ -110,11 +109,11 @@ public class PercolatorFieldMapper extends ParametrizedFieldMapper { static final String MINIMUM_SHOULD_MATCH_FIELD_NAME = "minimum_should_match_field"; @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), queryShardContext).init(this); } - static class Builder extends ParametrizedFieldMapper.Builder { + static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); @@ -185,7 +184,7 @@ static NumberFieldMapper createMinimumShouldMatchField(BuilderContext context) { } - static class TypeParser implements FieldMapper.TypeParser { + static class TypeParser implements Mapper.TypeParser { @Override public Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index c8ddf6da9bc6a..15c4b67e8c903 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -50,7 +50,7 @@ import java.util.Map; import java.util.function.Supplier; -public class ICUCollationKeywordFieldMapper extends ParametrizedFieldMapper { +public class ICUCollationKeywordFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "icu_collation_keyword"; @@ -186,7 +186,7 @@ private static ICUCollationKeywordFieldMapper toType(FieldMapper in) { return (ICUCollationKeywordFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -441,7 +441,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 442cf76c5bb06..9b1b36cf5129c 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -36,7 +36,6 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextParams; @@ -70,7 +69,7 @@ * This code is largely a copy of TextFieldMapper which is less than ideal - * my attempts to subclass TextFieldMapper failed but we can revisit this. **/ -public class AnnotatedTextFieldMapper extends ParametrizedFieldMapper { +public class AnnotatedTextFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "annotated_text"; private static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1; @@ -79,7 +78,7 @@ private static Builder builder(FieldMapper in) { return ((AnnotatedTextFieldMapper)in).builder; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter store = Parameter.storeParam(m -> builder(m).store.getValue(), false); @@ -531,11 +530,6 @@ protected AnnotatedTextFieldMapper(String simpleName, FieldType fieldType, Annot this.builder = builder; } - @Override - protected AnnotatedTextFieldMapper clone() { - return (AnnotatedTextFieldMapper) super.clone(); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { final String value; @@ -564,7 +558,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), builder.analyzers.indexAnalyzer::getDefaultValue).init(this); } } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 291f488647ba4..d96f48e508663 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -32,7 +32,6 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; @@ -45,7 +44,7 @@ import java.util.Map; import java.util.function.Supplier; -public class Murmur3FieldMapper extends ParametrizedFieldMapper { +public class Murmur3FieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "murmur3"; @@ -61,7 +60,7 @@ private static Murmur3FieldMapper toType(FieldMapper in) { return (Murmur3FieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter stored = Parameter.storeParam(m -> toType(m).fieldType().isStored(), false); final Parameter> meta = Parameter.metaParam(); @@ -123,7 +122,7 @@ protected Murmur3FieldMapper(String simpleName, } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index 86a1123287294..ca3321891d67b 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -25,7 +25,6 @@ import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import java.io.IOException; @@ -90,7 +89,7 @@ public void postParse(ParseContext context) throws IOException { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder().init(this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index af59dc64ec9fa..5a7d59d2879ac 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -42,7 +42,7 @@ /** * Base field mapper class for all spatial field types */ -public abstract class AbstractGeometryFieldMapper extends ParametrizedFieldMapper { +public abstract class AbstractGeometryFieldMapper extends FieldMapper { public static Parameter> ignoreMalformedParam(Function> initializer, boolean ignoreMalformedByDefault) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index eb3bf22929a7f..15055854348dc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -44,7 +44,7 @@ import java.util.Map; import java.util.function.Supplier; -public class BinaryFieldMapper extends ParametrizedFieldMapper { +public class BinaryFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "binary"; @@ -52,7 +52,7 @@ private static BinaryFieldMapper toType(FieldMapper in) { return (BinaryFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, false); @@ -184,7 +184,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new BinaryFieldMapper.Builder(simpleName()).init(this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 1c34517426b34..082355a76348e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -48,7 +48,7 @@ /** * A field mapper for boolean fields. */ -public class BooleanFieldMapper extends ParametrizedFieldMapper { +public class BooleanFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "boolean"; @@ -72,7 +72,7 @@ private static BooleanFieldMapper toType(FieldMapper in) { return (BooleanFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter docValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); @@ -267,7 +267,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 30d1f44b01b9e..7a7eb887de1c5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -76,7 +76,7 @@ * This field can also be extended to add search criteria to suggestions * for query-time filtering and boosting (see {@link ContextMappings} */ -public class CompletionFieldMapper extends ParametrizedFieldMapper { +public class CompletionFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "completion"; /** @@ -85,7 +85,7 @@ public class CompletionFieldMapper extends ParametrizedFieldMapper { static final int COMPLETION_CONTEXTS_LIMIT = 10; @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), defaultAnalyzer, indexVersionCreated).init(this); } @@ -118,7 +118,7 @@ private static CompletionFieldMapper toType(FieldMapper in) { /** * Builder for {@link CompletionFieldMapper} */ - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter analyzer; private final Parameter searchAnalyzer; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index d6b594cbaa353..f188563d05a1b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -66,7 +66,7 @@ import static org.elasticsearch.common.time.DateUtils.toLong; /** A {@link FieldMapper} for dates. */ -public final class DateFieldMapper extends ParametrizedFieldMapper { +public final class DateFieldMapper extends FieldMapper { private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(DateFieldMapper.class); @@ -181,7 +181,7 @@ private static DateFieldMapper toType(FieldMapper in) { return (DateFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter index = Parameter.indexParam(m -> toType(m).indexed, true); private final Parameter docValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -561,7 +561,7 @@ private DateFieldMapper( } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), resolution, null, ignoreMalformedByDefault, indexCreatedVersion).init(this); } @@ -575,11 +575,6 @@ protected String contentType() { return fieldType().resolution.type(); } - @Override - protected DateFieldMapper clone() { - return (DateFieldMapper) super.clone(); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { String dateAsString; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java index fb36d80bb636d..2811d366a13b1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java @@ -37,7 +37,7 @@ * Note: we anticipate that 'flattened' fields will be the only implementation of this * interface. Flattened object fields live in the 'mapper-flattened' module. */ -public abstract class DynamicKeyFieldMapper extends ParametrizedFieldMapper { +public abstract class DynamicKeyFieldMapper extends FieldMapper { public DynamicKeyFieldMapper(String simpleName, MappedFieldType defaultFieldType, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index cb14192aeee80..69ec5ba5874a6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -22,17 +22,21 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.IndexOptions; +import org.elasticsearch.Version; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.AbstractXContentParser; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType; +import org.elasticsearch.index.mapper.Mapper.TypeParser.ParserContext; import java.io.IOException; import java.util.ArrayList; @@ -41,10 +45,15 @@ import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.TreeMap; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.StreamSupport; public abstract class FieldMapper extends Mapper implements Cloneable { @@ -52,147 +61,19 @@ public abstract class FieldMapper extends Mapper implements Cloneable { Setting.boolSetting("index.mapping.ignore_malformed", false, Property.IndexScope); public static final Setting COERCE_SETTING = Setting.boolSetting("index.mapping.coerce", false, Property.IndexScope); - public abstract static class Builder extends Mapper.Builder { - - protected final FieldType fieldType; - protected boolean omitNormsSet = false; - protected boolean indexOptionsSet = false; - protected boolean hasDocValues = true; - protected boolean indexed = true; - protected final MultiFields.Builder multiFieldsBuilder; - protected CopyTo copyTo = CopyTo.empty(); - protected Map meta = Collections.emptyMap(); - // TODO move to KeywordFieldMapper.Builder - protected boolean eagerGlobalOrdinals; - // TODO move to text-specific builder base class - protected NamedAnalyzer indexAnalyzer; - protected NamedAnalyzer searchAnalyzer; - protected NamedAnalyzer searchQuoteAnalyzer; - - protected Builder(String name, FieldType fieldType) { - super(name); - this.fieldType = new FieldType(fieldType); - multiFieldsBuilder = new MultiFields.Builder(); - } - - public Builder index(boolean index) { - this.indexed = index; - if (index == false) { - this.fieldType.setIndexOptions(IndexOptions.NONE); - } - return this; - } - - public Builder store(boolean store) { - this.fieldType.setStored(store); - return this; - } - - public FieldMapper.Builder docValues(boolean docValues) { - this.hasDocValues = docValues; - return this; - } - - public Builder storeTermVectors(boolean termVectors) { - if (termVectors != this.fieldType.storeTermVectors()) { - this.fieldType.setStoreTermVectors(termVectors); - } // don't set it to false, it is default and might be flipped by a more specific option - return this; - } - - public Builder storeTermVectorOffsets(boolean termVectorOffsets) { - if (termVectorOffsets) { - this.fieldType.setStoreTermVectors(termVectorOffsets); - } - this.fieldType.setStoreTermVectorOffsets(termVectorOffsets); - return this; - } - - public Builder storeTermVectorPositions(boolean termVectorPositions) { - if (termVectorPositions) { - this.fieldType.setStoreTermVectors(termVectorPositions); - } - this.fieldType.setStoreTermVectorPositions(termVectorPositions); - return this; - } - - public Builder storeTermVectorPayloads(boolean termVectorPayloads) { - if (termVectorPayloads) { - this.fieldType.setStoreTermVectors(termVectorPayloads); - } - this.fieldType.setStoreTermVectorPayloads(termVectorPayloads); - return this; - } - - public Builder omitNorms(boolean omitNorms) { - this.fieldType.setOmitNorms(omitNorms); - this.omitNormsSet = true; - return this; - } - - public Builder indexOptions(IndexOptions indexOptions) { - this.fieldType.setIndexOptions(indexOptions); - this.indexOptionsSet = true; - return this; - } - - public Builder indexAnalyzer(NamedAnalyzer indexAnalyzer) { - this.indexAnalyzer = indexAnalyzer; - return this; - } - - public Builder searchAnalyzer(NamedAnalyzer searchAnalyzer) { - this.searchAnalyzer = searchAnalyzer; - return this; - } - - public Builder searchQuoteAnalyzer(NamedAnalyzer searchQuoteAnalyzer) { - this.searchQuoteAnalyzer = searchQuoteAnalyzer; - return this; - } - - public Builder setEagerGlobalOrdinals(boolean eagerGlobalOrdinals) { - this.eagerGlobalOrdinals = eagerGlobalOrdinals; - return this; - } - - public Builder addMultiField(Mapper.Builder mapperBuilder) { - multiFieldsBuilder.add(mapperBuilder); - return this; - } - - public Builder copyTo(CopyTo copyTo) { - this.copyTo = copyTo; - return this; - } - - protected String buildFullName(BuilderContext context) { - return context.path().pathAsText(name); - } - /** Set metadata on this field. */ - public Builder meta(Map meta) { - this.meta = meta; - return this; - } - - @Override - public abstract FieldMapper build(BuilderContext context); - } + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(FieldMapper.class); - protected FieldType fieldType; protected MappedFieldType mappedFieldType; protected MultiFields multiFields; protected CopyTo copyTo; - protected FieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, + protected FieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { super(simpleName); if (mappedFieldType.name().isEmpty()) { throw new IllegalArgumentException("name cannot be empty string"); } - fieldType.freeze(); - this.fieldType = fieldType; this.mappedFieldType = mappedFieldType; this.multiFields = multiFields; this.copyTo = Objects.requireNonNull(copyTo); @@ -285,15 +166,6 @@ public Iterator iterator() { return multiFields.iterator(); } - @Override - protected FieldMapper clone() { - try { - return (FieldMapper) super.clone(); - } catch (CloneNotSupportedException e) { - throw new AssertionError(e); - } - } - @Override public final void validate(MappingLookup mappers) { if (this.copyTo() != null && this.copyTo().copyToFields().isEmpty() == false) { @@ -337,86 +209,51 @@ private static void checkNestedScopeCompatibility(String source, String target) } } - @Override - public FieldMapper merge(Mapper mergeWith) { - FieldMapper merged = clone(); - List conflicts = new ArrayList<>(); - if (mergeWith instanceof FieldMapper == false) { - throw new IllegalArgumentException("mapper [" + mappedFieldType.name() + "] cannot be changed from type [" - + contentType() + "] to [" + mergeWith.getClass().getSimpleName() + "]"); - } - FieldMapper toMerge = (FieldMapper) mergeWith; - merged.mergeSharedOptions(toMerge, conflicts); - merged.mergeOptions(toMerge, conflicts); - if (conflicts.isEmpty() == false) { - throw new IllegalArgumentException("Mapper for [" + name() + - "] conflicts with existing mapping:\n" + conflicts.toString()); - } - merged.multiFields = multiFields.merge(toMerge.multiFields); - // apply changeable values - merged.mappedFieldType = toMerge.mappedFieldType; - merged.fieldType = toMerge.fieldType; - merged.copyTo = toMerge.copyTo; - return merged; - } + /** + * Returns a {@link Builder} to be used for merging and serialization + * + * Implement as follows: + * {@code return new MyBuilder(simpleName()).init(this); } + */ + public abstract Builder getMergeBuilder(); - private void mergeSharedOptions(FieldMapper mergeWith, List conflicts) { + @Override + public final FieldMapper merge(Mapper mergeWith) { - if (Objects.equals(this.contentType(), mergeWith.contentType()) == false) { - throw new IllegalArgumentException("mapper [" + fieldType().name() + "] cannot be changed from type [" + contentType() - + "] to [" + mergeWith.contentType() + "]"); + if (mergeWith instanceof FieldMapper == false) { + throw new IllegalArgumentException("mapper [" + name() + "] cannot be changed from type [" + + contentType() + "] to [" + mergeWith.getClass().getSimpleName() + "]"); } + checkIncomingMergeType((FieldMapper)mergeWith); - FieldType other = mergeWith.fieldType; - MappedFieldType otherm = mergeWith.mappedFieldType; - - boolean indexed = fieldType.indexOptions() != IndexOptions.NONE; - boolean mergeWithIndexed = other.indexOptions() != IndexOptions.NONE; - if (indexed != mergeWithIndexed) { - conflicts.add("mapper [" + name() + "] has different [index] values"); - } - // TODO: should be validating if index options go "up" (but "down" is ok) - if (fieldType.indexOptions() != other.indexOptions()) { - conflicts.add("mapper [" + name() + "] has different [index_options] values"); - } - if (fieldType.stored() != other.stored()) { - conflicts.add("mapper [" + name() + "] has different [store] values"); - } - if (this.mappedFieldType.hasDocValues() != otherm.hasDocValues()) { - conflicts.add("mapper [" + name() + "] has different [doc_values] values"); - } - if (fieldType.omitNorms() && !other.omitNorms()) { - conflicts.add("mapper [" + name() + "] has different [norms] values, cannot change from disable to enabled"); - } - if (fieldType.storeTermVectors() != other.storeTermVectors()) { - conflicts.add("mapper [" + name() + "] has different [term_vector] values"); - } - if (fieldType.storeTermVectorOffsets() != other.storeTermVectorOffsets()) { - conflicts.add("mapper [" + name() + "] has different [store_term_vector_offsets] values"); - } - if (fieldType.storeTermVectorPositions() != other.storeTermVectorPositions()) { - conflicts.add("mapper [" + name() + "] has different [store_term_vector_positions] values"); - } - if (fieldType.storeTermVectorPayloads() != other.storeTermVectorPayloads()) { - conflicts.add("mapper [" + name() + "] has different [store_term_vector_payloads] values"); + Builder builder = getMergeBuilder(); + if (builder == null) { + return (FieldMapper) mergeWith; } + Conflicts conflicts = new Conflicts(name()); + builder.merge((FieldMapper) mergeWith, conflicts); + conflicts.check(); + return builder.build(new BuilderContext(Settings.EMPTY, parentPath(name()))); + } - // null and "default"-named index analyzers both mean the default is used - if (mappedFieldType.indexAnalyzer() == null || "default".equals(mappedFieldType.indexAnalyzer().name())) { - if (otherm.indexAnalyzer() != null && "default".equals(otherm.indexAnalyzer().name()) == false) { - conflicts.add("mapper [" + name() + "] has different [analyzer]"); - } - } else if (otherm.indexAnalyzer() == null || "default".equals(otherm.indexAnalyzer().name())) { - conflicts.add("mapper [" + name() + "] has different [analyzer]"); - } else if (mappedFieldType.indexAnalyzer().name().equals(otherm.indexAnalyzer().name()) == false) { - conflicts.add("mapper [" + name() + "] has different [analyzer]"); + private static ContentPath parentPath(String name) { + int endPos = name.lastIndexOf("."); + if (endPos == -1) { + return new ContentPath(0); } + return new ContentPath(name.substring(0, endPos)); } - /** - * Merge type-specific options and check for incompatible settings in mappings to be merged - */ - protected abstract void mergeOptions(FieldMapper other, List conflicts); + protected void checkIncomingMergeType(FieldMapper mergeWith) { + if (Objects.equals(this.getClass(), mergeWith.getClass()) == false) { + throw new IllegalArgumentException("mapper [" + name() + "] cannot be changed from type [" + + contentType() + "] to [" + mergeWith.contentType() + "]"); + } + if (Objects.equals(contentType(), mergeWith.contentType()) == false) { + throw new IllegalArgumentException("mapper [" + name() + "] cannot be changed from type [" + + contentType() + "] to [" + mergeWith.contentType() + "]"); + } + } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { @@ -439,69 +276,10 @@ protected boolean storedByDefault() { } protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - builder.field("type", contentType()); - - if (includeDefaults || mappedFieldType.isSearchable() != indexedByDefault()) { - builder.field("index", mappedFieldType.isSearchable()); - } - if (includeDefaults || mappedFieldType.hasDocValues() != docValuesByDefault()) { - builder.field("doc_values", mappedFieldType.hasDocValues()); - } - if (includeDefaults || fieldType.stored() != storedByDefault()) { - builder.field("store", fieldType.stored()); - } - + getMergeBuilder().toXContent(builder, includeDefaults); multiFields.toXContent(builder, params); copyTo.toXContent(builder, params); - - if (includeDefaults || fieldType().meta().isEmpty() == false) { - builder.field("meta", new TreeMap<>(fieldType().meta())); // ensure consistent order - } - } - - protected final void doXContentAnalyzers(XContentBuilder builder, boolean includeDefaults) throws IOException { - if (fieldType.tokenized() == false) { - return; - } - if (fieldType().indexAnalyzer() == null) { - if (includeDefaults) { - builder.field("analyzer", "default"); - } - } else { - boolean hasDefaultIndexAnalyzer = fieldType().indexAnalyzer().name().equals("default"); - final String searchAnalyzerName = fieldType().getTextSearchInfo().getSearchAnalyzer() == null - ? "default" : fieldType().getTextSearchInfo().getSearchAnalyzer().name(); - final String searchQuoteAnalyzerName = fieldType().getTextSearchInfo().getSearchQuoteAnalyzer() == null - ? searchAnalyzerName : fieldType().getTextSearchInfo().getSearchQuoteAnalyzer().name(); - boolean hasDifferentSearchAnalyzer = searchAnalyzerName.equals(fieldType().indexAnalyzer().name()) == false; - boolean hasDifferentSearchQuoteAnalyzer - = Objects.equals(searchAnalyzerName, searchQuoteAnalyzerName) == false; - if (includeDefaults || hasDefaultIndexAnalyzer == false || hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) { - builder.field("analyzer", fieldType().indexAnalyzer().name()); - if (includeDefaults || hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) { - builder.field("search_analyzer", searchAnalyzerName); - if (includeDefaults || hasDifferentSearchQuoteAnalyzer) { - builder.field("search_quote_analyzer", searchQuoteAnalyzerName); - } - } - } - } - } - - protected static String indexOptionToString(IndexOptions indexOption) { - switch (indexOption) { - case DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS: - return TypeParsers.INDEX_OPTIONS_OFFSETS; - case DOCS_AND_FREQS: - return TypeParsers.INDEX_OPTIONS_FREQS; - case DOCS_AND_FREQS_AND_POSITIONS: - return TypeParsers.INDEX_OPTIONS_POSITIONS; - case DOCS: - return TypeParsers.INDEX_OPTIONS_DOCS; - default: - throw new IllegalArgumentException("Unknown IndexOptions [" + indexOption + "]"); - } } protected abstract String contentType(); @@ -687,4 +465,573 @@ public List copyToFields() { } } + /** + * Serializes a parameter + */ + protected interface Serializer { + void serialize(XContentBuilder builder, String name, T value) throws IOException; + } + + protected interface MergeValidator { + boolean canMerge(T previous, T current, Conflicts conflicts); + } + + /** + * Check on whether or not a parameter should be serialized + */ + protected interface SerializerCheck { + /** + * Check on whether or not a parameter should be serialized + * @param includeDefaults if defaults have been requested + * @param isConfigured if the parameter has a different value to the default + * @param value the parameter value + * @return {@code true} if the value should be serialized + */ + boolean check(boolean includeDefaults, boolean isConfigured, T value); + } + + /** + * A configurable parameter for a field mapper + * @param the type of the value the parameter holds + */ + public static final class Parameter implements Supplier { + + public final String name; + private final List deprecatedNames = new ArrayList<>(); + private final Supplier defaultValue; + private final TriFunction parser; + private final Function initializer; + private boolean acceptsNull = false; + private Consumer validator = null; + private Serializer serializer = XContentBuilder::field; + private SerializerCheck serializerCheck = (includeDefaults, isConfigured, value) -> includeDefaults || isConfigured; + private Function conflictSerializer = Objects::toString; + private boolean deprecated; + private MergeValidator mergeValidator; + private T value; + private boolean isSet; + + /** + * Creates a new Parameter + * @param name the parameter name, used in parsing and serialization + * @param updateable whether the parameter can be updated with a new value during a mapping update + * @param defaultValue the default value for the parameter, used if unspecified in mappings + * @param parser a function that converts an object to a parameter value + * @param initializer a function that reads a parameter value from an existing mapper + */ + public Parameter(String name, boolean updateable, Supplier defaultValue, + TriFunction parser, Function initializer) { + this.name = name; + this.defaultValue = Objects.requireNonNull(defaultValue); + this.value = null; + this.parser = parser; + this.initializer = initializer; + this.mergeValidator = (previous, toMerge, conflicts) -> updateable || Objects.equals(previous, toMerge); + } + + /** + * Returns the current value of the parameter + */ + public T getValue() { + return isSet ? value : defaultValue.get(); + } + + @Override + public T get() { + return getValue(); + } + + /** + * Returns the default value of the parameter + */ + public T getDefaultValue() { + return defaultValue.get(); + } + + /** + * Sets the current value of the parameter + */ + public void setValue(T value) { + this.isSet = true; + this.value = value; + } + + public boolean isConfigured() { + return isSet && Objects.equals(value, defaultValue.get()) == false; + } + + /** + * Allows the parameter to accept a {@code null} value + */ + public Parameter acceptsNull() { + this.acceptsNull = true; + return this; + } + + /** + * Adds a deprecated parameter name. + * + * If this parameter name is encountered during parsing, a deprecation warning will + * be emitted. The parameter will be serialized with its main name. + */ + public Parameter addDeprecatedName(String deprecatedName) { + this.deprecatedNames.add(deprecatedName); + return this; + } + + /** + * Deprecates the entire parameter. + * + * If this parameter is encountered during parsing, a deprecation warning will + * be emitted. + */ + public Parameter deprecated() { + this.deprecated = true; + return this; + } + + /** + * Adds validation to a parameter, called after parsing and merging + */ + public Parameter setValidator(Consumer validator) { + this.validator = validator; + return this; + } + + /** + * Configure a custom serializer for this parameter + */ + public Parameter setSerializer(Serializer serializer, Function conflictSerializer) { + this.serializer = serializer; + this.conflictSerializer = conflictSerializer; + return this; + } + + /** + * Configure a custom serialization check for this parameter + */ + public Parameter setSerializerCheck(SerializerCheck check) { + this.serializerCheck = check; + return this; + } + + /** + * Always serialize this parameter, no matter its value + */ + public Parameter alwaysSerialize() { + this.serializerCheck = (id, ic, v) -> true; + return this; + } + + /** + * Never serialize this parameter, no matter its value + */ + public Parameter neverSerialize() { + this.serializerCheck = (id, ic, v) -> false; + return this; + } + + /** + * Sets a custom merge validator. By default, merges are accepted if the + * parameter is updateable, or if the previous and new values are equal + */ + public Parameter setMergeValidator(MergeValidator mergeValidator) { + this.mergeValidator = mergeValidator; + return this; + } + + private void validate() { + if (validator != null) { + validator.accept(getValue()); + } + } + + private void init(FieldMapper toInit) { + setValue(initializer.apply(toInit)); + } + + private void parse(String field, ParserContext context, Object in) { + setValue(parser.apply(field, context, in)); + } + + private void merge(FieldMapper toMerge, Conflicts conflicts) { + T value = initializer.apply(toMerge); + T current = getValue(); + if (mergeValidator.canMerge(current, value, conflicts)) { + setValue(value); + } else { + conflicts.addConflict(name, conflictSerializer.apply(current), conflictSerializer.apply(value)); + } + } + + protected void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { + if (serializerCheck.check(includeDefaults, isConfigured(), get())) { + serializer.serialize(builder, name, getValue()); + } + } + + /** + * Defines a parameter that takes the values {@code true} or {@code false} + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultValue the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter boolParam(String name, boolean updateable, + Function initializer, boolean defaultValue) { + return new Parameter<>(name, updateable, () -> defaultValue, (n, c, o) -> XContentMapValues.nodeBooleanValue(o), initializer); + } + + /** + * Defines a parameter that takes the values {@code true} or {@code false}, and will always serialize + * its value if configured. + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultValue the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter> explicitBoolParam(String name, boolean updateable, + Function> initializer, + boolean defaultValue) { + Explicit defaultExplicit = new Explicit<>(defaultValue, false); + return new Parameter<>(name, updateable, () -> defaultExplicit, + (n, c, o) -> new Explicit<>(XContentMapValues.nodeBooleanValue(o), true), initializer) + .setSerializer((b, n, v) -> b.field(n, v.value()), v -> Boolean.toString(v.value())); + } + + /** + * Defines a parameter that takes a double value + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultValue the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter doubleParam(String name, boolean updateable, + Function initializer, double defaultValue) { + return new Parameter<>(name, updateable, () -> defaultValue, (n, c, o) -> XContentMapValues.nodeDoubleValue(o), initializer); + } + + /** + * Defines a parameter that takes a float value + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultValue the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter floatParam(String name, boolean updateable, + Function initializer, float defaultValue) { + return new Parameter<>(name, updateable, () -> defaultValue, (n, c, o) -> XContentMapValues.nodeFloatValue(o), initializer); + } + + /** + * Defines a parameter that takes an integer value + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultValue the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter intParam(String name, boolean updateable, + Function initializer, int defaultValue) { + return new Parameter<>(name, updateable, () -> defaultValue, (n, c, o) -> XContentMapValues.nodeIntegerValue(o), initializer); + } + + /** + * Defines a parameter that takes a string value + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultValue the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter stringParam(String name, boolean updateable, + Function initializer, String defaultValue) { + return new Parameter<>(name, updateable, () -> defaultValue, + (n, c, o) -> XContentMapValues.nodeStringValue(o), initializer); + } + + @SuppressWarnings("unchecked") + public static Parameter> stringArrayParam(String name, boolean updateable, + Function> initializer, List defaultValue) { + return new Parameter<>(name, updateable, () -> defaultValue, + (n, c, o) -> { + List values = (List) o; + List strValues = new ArrayList<>(); + for (Object item : values) { + strValues.add(item.toString()); + } + return strValues; + }, initializer); + } + + /** + * Defines a parameter that takes one of a restricted set of string values + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param values the set of values that the parameter can take. The first value in the list + * is the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter restrictedStringParam(String name, boolean updateable, + Function initializer, String... values) { + assert values.length > 0; + Set acceptedValues = new LinkedHashSet<>(Arrays.asList(values)); + return stringParam(name, updateable, initializer, values[0]) + .setValidator(v -> { + if (acceptedValues.contains(v)) { + return; + } + throw new MapperParsingException("Unknown value [" + v + "] for field [" + name + + "] - accepted values are " + acceptedValues.toString()); + }); + } + + /** + * Defines a parameter that takes an analyzer name + * @param name the parameter name + * @param updateable whether the parameter can be changed by a mapping update + * @param initializer a function that reads the parameter value from an existing mapper + * @param defaultAnalyzer the default value, to be used if the parameter is undefined in a mapping + */ + public static Parameter analyzerParam(String name, boolean updateable, + Function initializer, + Supplier defaultAnalyzer) { + return new Parameter<>(name, updateable, defaultAnalyzer, (n, c, o) -> { + String analyzerName = o.toString(); + NamedAnalyzer a = c.getIndexAnalyzers().get(analyzerName); + if (a == null) { + throw new IllegalArgumentException("analyzer [" + analyzerName + "] has not been configured in mappings"); + } + return a; + }, initializer).setSerializer((b, n, v) -> b.field(n, v.name()), NamedAnalyzer::name); + } + + /** + * Declares a metadata parameter + */ + public static Parameter> metaParam() { + return new Parameter<>("meta", true, Collections::emptyMap, + (n, c, o) -> TypeParsers.parseMeta(n, o), m -> m.fieldType().meta()); + } + + public static Parameter indexParam(Function initializer, boolean defaultValue) { + return Parameter.boolParam("index", false, initializer, defaultValue); + } + + public static Parameter storeParam(Function initializer, boolean defaultValue) { + return Parameter.boolParam("store", false, initializer, defaultValue); + } + + public static Parameter docValuesParam(Function initializer, boolean defaultValue) { + return Parameter.boolParam("doc_values", false, initializer, defaultValue); + } + + } + + public static final class Conflicts { + + private final String mapperName; + private final List conflicts = new ArrayList<>(); + + Conflicts(String mapperName) { + this.mapperName = mapperName; + } + + public void addConflict(String parameter, String conflict) { + conflicts.add("Conflict in parameter [" + parameter + "]: " + conflict); + } + + void addConflict(String parameter, String existing, String toMerge) { + conflicts.add("Cannot update parameter [" + parameter + "] from [" + existing + "] to [" + toMerge + "]"); + } + + void check() { + if (conflicts.isEmpty()) { + return; + } + String message = "Mapper for [" + mapperName + "] conflicts with existing mapper:\n\t" + + String.join("\n\t", conflicts); + throw new IllegalArgumentException(message); + } + + } + + /** + * A Builder for a ParametrizedFieldMapper + */ + public abstract static class Builder extends Mapper.Builder { + + protected final MultiFields.Builder multiFieldsBuilder = new MultiFields.Builder(); + protected final CopyTo.Builder copyTo = new CopyTo.Builder(); + + /** + * Creates a new Builder with a field name + */ + protected Builder(String name) { + super(name); + } + + /** + * Initialises all parameters from an existing mapper + */ + public Builder init(FieldMapper initializer) { + for (Parameter param : getParameters()) { + param.init(initializer); + } + for (Mapper subField : initializer.multiFields) { + multiFieldsBuilder.add(subField); + } + return this; + } + + private void merge(FieldMapper in, Conflicts conflicts) { + for (Parameter param : getParameters()) { + param.merge(in, conflicts); + } + for (Mapper newSubField : in.multiFields) { + multiFieldsBuilder.update(newSubField, parentPath(newSubField.name())); + } + this.copyTo.reset(in.copyTo); + validate(); + } + + private void validate() { + for (Parameter param : getParameters()) { + param.validate(); + } + } + + /** + * @return the list of parameters defined for this mapper + */ + protected abstract List> getParameters(); + + @Override + public abstract FieldMapper build(BuilderContext context); + + /** + * Builds the full name of the field, taking into account parent objects + */ + protected String buildFullName(BuilderContext context) { + return context.path().pathAsText(name); + } + + /** + * Writes the current builder parameter values as XContent + */ + protected final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { + for (Parameter parameter : getParameters()) { + parameter.toXContent(builder, includeDefaults); + } + } + + /** + * Parse mapping parameters from a map of mappings + * @param name the field mapper name + * @param parserContext the parser context + * @param fieldNode the root node of the map of mappings for this field + */ + public final void parse(String name, ParserContext parserContext, Map fieldNode) { + Map> paramsMap = new HashMap<>(); + Map> deprecatedParamsMap = new HashMap<>(); + for (Parameter param : getParameters()) { + paramsMap.put(param.name, param); + for (String deprecatedName : param.deprecatedNames) { + deprecatedParamsMap.put(deprecatedName, param); + } + } + String type = (String) fieldNode.remove("type"); + for (Iterator> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + final String propName = entry.getKey(); + final Object propNode = entry.getValue(); + if (Objects.equals("fields", propName)) { + TypeParsers.parseMultiField(multiFieldsBuilder::add, name, parserContext, propName, propNode); + iterator.remove(); + continue; + } + if (Objects.equals("copy_to", propName)) { + TypeParsers.parseCopyFields(propNode).forEach(copyTo::add); + iterator.remove(); + continue; + } + if (Objects.equals("boost", propName)) { + if (parserContext.indexVersionCreated().before(Version.V_8_0_0)) { + deprecationLogger.deprecate( + "boost", + "Parameter [boost] on field [{}] is deprecated and has no effect", + name); + iterator.remove(); + continue; + } else { + throw new MapperParsingException("Unknown parameter [boost] on mapper [" + name + "]"); + } + } + Parameter parameter = deprecatedParamsMap.get(propName); + if (parameter != null) { + deprecationLogger.deprecate(propName, "Parameter [{}] on mapper [{}] is deprecated, use [{}]", + propName, name, parameter.name); + } else { + parameter = paramsMap.get(propName); + } + if (parameter == null) { + if (isDeprecatedParameter(propName, parserContext.indexVersionCreated())) { + deprecationLogger.deprecate(propName, + "Parameter [{}] has no effect on type [{}] and will be removed in future", propName, type); + iterator.remove(); + continue; + } + throw new MapperParsingException("unknown parameter [" + propName + + "] on mapper [" + name + "] of type [" + type + "]"); + } + if (parameter.deprecated) { + deprecationLogger.deprecate(propName, + "Parameter [{}] is deprecated and will be removed in a future version", + propName); + } + if (propNode == null && parameter.acceptsNull == false) { + throw new MapperParsingException("[" + propName + "] on mapper [" + name + + "] of type [" + type + "] must not have a [null] value"); + } + parameter.parse(name, parserContext, propNode); + iterator.remove(); + } + validate(); + } + + // These parameters were previously *always* parsed by TypeParsers#parseField(), even if they + // made no sense; if we've got here, that means that they're not declared on a current mapper, + // and so we emit a deprecation warning rather than failing a previously working mapping. + private static final Set DEPRECATED_PARAMS + = Set.of("store", "meta", "index", "doc_values", "index_options", "similarity"); + + private static boolean isDeprecatedParameter(String propName, Version indexCreatedVersion) { + if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) { + return false; + } + return DEPRECATED_PARAMS.contains(propName); + } + } + + /** + * TypeParser implementation that automatically handles parsing + */ + public static final class TypeParser implements Mapper.TypeParser { + + private final BiFunction builderFunction; + + /** + * Creates a new TypeParser + * @param builderFunction a function that produces a Builder from a name and parsercontext + */ + public TypeParser(BiFunction builderFunction) { + this.builderFunction = builderFunction; + } + + @Override + public Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + Builder builder = builderFunction.apply(name, parserContext); + builder.parse(name, parserContext, node); + return builder; + } + } + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java index 5525a8c551f9d..53021c4c7660f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java @@ -47,7 +47,7 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { public static final String CONTENT_TYPE = "_field_names"; @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(indexVersionCreated).init(this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 938f6bec3007b..259e7bac2b249 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -60,7 +60,7 @@ private static Builder builder(FieldMapper in) { return ((GeoPointFieldMapper)in).builder; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter> ignoreMalformed; final Parameter> ignoreZValue = ignoreZValueParam(m -> builder(m).ignoreZValue.get()); @@ -109,7 +109,7 @@ private static ParsedGeoPoint parseNullValue(Object nullValue, boolean ignoreZVa } @Override - public ParametrizedFieldMapper build(BuilderContext context) { + public FieldMapper build(BuilderContext context) { Parser> geoParser = new PointParser<>(name, ParsedGeoPoint::new, (parser, point) -> { GeoUtils.parseGeoPoint(parser, point, ignoreZValue.get().value()); return point; @@ -138,7 +138,7 @@ public GeoPointFieldMapper(String simpleName, MappedFieldType mappedFieldType, } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), builder.ignoreMalformed.getDefaultValue().value()).init(this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 5c4b1958cad0d..f1afd145f20f4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -61,7 +61,7 @@ private static Builder builder(FieldMapper in) { return ((GeoShapeFieldMapper)in).builder; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter indexed = Parameter.indexParam(m -> builder(m).indexed.get(), true); @@ -130,7 +130,7 @@ public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relat @SuppressWarnings("deprecation") public static Mapper.TypeParser PARSER = (name, node, parserContext) -> { - ParametrizedFieldMapper.Builder builder; + FieldMapper.Builder builder; boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(parserContext.getSettings()); boolean coerceByDefault = COERCE_SETTING.get(parserContext.getSettings()); if (LegacyGeoShapeFieldMapper.containsDeprecatedParameter(node.keySet())) { @@ -165,7 +165,7 @@ public GeoShapeFieldMapper(String simpleName, MappedFieldType mappedFieldType, } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder( simpleName(), builder.ignoreMalformed.getDefaultValue().value(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 2bb555ba04fd6..52af5ca20b722 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -51,7 +51,7 @@ import java.util.function.Supplier; /** A {@link FieldMapper} for ip addresses. */ -public class IpFieldMapper extends ParametrizedFieldMapper { +public class IpFieldMapper extends FieldMapper { private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(IpFieldMapper.class); @@ -61,7 +61,7 @@ private static IpFieldMapper toType(FieldMapper in) { return (IpFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -387,11 +387,6 @@ protected String contentType() { return fieldType().typeName(); } - @Override - protected IpFieldMapper clone() { - return (IpFieldMapper) super.clone(); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { Object addressAsObject; @@ -440,7 +435,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).init(this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 81440ccf9703d..c66fde9ccbf2e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -47,7 +47,7 @@ /** * A field mapper for keywords. This mapper accepts strings and indexes them as-is. */ -public final class KeywordFieldMapper extends ParametrizedFieldMapper { +public final class KeywordFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "keyword"; @@ -74,7 +74,7 @@ private static KeywordFieldMapper toType(FieldMapper in) { return (KeywordFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -335,11 +335,6 @@ public int ignoreAbove() { return ignoreAbove; } - @Override - protected KeywordFieldMapper clone() { - return (KeywordFieldMapper) super.clone(); - } - @Override public KeywordFieldType fieldType() { return (KeywordFieldType) super.fieldType(); @@ -410,7 +405,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexAnalyzers).init(this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index aa4541dd47004..9e1a935b4ab89 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -133,7 +133,7 @@ private static Builder builder(FieldMapper in) { return ((LegacyGeoShapeFieldMapper)in).builder; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { Parameter indexed = Parameter.indexParam(m -> builder(m).indexed.get(), true); @@ -496,7 +496,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexCreatedVersion, builder.ignoreMalformed.getDefaultValue().value(), builder.coerce.getDefaultValue().value()).init(this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index bbba108e2e803..5f70a970aaace 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -31,7 +31,7 @@ /** * A mapper for a builtin field containing metadata about a document. */ -public abstract class MetadataFieldMapper extends ParametrizedFieldMapper { +public abstract class MetadataFieldMapper extends FieldMapper { public interface TypeParser extends Mapper.TypeParser { @@ -112,7 +112,7 @@ public MetadataFieldMapper getDefault(ParserContext parserContext) { } } - public abstract static class Builder extends ParametrizedFieldMapper.Builder { + public abstract static class Builder extends FieldMapper.Builder { protected Builder(String name) { super(name); @@ -136,7 +136,7 @@ protected MetadataFieldMapper(MappedFieldType mappedFieldType) { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return null; // by default, things can't be configured so we have no builder } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index c7e22ae212250..e5516d03a57ee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -64,7 +64,7 @@ import java.util.function.Supplier; /** A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. */ -public class NumberFieldMapper extends ParametrizedFieldMapper { +public class NumberFieldMapper extends FieldMapper { public static final Setting COERCE_SETTING = Setting.boolSetting("index.mapping.coerce", true, Property.IndexScope); @@ -73,7 +73,7 @@ private static NumberFieldMapper toType(FieldMapper in) { return (NumberFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -1047,11 +1047,6 @@ protected String contentType() { return fieldType().type.typeName(); } - @Override - protected NumberFieldMapper clone() { - return (NumberFieldMapper) super.clone(); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { XContentParser parser = context.parser(); @@ -1100,7 +1095,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), type, ignoreMalformedByDefault, coerceByDefault).init(this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java deleted file mode 100644 index f15ffd6945e77..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java +++ /dev/null @@ -1,697 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import org.apache.lucene.document.FieldType; -import org.elasticsearch.Version; -import org.elasticsearch.common.Explicit; -import org.elasticsearch.common.TriFunction; -import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.mapper.Mapper.TypeParser.ParserContext; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.BiFunction; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Supplier; - -/** - * Defines how a particular field should be indexed and searched - * - * Configuration {@link Parameter}s for the mapper are defined on a {@link Builder} subclass, - * and returned by its {@link Builder#getParameters()} method. Merging, serialization - * and parsing of the mapper are all mediated through this set of parameters. - * - * Subclasses should implement a {@link Builder} that is returned from the - * {@link #getMergeBuilder()} method, initialised with the existing builder. - */ -public abstract class ParametrizedFieldMapper extends FieldMapper { - - private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ParametrizedFieldMapper.class); - - /** - * Creates a new ParametrizedFieldMapper - */ - protected ParametrizedFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, new FieldType(), mappedFieldType, multiFields, copyTo); - } - - /** - * Returns a {@link Builder} to be used for merging and serialization - * - * Implement as follows: - * {@code return new MyBuilder(simpleName()).init(this); } - */ - public abstract ParametrizedFieldMapper.Builder getMergeBuilder(); - - @Override - public final ParametrizedFieldMapper merge(Mapper mergeWith) { - - if (mergeWith instanceof FieldMapper == false) { - throw new IllegalArgumentException("mapper [" + name() + "] cannot be changed from type [" - + contentType() + "] to [" + mergeWith.getClass().getSimpleName() + "]"); - } - checkIncomingMergeType((FieldMapper)mergeWith); - - ParametrizedFieldMapper.Builder builder = getMergeBuilder(); - if (builder == null) { - return (ParametrizedFieldMapper) mergeWith; - } - Conflicts conflicts = new Conflicts(name()); - builder.merge((FieldMapper) mergeWith, conflicts); - conflicts.check(); - return builder.build(new BuilderContext(Settings.EMPTY, parentPath(name()))); - } - - protected void checkIncomingMergeType(FieldMapper mergeWith) { - if (Objects.equals(this.getClass(), mergeWith.getClass()) == false) { - throw new IllegalArgumentException("mapper [" + name() + "] cannot be changed from type [" - + contentType() + "] to [" + mergeWith.contentType() + "]"); - } - if (Objects.equals(contentType(), mergeWith.contentType()) == false) { - throw new IllegalArgumentException("mapper [" + name() + "] cannot be changed from type [" - + contentType() + "] to [" + mergeWith.contentType() + "]"); - } - } - - private static ContentPath parentPath(String name) { - int endPos = name.lastIndexOf("."); - if (endPos == -1) { - return new ContentPath(0); - } - return new ContentPath(name.substring(0, endPos)); - } - - @Override - protected final void mergeOptions(FieldMapper other, List conflicts) { - // TODO remove when everything is parametrized - } - - @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - builder.field("type", contentType()); - getMergeBuilder().toXContent(builder, includeDefaults); - multiFields.toXContent(builder, params); - copyTo.toXContent(builder, params); - } - - /** - * Serializes a parameter - */ - protected interface Serializer { - void serialize(XContentBuilder builder, String name, T value) throws IOException; - } - - protected interface MergeValidator { - boolean canMerge(T previous, T current, Conflicts conflicts); - } - - /** - * Check on whether or not a parameter should be serialized - */ - protected interface SerializerCheck { - /** - * Check on whether or not a parameter should be serialized - * @param includeDefaults if defaults have been requested - * @param isConfigured if the parameter has a different value to the default - * @param value the parameter value - * @return {@code true} if the value should be serialized - */ - boolean check(boolean includeDefaults, boolean isConfigured, T value); - } - - /** - * A configurable parameter for a field mapper - * @param the type of the value the parameter holds - */ - public static final class Parameter implements Supplier { - - public final String name; - private final List deprecatedNames = new ArrayList<>(); - private final Supplier defaultValue; - private final TriFunction parser; - private final Function initializer; - private boolean acceptsNull = false; - private Consumer validator = null; - private Serializer serializer = XContentBuilder::field; - private SerializerCheck serializerCheck = (includeDefaults, isConfigured, value) -> includeDefaults || isConfigured; - private Function conflictSerializer = Objects::toString; - private boolean deprecated; - private MergeValidator mergeValidator; - private T value; - private boolean isSet; - - /** - * Creates a new Parameter - * @param name the parameter name, used in parsing and serialization - * @param updateable whether the parameter can be updated with a new value during a mapping update - * @param defaultValue the default value for the parameter, used if unspecified in mappings - * @param parser a function that converts an object to a parameter value - * @param initializer a function that reads a parameter value from an existing mapper - */ - public Parameter(String name, boolean updateable, Supplier defaultValue, - TriFunction parser, Function initializer) { - this.name = name; - this.defaultValue = Objects.requireNonNull(defaultValue); - this.value = null; - this.parser = parser; - this.initializer = initializer; - this.mergeValidator = (previous, toMerge, conflicts) -> updateable || Objects.equals(previous, toMerge); - } - - /** - * Returns the current value of the parameter - */ - public T getValue() { - return isSet ? value : defaultValue.get(); - } - - @Override - public T get() { - return getValue(); - } - - /** - * Returns the default value of the parameter - */ - public T getDefaultValue() { - return defaultValue.get(); - } - - /** - * Sets the current value of the parameter - */ - public void setValue(T value) { - this.isSet = true; - this.value = value; - } - - public boolean isConfigured() { - return isSet && Objects.equals(value, defaultValue.get()) == false; - } - - /** - * Allows the parameter to accept a {@code null} value - */ - public Parameter acceptsNull() { - this.acceptsNull = true; - return this; - } - - /** - * Adds a deprecated parameter name. - * - * If this parameter name is encountered during parsing, a deprecation warning will - * be emitted. The parameter will be serialized with its main name. - */ - public Parameter addDeprecatedName(String deprecatedName) { - this.deprecatedNames.add(deprecatedName); - return this; - } - - /** - * Deprecates the entire parameter. - * - * If this parameter is encountered during parsing, a deprecation warning will - * be emitted. - */ - public Parameter deprecated() { - this.deprecated = true; - return this; - } - - /** - * Adds validation to a parameter, called after parsing and merging - */ - public Parameter setValidator(Consumer validator) { - this.validator = validator; - return this; - } - - /** - * Configure a custom serializer for this parameter - */ - public Parameter setSerializer(Serializer serializer, Function conflictSerializer) { - this.serializer = serializer; - this.conflictSerializer = conflictSerializer; - return this; - } - - /** - * Configure a custom serialization check for this parameter - */ - public Parameter setSerializerCheck(SerializerCheck check) { - this.serializerCheck = check; - return this; - } - - /** - * Always serialize this parameter, no matter its value - */ - public Parameter alwaysSerialize() { - this.serializerCheck = (id, ic, v) -> true; - return this; - } - - /** - * Never serialize this parameter, no matter its value - */ - public Parameter neverSerialize() { - this.serializerCheck = (id, ic, v) -> false; - return this; - } - - /** - * Sets a custom merge validator. By default, merges are accepted if the - * parameter is updateable, or if the previous and new values are equal - */ - public Parameter setMergeValidator(MergeValidator mergeValidator) { - this.mergeValidator = mergeValidator; - return this; - } - - private void validate() { - if (validator != null) { - validator.accept(getValue()); - } - } - - private void init(FieldMapper toInit) { - setValue(initializer.apply(toInit)); - } - - private void parse(String field, ParserContext context, Object in) { - setValue(parser.apply(field, context, in)); - } - - private void merge(FieldMapper toMerge, Conflicts conflicts) { - T value = initializer.apply(toMerge); - T current = getValue(); - if (mergeValidator.canMerge(current, value, conflicts)) { - setValue(value); - } else { - conflicts.addConflict(name, conflictSerializer.apply(current), conflictSerializer.apply(value)); - } - } - - protected void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { - if (serializerCheck.check(includeDefaults, isConfigured(), get())) { - serializer.serialize(builder, name, getValue()); - } - } - - /** - * Defines a parameter that takes the values {@code true} or {@code false} - * @param name the parameter name - * @param updateable whether the parameter can be changed by a mapping update - * @param initializer a function that reads the parameter value from an existing mapper - * @param defaultValue the default value, to be used if the parameter is undefined in a mapping - */ - public static Parameter boolParam(String name, boolean updateable, - Function initializer, boolean defaultValue) { - return new Parameter<>(name, updateable, () -> defaultValue, (n, c, o) -> XContentMapValues.nodeBooleanValue(o), initializer); - } - - /** - * Defines a parameter that takes the values {@code true} or {@code false}, and will always serialize - * its value if configured. - * @param name the parameter name - * @param updateable whether the parameter can be changed by a mapping update - * @param initializer a function that reads the parameter value from an existing mapper - * @param defaultValue the default value, to be used if the parameter is undefined in a mapping - */ - public static Parameter> explicitBoolParam(String name, boolean updateable, - Function> initializer, - boolean defaultValue) { - Explicit defaultExplicit = new Explicit<>(defaultValue, false); - return new Parameter<>(name, updateable, () -> defaultExplicit, - (n, c, o) -> new Explicit<>(XContentMapValues.nodeBooleanValue(o), true), initializer) - .setSerializer((b, n, v) -> b.field(n, v.value()), v -> Boolean.toString(v.value())); - } - - /** - * Defines a parameter that takes a double value - * @param name the parameter name - * @param updateable whether the parameter can be changed by a mapping update - * @param initializer a function that reads the parameter value from an existing mapper - * @param defaultValue the default value, to be used if the parameter is undefined in a mapping - */ - public static Parameter doubleParam(String name, boolean updateable, - Function initializer, double defaultValue) { - return new Parameter<>(name, updateable, () -> defaultValue, (n, c, o) -> XContentMapValues.nodeDoubleValue(o), initializer); - } - - /** - * Defines a parameter that takes a float value - * @param name the parameter name - * @param updateable whether the parameter can be changed by a mapping update - * @param initializer a function that reads the parameter value from an existing mapper - * @param defaultValue the default value, to be used if the parameter is undefined in a mapping - */ - public static Parameter floatParam(String name, boolean updateable, - Function initializer, float defaultValue) { - return new Parameter<>(name, updateable, () -> defaultValue, (n, c, o) -> XContentMapValues.nodeFloatValue(o), initializer); - } - - /** - * Defines a parameter that takes an integer value - * @param name the parameter name - * @param updateable whether the parameter can be changed by a mapping update - * @param initializer a function that reads the parameter value from an existing mapper - * @param defaultValue the default value, to be used if the parameter is undefined in a mapping - */ - public static Parameter intParam(String name, boolean updateable, - Function initializer, int defaultValue) { - return new Parameter<>(name, updateable, () -> defaultValue, (n, c, o) -> XContentMapValues.nodeIntegerValue(o), initializer); - } - - /** - * Defines a parameter that takes a string value - * @param name the parameter name - * @param updateable whether the parameter can be changed by a mapping update - * @param initializer a function that reads the parameter value from an existing mapper - * @param defaultValue the default value, to be used if the parameter is undefined in a mapping - */ - public static Parameter stringParam(String name, boolean updateable, - Function initializer, String defaultValue) { - return new Parameter<>(name, updateable, () -> defaultValue, - (n, c, o) -> XContentMapValues.nodeStringValue(o), initializer); - } - - @SuppressWarnings("unchecked") - public static Parameter> stringArrayParam(String name, boolean updateable, - Function> initializer, List defaultValue) { - return new Parameter<>(name, updateable, () -> defaultValue, - (n, c, o) -> { - List values = (List) o; - List strValues = new ArrayList<>(); - for (Object item : values) { - strValues.add(item.toString()); - } - return strValues; - }, initializer); - } - - /** - * Defines a parameter that takes one of a restricted set of string values - * @param name the parameter name - * @param updateable whether the parameter can be changed by a mapping update - * @param initializer a function that reads the parameter value from an existing mapper - * @param values the set of values that the parameter can take. The first value in the list - * is the default value, to be used if the parameter is undefined in a mapping - */ - public static Parameter restrictedStringParam(String name, boolean updateable, - Function initializer, String... values) { - assert values.length > 0; - Set acceptedValues = new LinkedHashSet<>(Arrays.asList(values)); - return stringParam(name, updateable, initializer, values[0]) - .setValidator(v -> { - if (acceptedValues.contains(v)) { - return; - } - throw new MapperParsingException("Unknown value [" + v + "] for field [" + name + - "] - accepted values are " + acceptedValues.toString()); - }); - } - - /** - * Defines a parameter that takes an analyzer name - * @param name the parameter name - * @param updateable whether the parameter can be changed by a mapping update - * @param initializer a function that reads the parameter value from an existing mapper - * @param defaultAnalyzer the default value, to be used if the parameter is undefined in a mapping - */ - public static Parameter analyzerParam(String name, boolean updateable, - Function initializer, - Supplier defaultAnalyzer) { - return new Parameter<>(name, updateable, defaultAnalyzer, (n, c, o) -> { - String analyzerName = o.toString(); - NamedAnalyzer a = c.getIndexAnalyzers().get(analyzerName); - if (a == null) { - throw new IllegalArgumentException("analyzer [" + analyzerName + "] has not been configured in mappings"); - } - return a; - }, initializer).setSerializer((b, n, v) -> b.field(n, v.name()), NamedAnalyzer::name); - } - - /** - * Declares a metadata parameter - */ - public static Parameter> metaParam() { - return new Parameter<>("meta", true, Collections::emptyMap, - (n, c, o) -> TypeParsers.parseMeta(n, o), m -> m.fieldType().meta()); - } - - public static Parameter indexParam(Function initializer, boolean defaultValue) { - return Parameter.boolParam("index", false, initializer, defaultValue); - } - - public static Parameter storeParam(Function initializer, boolean defaultValue) { - return Parameter.boolParam("store", false, initializer, defaultValue); - } - - public static Parameter docValuesParam(Function initializer, boolean defaultValue) { - return Parameter.boolParam("doc_values", false, initializer, defaultValue); - } - - } - - public static final class Conflicts { - - private final String mapperName; - private final List conflicts = new ArrayList<>(); - - Conflicts(String mapperName) { - this.mapperName = mapperName; - } - - public void addConflict(String parameter, String conflict) { - conflicts.add("Conflict in parameter [" + parameter + "]: " + conflict); - } - - void addConflict(String parameter, String existing, String toMerge) { - conflicts.add("Cannot update parameter [" + parameter + "] from [" + existing + "] to [" + toMerge + "]"); - } - - void check() { - if (conflicts.isEmpty()) { - return; - } - String message = "Mapper for [" + mapperName + "] conflicts with existing mapper:\n\t" - + String.join("\n\t", conflicts); - throw new IllegalArgumentException(message); - } - - } - - /** - * A Builder for a ParametrizedFieldMapper - */ - public abstract static class Builder extends Mapper.Builder { - - protected final MultiFields.Builder multiFieldsBuilder = new MultiFields.Builder(); - protected final CopyTo.Builder copyTo = new CopyTo.Builder(); - - /** - * Creates a new Builder with a field name - */ - protected Builder(String name) { - super(name); - } - - /** - * Initialises all parameters from an existing mapper - */ - public Builder init(FieldMapper initializer) { - for (Parameter param : getParameters()) { - param.init(initializer); - } - for (Mapper subField : initializer.multiFields) { - multiFieldsBuilder.add(subField); - } - return this; - } - - private void merge(FieldMapper in, Conflicts conflicts) { - for (Parameter param : getParameters()) { - param.merge(in, conflicts); - } - for (Mapper newSubField : in.multiFields) { - multiFieldsBuilder.update(newSubField, parentPath(newSubField.name())); - } - this.copyTo.reset(in.copyTo); - validate(); - } - - private void validate() { - for (Parameter param : getParameters()) { - param.validate(); - } - } - - /** - * @return the list of parameters defined for this mapper - */ - protected abstract List> getParameters(); - - @Override - public abstract ParametrizedFieldMapper build(BuilderContext context); - - /** - * Builds the full name of the field, taking into account parent objects - */ - protected String buildFullName(BuilderContext context) { - return context.path().pathAsText(name); - } - - /** - * Writes the current builder parameter values as XContent - */ - protected final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { - for (Parameter parameter : getParameters()) { - parameter.toXContent(builder, includeDefaults); - } - } - - /** - * Parse mapping parameters from a map of mappings - * @param name the field mapper name - * @param parserContext the parser context - * @param fieldNode the root node of the map of mappings for this field - */ - public final void parse(String name, ParserContext parserContext, Map fieldNode) { - Map> paramsMap = new HashMap<>(); - Map> deprecatedParamsMap = new HashMap<>(); - for (Parameter param : getParameters()) { - paramsMap.put(param.name, param); - for (String deprecatedName : param.deprecatedNames) { - deprecatedParamsMap.put(deprecatedName, param); - } - } - String type = (String) fieldNode.remove("type"); - for (Iterator> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - final String propName = entry.getKey(); - final Object propNode = entry.getValue(); - if (Objects.equals("fields", propName)) { - TypeParsers.parseMultiField(multiFieldsBuilder::add, name, parserContext, propName, propNode); - iterator.remove(); - continue; - } - if (Objects.equals("copy_to", propName)) { - TypeParsers.parseCopyFields(propNode).forEach(copyTo::add); - iterator.remove(); - continue; - } - if (Objects.equals("boost", propName)) { - if (parserContext.indexVersionCreated().before(Version.V_8_0_0)) { - deprecationLogger.deprecate( - "boost", - "Parameter [boost] on field [{}] is deprecated and has no effect", - name); - iterator.remove(); - continue; - } else { - throw new MapperParsingException("Unknown parameter [boost] on mapper [" + name + "]"); - } - } - Parameter parameter = deprecatedParamsMap.get(propName); - if (parameter != null) { - deprecationLogger.deprecate(propName, "Parameter [{}] on mapper [{}] is deprecated, use [{}]", - propName, name, parameter.name); - } else { - parameter = paramsMap.get(propName); - } - if (parameter == null) { - if (isDeprecatedParameter(propName, parserContext.indexVersionCreated())) { - deprecationLogger.deprecate(propName, - "Parameter [{}] has no effect on type [{}] and will be removed in future", propName, type); - iterator.remove(); - continue; - } - throw new MapperParsingException("unknown parameter [" + propName - + "] on mapper [" + name + "] of type [" + type + "]"); - } - if (parameter.deprecated) { - deprecationLogger.deprecate(propName, - "Parameter [{}] is deprecated and will be removed in a future version", - propName); - } - if (propNode == null && parameter.acceptsNull == false) { - throw new MapperParsingException("[" + propName + "] on mapper [" + name - + "] of type [" + type + "] must not have a [null] value"); - } - parameter.parse(name, parserContext, propNode); - iterator.remove(); - } - validate(); - } - - // These parameters were previously *always* parsed by TypeParsers#parseField(), even if they - // made no sense; if we've got here, that means that they're not declared on a current mapper, - // and so we emit a deprecation warning rather than failing a previously working mapping. - private static final Set DEPRECATED_PARAMS - = Set.of("store", "meta", "index", "doc_values", "index_options", "similarity"); - - private static boolean isDeprecatedParameter(String propName, Version indexCreatedVersion) { - if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) { - return false; - } - return DEPRECATED_PARAMS.contains(propName); - } - } - - /** - * TypeParser implementation that automatically handles parsing - */ - public static final class TypeParser implements Mapper.TypeParser { - - private final BiFunction builderFunction; - - /** - * Creates a new TypeParser - * @param builderFunction a function that produces a Builder from a name and parsercontext - */ - public TypeParser(BiFunction builderFunction) { - this.builderFunction = builderFunction; - } - - @Override - public Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = builderFunction.apply(name, parserContext); - builder.parse(name, parserContext, node); - return builder; - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 7ac70fe6cb376..2da4da9d5bea3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -61,7 +61,7 @@ import static org.elasticsearch.index.query.RangeQueryBuilder.LT_FIELD; /** A {@link FieldMapper} for indexing numeric and date ranges, and creating queries */ -public class RangeFieldMapper extends ParametrizedFieldMapper { +public class RangeFieldMapper extends FieldMapper { public static final boolean DEFAULT_INCLUDE_UPPER = true; public static final boolean DEFAULT_INCLUDE_LOWER = true; @@ -78,7 +78,7 @@ private static RangeFieldMapper toType(FieldMapper in) { return (RangeFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter index = Parameter.indexParam(m -> toType(m).index, true); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -299,7 +299,7 @@ boolean coerce() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), type, coerceByDefault).init(this); } @@ -313,11 +313,6 @@ protected String contentType() { return fieldType().typeName(); } - @Override - protected RangeFieldMapper clone() { - return (RangeFieldMapper) super.clone(); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { Range range; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java index 0704a7caec682..577d2d1d06035 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java @@ -707,7 +707,7 @@ public abstract Query dvRangeQuery(String field, BinaryDocValuesRangeQuery.Query boolean includeFrom, boolean includeTo); public final Mapper.TypeParser parser() { - return new ParametrizedFieldMapper.TypeParser((n, c) -> new RangeFieldMapper.Builder(n, this, c.getSettings())); + return new FieldMapper.TypeParser((n, c) -> new RangeFieldMapper.Builder(n, this, c.getSettings())); } public final String name; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 54bc2f2230e6a..874eb8ac84a1d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -34,7 +34,7 @@ public class RoutingFieldMapper extends MetadataFieldMapper { public static final String CONTENT_TYPE = "_routing"; @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder().init(this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index a68a6c8ce1635..8609aec8c4857 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -203,7 +203,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder().init(this); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index a3a4795df260a..95ba93088c66c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -87,7 +87,7 @@ import java.util.function.Supplier; /** A {@link FieldMapper} for full-text fields. */ -public class TextFieldMapper extends ParametrizedFieldMapper { +public class TextFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "text"; private static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1; @@ -237,7 +237,7 @@ private static FielddataFrequencyFilter parseFrequencyFilter(String name, Parser return new FielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize); } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Version indexCreatedVersion; @@ -569,8 +569,11 @@ public Query existsQuery(QueryShardContext context) { private static final class PhraseFieldMapper extends FieldMapper { + private final FieldType fieldType; + PhraseFieldMapper(FieldType fieldType, PhraseFieldType mappedFieldType) { - super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); + super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); + this.fieldType = fieldType; } @Override @@ -579,8 +582,8 @@ protected void parseCreateField(ParseContext context) { } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - + public Builder getMergeBuilder() { + return null; } @Override @@ -591,8 +594,11 @@ protected String contentType() { private static final class PrefixFieldMapper extends FieldMapper { + private final FieldType fieldType; + protected PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType) { - super(mappedFieldType.name(), fieldType, mappedFieldType, MultiFields.empty(), CopyTo.empty()); + super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); + this.fieldType = fieldType; } void addField(ParseContext context, String value) { @@ -605,8 +611,8 @@ protected void parseCreateField(ParseContext context) { } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - + public Builder getMergeBuilder() { + return null; } @Override @@ -850,12 +856,7 @@ protected TextFieldMapper(String simpleName, FieldType fieldType, } @Override - protected TextFieldMapper clone() { - return (TextFieldMapper) super.clone(); - } - - @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzer::getDefaultValue).init(this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java b/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java index a22fd0c23e785..bd567e11fac74 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java @@ -23,7 +23,7 @@ import org.apache.lucene.index.IndexOptions; import org.elasticsearch.index.analysis.AnalysisMode; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter; +import org.elasticsearch.index.mapper.FieldMapper.Parameter; import org.elasticsearch.index.similarity.SimilarityProvider; import java.util.Objects; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java index a204557b21392..114f7b4304740 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java @@ -19,17 +19,13 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.index.IndexOptions; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.similarity.SimilarityProvider; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -98,64 +94,7 @@ public static Map parseMeta(String name, Object metaObject) { return Collections.unmodifiableMap(sortedMeta); } - /** - * Parse common field attributes such as {@code doc_values} or {@code store}. - */ - public static void parseField(FieldMapper.Builder builder, String name, Map fieldNode, - Mapper.TypeParser.ParserContext parserContext) { - for (Iterator> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - final String propName = entry.getKey(); - final Object propNode = entry.getValue(); - checkNull(propName, propNode); - if (propName.equals("store")) { - builder.store(XContentMapValues.nodeBooleanValue(propNode, name + ".store")); - iterator.remove(); - } else if (propName.equals("meta")) { - builder.meta(parseMeta(name, propNode)); - iterator.remove(); - } else if (propName.equals("index")) { - builder.index(XContentMapValues.nodeBooleanValue(propNode, name + ".index")); - iterator.remove(); - } else if (propName.equals(DOC_VALUES)) { - builder.docValues(XContentMapValues.nodeBooleanValue(propNode, name + "." + DOC_VALUES)); - iterator.remove(); - } else if (propName.equals("boost")) { - if (parserContext.indexVersionCreated().before(Version.V_8_0_0)) { - deprecationLogger.deprecate( - "boost", - "Parameter [boost] on field [{}] is deprecated and has no effect", - name); - iterator.remove(); - } - else { - throw new MapperParsingException("Unknown parameter [boost] on mapper [" + name + "]"); - } - } else if (propName.equals("index_options")) { - builder.indexOptions(nodeIndexOptionValue(propNode)); - iterator.remove(); - } else if (propName.equals("similarity")) { - deprecationLogger.deprecate("similarity", - "The [similarity] parameter has no effect on field [" + name + "] and will be removed in 8.0"); - iterator.remove(); - } else if (parseMultiField(builder::addMultiField, name, parserContext, propName, propNode)) { - iterator.remove(); - } else if (propName.equals("copy_to")) { - if (parserContext.isWithinMultiField()) { - throw new MapperParsingException("copy_to in multi fields is not allowed. Found the copy_to in field [" + name + "] " + - "which is within a multi field."); - } else { - List copyFields = parseCopyFields(propNode); - FieldMapper.CopyTo.Builder cpBuilder = new FieldMapper.CopyTo.Builder(); - copyFields.forEach(cpBuilder::add); - builder.copyTo(cpBuilder.build()); - } - iterator.remove(); - } - } - } - - @SuppressWarnings({"rawtypes", "unchecked"}) + @SuppressWarnings({"unchecked"}) public static boolean parseMultiField(Consumer multiFieldsBuilder, String name, Mapper.TypeParser.ParserContext parserContext, String propName, Object propNode) { if (propName.equals("fields")) { @@ -223,21 +162,6 @@ public static boolean parseMultiField(Consumer multiFieldsBuilde return false; } - private static IndexOptions nodeIndexOptionValue(final Object propNode) { - final String value = propNode.toString(); - if (INDEX_OPTIONS_OFFSETS.equalsIgnoreCase(value)) { - return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; - } else if (INDEX_OPTIONS_POSITIONS.equalsIgnoreCase(value)) { - return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - } else if (INDEX_OPTIONS_FREQS.equalsIgnoreCase(value)) { - return IndexOptions.DOCS_AND_FREQS; - } else if (INDEX_OPTIONS_DOCS.equalsIgnoreCase(value)) { - return IndexOptions.DOCS; - } else { - throw new ElasticsearchParseException("failed to parse index option [{}]", value); - } - } - public static DateFormatter parseDateTimeFormatter(Object node) { if (node instanceof String) { return DateFormatter.forPattern((String) node); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 999ace6007901..536b6f17d2dbb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -47,7 +47,6 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; @@ -79,7 +78,7 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; import static org.elasticsearch.common.settings.Settings.builder; -import static org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter; +import static org.elasticsearch.index.mapper.FieldMapper.Parameter; import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.containsStringIgnoringCase; @@ -1550,7 +1549,7 @@ protected MetadataTimestampFieldBuilder() { } @Override - protected List> getParameters() { + protected List> getParameters() { return List.of(enabled); } @@ -1589,7 +1588,7 @@ public Query existsQuery(QueryShardContext context) { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new MetadataTimestampFieldBuilder().init(this); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java index 87e95f4741b99..a16105fa05bd2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -30,7 +32,6 @@ import java.io.OutputStream; import java.util.Arrays; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; public class BinaryFieldMapperTests extends MapperTestCase { @@ -90,7 +91,10 @@ public void testDefaultMapping() throws Exception { FieldMapper mapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); assertThat(mapper, instanceOf(BinaryFieldMapper.class)); - assertThat(mapper.fieldType.stored(), equalTo(false)); + + byte[] value = new byte[100]; + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("field", value))); + assertEquals(0, doc.rootDoc().getFields("field").length); } public void testStoredValue() throws IOException { @@ -116,6 +120,9 @@ public void testStoredValue() throws IOException { ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("field", value))); BytesRef indexedValue = doc.rootDoc().getBinaryValue("field"); assertEquals(new BytesRef(value), indexedValue); + IndexableField field = doc.rootDoc().getField("field"); + assertTrue(field.fieldType().stored()); + assertEquals(IndexOptions.NONE, field.fieldType().indexOptions()); MappedFieldType fieldType = mapperService.fieldType("field"); Object originalValue = fieldType.valueForDisplay(indexedValue); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 4c90806e941fe..3ab58de67ed6d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -81,7 +81,7 @@ public String typeName() { } } - static class FakeFieldMapper extends ParametrizedFieldMapper { + static class FakeFieldMapper extends FieldMapper { FakeFieldMapper(FakeFieldType fieldType) { super(fieldType.name(), fieldType, MultiFields.empty(), CopyTo.empty()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index d408f38ee7549..393eea9696339 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -41,7 +41,7 @@ * .point GeoPoint type * .shape GeoShape type */ -public class ExternalMapper extends ParametrizedFieldMapper { +public class ExternalMapper extends FieldMapper { public static class Names { public static final String FIELD_BIN = "bin"; @@ -50,7 +50,7 @@ public static class Names { public static final String FIELD_SHAPE = "shape"; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final BinaryFieldMapper.Builder binBuilder = new BinaryFieldMapper.Builder(Names.FIELD_BIN); private final BooleanFieldMapper.Builder boolBuilder = new BooleanFieldMapper.Builder(Names.FIELD_BOOL); @@ -173,7 +173,7 @@ public Iterator iterator() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), generatedValue, mapperName); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 398e5da9c46b8..307026e3c2434 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -30,7 +30,7 @@ import java.util.List; // Like a String mapper but with very few options. We just use it to test if highlighting on a custom string mapped field works as expected. -public class FakeStringFieldMapper extends ParametrizedFieldMapper { +public class FakeStringFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "fake_string"; @@ -41,7 +41,7 @@ public class FakeStringFieldMapper extends ParametrizedFieldMapper { FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { public Builder(String name) { super(name); @@ -109,7 +109,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 4fb708398058a..65139156141a7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -32,7 +32,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter; +import org.elasticsearch.index.mapper.FieldMapper.Parameter; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; @@ -87,7 +87,7 @@ private static TestMapper toType(Mapper in) { return (TestMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter fixed = Parameter.boolParam("fixed", false, m -> toType(m).fixed, true); @@ -137,7 +137,7 @@ protected List> getParameters() { } @Override - public ParametrizedFieldMapper build(Mapper.BuilderContext context) { + public FieldMapper build(Mapper.BuilderContext context) { return new TestMapper(name(), buildFullName(context), multiFieldsBuilder.build(this, context), copyTo.build(), this); } @@ -153,7 +153,7 @@ public Mapper.Builder parse(String name, Map node, ParserContext } } - public static class TestMapper extends ParametrizedFieldMapper { + public static class TestMapper extends FieldMapper { private final boolean fixed; private final boolean fixed2; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 1bc46afae8a5e..3b14cd2873804 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -30,7 +30,6 @@ import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -567,7 +566,9 @@ public void testIndexPrefixIndexTypes() throws IOException { ); FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix"); assertEquals(prefix.name(), "field._index_prefix"); - assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, prefix.fieldType.indexOptions()); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); + IndexableField field = doc.rootDoc().getField("field._index_prefix"); + assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, field.fieldType().indexOptions()); } { @@ -581,10 +582,11 @@ public void testIndexPrefixIndexTypes() throws IOException { ) ); FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix"); - FieldType ft = prefix.fieldType; + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); + IndexableField field = doc.rootDoc().getField("field._index_prefix"); assertEquals(prefix.name(), "field._index_prefix"); - assertEquals(IndexOptions.DOCS, ft.indexOptions()); - assertFalse(ft.storeTermVectors()); + assertEquals(IndexOptions.DOCS, field.fieldType().indexOptions()); + assertFalse(field.fieldType().storeTermVectors()); } { @@ -598,10 +600,11 @@ public void testIndexPrefixIndexTypes() throws IOException { ) ); FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix"); - FieldType ft = prefix.fieldType; + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); + IndexableField field = doc.rootDoc().getField("field._index_prefix"); assertEquals(prefix.name(), "field._index_prefix"); - assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions()); - assertFalse(ft.storeTermVectors()); + assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, field.fieldType().indexOptions()); + assertFalse(field.fieldType().storeTermVectors()); } { @@ -615,10 +618,11 @@ public void testIndexPrefixIndexTypes() throws IOException { ) ); FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix"); - FieldType ft = prefix.fieldType; + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); + IndexableField field = doc.rootDoc().getField("field._index_prefix"); assertEquals(prefix.name(), "field._index_prefix"); - assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions()); - assertTrue(ft.storeTermVectorOffsets()); + assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, field.fieldType().indexOptions()); + assertTrue(field.fieldType().storeTermVectorOffsets()); } { @@ -632,10 +636,11 @@ public void testIndexPrefixIndexTypes() throws IOException { ) ); FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix"); - FieldType ft = prefix.fieldType; + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); + IndexableField field = doc.rootDoc().getField("field._index_prefix"); assertEquals(prefix.name(), "field._index_prefix"); - assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, ft.indexOptions()); - assertFalse(ft.storeTermVectorOffsets()); + assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, field.fieldType().indexOptions()); + assertFalse(field.fieldType().storeTermVectorOffsets()); } } @@ -655,10 +660,10 @@ public void testNestedIndexPrefixes() throws IOException { assertThat(textField, instanceOf(TextFieldType.class)); MappedFieldType prefix = ((TextFieldType) textField).getPrefixFieldType(); assertEquals(prefix.name(), "object.field._index_prefix"); - FieldMapper mapper - = (FieldMapper) mapperService.documentMapper().mappers().getMapper("object.field._index_prefix"); - assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, mapper.fieldType.indexOptions()); - assertFalse(mapper.fieldType.storeTermVectorOffsets()); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("object.field", "some text"))); + IndexableField field = doc.rootDoc().getField("object.field._index_prefix"); + assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, field.fieldType().indexOptions()); + assertFalse(field.fieldType().storeTermVectorOffsets()); } { @@ -676,10 +681,11 @@ public void testNestedIndexPrefixes() throws IOException { assertThat(textField, instanceOf(TextFieldType.class)); MappedFieldType prefix = ((TextFieldType) textField).getPrefixFieldType(); assertEquals(prefix.name(), "body.with_prefix._index_prefix"); - FieldMapper mapper - = (FieldMapper) mapperService.documentMapper().mappers().getMapper("body.with_prefix._index_prefix"); - assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, mapper.fieldType.indexOptions()); - assertFalse(mapper.fieldType.storeTermVectorOffsets()); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("body", "some text"))); + IndexableField field = doc.rootDoc().getField("body.with_prefix._index_prefix"); + + assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, field.fieldType().indexOptions()); + assertFalse(field.fieldType().storeTermVectorOffsets()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index f43b34eb29af8..886c22bbee22b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -28,7 +28,7 @@ import java.util.List; // this sucks how much must be overridden just do get a dummy field mapper... -public class MockFieldMapper extends ParametrizedFieldMapper { +public class MockFieldMapper extends FieldMapper { static Settings DEFAULT_SETTINGS = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id) .build(); @@ -50,7 +50,7 @@ public MockFieldMapper(String fullName, } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()); } @@ -84,7 +84,7 @@ protected String contentType() { protected void parseCreateField(ParseContext context) { } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final MappedFieldType fieldType; protected Builder(String name) { diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index b9914c3f86abf..12842bcbc2624 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -35,7 +35,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; @@ -58,7 +57,7 @@ /** * Field Mapper for pre-aggregated histograms. */ -public class HistogramFieldMapper extends ParametrizedFieldMapper { +public class HistogramFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "histogram"; public static final ParseField COUNTS_FIELD = new ParseField("counts"); @@ -68,7 +67,7 @@ private static HistogramFieldMapper toType(FieldMapper in) { return (HistogramFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); private final Parameter> ignoreMalformed; @@ -114,7 +113,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault).init(this); } diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java index 096ef9cb5b885..8cfb89c68e658 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/mapper/DataStreamTimestampFieldMapper.java @@ -20,7 +20,6 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; @@ -108,7 +107,7 @@ private DataStreamTimestampFieldMapper(MappedFieldType mappedFieldType, boolean } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder().init(this); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 54b8418441829..54246b692c968 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -32,7 +32,6 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; @@ -50,7 +49,7 @@ /** * A {@link FieldMapper} that assigns every document the same value. */ -public class ConstantKeywordFieldMapper extends ParametrizedFieldMapper { +public class ConstantKeywordFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "constant_keyword"; @@ -59,11 +58,11 @@ private static ConstantKeywordFieldMapper toType(FieldMapper in) { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { // This is defined as updateable because it can be updated once, from [null] to any value, // by a dynamic mapping update. Once it has been set, however, the value cannot be changed. @@ -226,11 +225,6 @@ public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxD super(simpleName, mappedFieldType, MultiFields.empty(), CopyTo.empty()); } - @Override - protected ConstantKeywordFieldMapper clone() { - return (ConstantKeywordFieldMapper) super.clone(); - } - @Override public ConstantKeywordFieldType fieldType() { return (ConstantKeywordFieldType) super.fieldType(); diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java index 2a365eb307812..0fef5c266ecdc 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java @@ -29,10 +29,10 @@ import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.DynamicKeyFieldMapper; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; @@ -94,7 +94,7 @@ private static Builder builder(Mapper in) { return ((FlattenedFieldMapper)in).builder; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter depthLimit = Parameter.intParam("depth_limit", true, m -> builder(m).depthLimit.get(), Defaults.DEPTH_LIMIT) @@ -426,11 +426,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - protected FlattenedFieldMapper clone() { - return (FlattenedFieldMapper) super.clone(); - } - int depthLimit() { return builder.depthLimit.get(); } @@ -473,7 +468,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 6e8a219a2befa..50a19669f42f5 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -27,7 +27,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SimpleMappedFieldType; import org.elasticsearch.index.mapper.SourceValueFetcher; @@ -51,7 +50,7 @@ import static org.elasticsearch.xpack.unsignedlong.UnsignedLongLeafFieldData.convertUnsignedLongToDouble; -public class UnsignedLongFieldMapper extends ParametrizedFieldMapper { +public class UnsignedLongFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "unsigned_long"; private static final long MASK_2_63 = 0x8000000000000000L; @@ -62,7 +61,7 @@ private static UnsignedLongFieldMapper toType(FieldMapper in) { return (UnsignedLongFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); @@ -433,11 +432,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - protected UnsignedLongFieldMapper clone() { - return (UnsignedLongFieldMapper) super.clone(); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { XContentParser parser = context.parser(); @@ -492,7 +486,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), ignoreMalformedByDefault).init(this); } diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 798b1c9fd2785..4dcf2de16ce11 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -37,7 +37,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TermBasedFieldType; @@ -66,7 +65,7 @@ /** * A {@link FieldMapper} for indexing fields with version strings. */ -public class VersionStringFieldMapper extends ParametrizedFieldMapper { +public class VersionStringFieldMapper extends FieldMapper { private static final byte[] MIN_VALUE = new byte[16]; private static final byte[] MAX_VALUE = new byte[16]; @@ -88,7 +87,7 @@ public static class Defaults { } } - static class Builder extends ParametrizedFieldMapper.Builder { + static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); @@ -325,11 +324,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - protected VersionStringFieldMapper clone() { - return (VersionStringFieldMapper) super.clone(); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { String versionString; @@ -389,7 +383,7 @@ public String toString() { }; @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java index 630ca4df72f5e..75d832eb9ffbc 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeFieldMapper.java @@ -16,7 +16,6 @@ import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -27,7 +26,7 @@ import java.util.Map; import java.util.function.BiFunction; -public final class RuntimeFieldMapper extends ParametrizedFieldMapper { +public final class RuntimeFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "runtime"; @@ -58,7 +57,7 @@ protected RuntimeFieldMapper( } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new RuntimeFieldMapper.Builder(simpleName(), scriptCompiler).init(this); } @@ -72,7 +71,7 @@ protected String contentType() { return CONTENT_TYPE; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { static final Map>> FIELD_TYPE_RESOLVER = Map.of( BooleanFieldMapper.CONTENT_TYPE, diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index dab381eb963a4..09fee8dfff16f 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -26,7 +26,6 @@ import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor; @@ -69,7 +68,7 @@ private static Builder builder(FieldMapper in) { return ((GeoShapeWithDocValuesFieldMapper)in).builder; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter indexed = Parameter.indexParam(m -> builder(m).indexed.get(), true); final Parameter hasDocValues; @@ -169,7 +168,7 @@ public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relat @SuppressWarnings("deprecation") public static Mapper.TypeParser PARSER = (name, node, parserContext) -> { - ParametrizedFieldMapper.Builder builder; + FieldMapper.Builder builder; boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(parserContext.getSettings()); boolean coerceByDefault = COERCE_SETTING.get(parserContext.getSettings()); if (LegacyGeoShapeFieldMapper.containsDeprecatedParameter(node.keySet())) { @@ -206,7 +205,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder( simpleName(), builder.version, diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java index 6c25c0139e418..743f4a145e70c 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapper.java @@ -17,7 +17,6 @@ import org.elasticsearch.index.mapper.AbstractPointGeometryFieldMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.xpack.spatial.common.CartesianPoint; @@ -52,7 +51,7 @@ private static Builder builder(FieldMapper in) { return ((PointFieldMapper)in).builder; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter indexed = Parameter.indexParam(m -> builder(m).indexed.get(), true); final Parameter hasDocValues = Parameter.docValuesParam(m -> builder(m).hasDocValues.get(), true); @@ -92,7 +91,7 @@ private static ParsedPoint parseNullValue(Object nullValue, boolean ignoreZValue } @Override - public ParametrizedFieldMapper build(BuilderContext context) { + public FieldMapper build(BuilderContext context) { CartesianPointParser parser = new CartesianPointParser(name, nullValue.get(), ignoreZValue.get().value(), ignoreMalformed.get().value()); PointFieldType ft @@ -146,7 +145,7 @@ public PointFieldType fieldType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), builder.ignoreMalformed.getDefaultValue().value()).init(this); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index 78a1a5732f6d0..8e14aef969c66 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -17,7 +17,6 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.GeoShapeParser; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.xpack.spatial.index.query.ShapeQueryProcessor; @@ -49,7 +48,7 @@ private static Builder builder(FieldMapper in) { return ((ShapeFieldMapper)in).builder; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter indexed = Parameter.indexParam(m -> builder(m).indexed.get(), true); @@ -143,7 +142,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), builder.ignoreMalformed.getDefaultValue().value(), builder.coerce.getDefaultValue().value()) .init(this); } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index 32ce1788ac1aa..fedc2ed8051bd 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.vectors.mapper; import org.apache.lucene.document.BinaryDocValuesField; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; @@ -20,7 +19,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; @@ -42,7 +40,7 @@ /** * A {@link FieldMapper} for indexing a dense vector of floats. */ -public class DenseVectorFieldMapper extends ParametrizedFieldMapper { +public class DenseVectorFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "dense_vector"; public static short MAX_DIMS_COUNT = 2048; //maximum allowed number of dimensions @@ -52,7 +50,7 @@ private static DenseVectorFieldMapper toType(FieldMapper in) { return (DenseVectorFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { Parameter dims = new Parameter<>("dims", false, () -> null, (n, c, o) -> XContentMapValues.nodeIntegerValue(o), m -> toType(m).dims) @@ -152,16 +150,10 @@ public Query termQuery(Object value, QueryShardContext context) { private DenseVectorFieldMapper(String simpleName, MappedFieldType mappedFieldType, int dims, Version indexCreatedVersion, MultiFields multiFields, CopyTo copyTo) { super(simpleName, mappedFieldType, multiFields, copyTo); - assert fieldType.indexOptions() == IndexOptions.NONE; this.indexCreatedVersion = indexCreatedVersion; this.dims = dims; } - @Override - protected DenseVectorFieldMapper clone() { - return (DenseVectorFieldMapper) super.clone(); - } - @Override public DenseVectorFieldType fieldType() { return (DenseVectorFieldType) super.fieldType(); @@ -238,7 +230,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexCreatedVersion).init(this); } } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 30df174a3dd5d..65813d84d8aa9 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -13,7 +13,6 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; @@ -34,14 +33,14 @@ * TODO: remove in 9.0. */ @Deprecated -public class SparseVectorFieldMapper extends ParametrizedFieldMapper { +public class SparseVectorFieldMapper extends FieldMapper { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(SparseVectorFieldMapper.class); static final String ERROR_MESSAGE = "The [sparse_vector] field type is no longer supported."; static final String ERROR_MESSAGE_7X = "The [sparse_vector] field type is no longer supported. Old 7.x indices are allowed to " + "contain [sparse_vector] fields, but they cannot be indexed or searched."; public static final String CONTENT_TYPE = "sparse_vector"; - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter> meta = Parameter.metaParam(); @@ -109,11 +108,6 @@ private SparseVectorFieldMapper(String simpleName, MappedFieldType mappedFieldTy super(simpleName, mappedFieldType, multiFields, copyTo); } - @Override - protected SparseVectorFieldMapper clone() { - return (SparseVectorFieldMapper) super.clone(); - } - @Override public SparseVectorFieldType fieldType() { return (SparseVectorFieldType) super.fieldType(); @@ -135,7 +129,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 57b2d0ff58b2d..55ecf1fdd59d1 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -57,7 +57,6 @@ import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.SourceValueFetcher; @@ -82,7 +81,7 @@ /** * A {@link FieldMapper} for indexing fields with ngrams for efficient wildcard matching */ -public class WildcardFieldMapper extends ParametrizedFieldMapper { +public class WildcardFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "wildcard"; public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10; @@ -188,7 +187,7 @@ private static WildcardFieldMapper toType(FieldMapper in) { return (WildcardFieldMapper) in; } - public static class Builder extends ParametrizedFieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { final Parameter ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Defaults.IGNORE_ABOVE) @@ -950,11 +949,6 @@ int ignoreAbove() { return ignoreAbove; } - @Override - protected WildcardFieldMapper clone() { - return (WildcardFieldMapper) super.clone(); - } - @Override public WildcardFieldType fieldType() { return (WildcardFieldType) super.fieldType(); @@ -1008,7 +1002,7 @@ protected String contentType() { } @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { + public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexVersionCreated).init(this); } } From 639126e07d876bddb7008c98c2cb6617f9c9edca Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 29 Oct 2020 21:30:14 +0000 Subject: [PATCH 12/19] SAYT shouldn't try to add fields if index=false and store=false --- .../mapper/SearchAsYouTypeFieldMapper.java | 12 ++++++++--- .../SearchAsYouTypeFieldMapperTests.java | 20 +++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index ba09983317b4c..a915b49478f18 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -578,11 +578,17 @@ protected void parseCreateField(ParseContext context) throws IOException { return; } + if (this.index == false && this.store == false) { + return; + } + context.doc().add(new Field(fieldType().name(), value, fieldType().fieldType)); - for (ShingleFieldMapper subFieldMapper : shingleFields) { - context.doc().add(new Field(subFieldMapper.fieldType().name(), value, subFieldMapper.getLuceneFieldType())); + if (this.index) { + for (ShingleFieldMapper subFieldMapper : shingleFields) { + context.doc().add(new Field(subFieldMapper.fieldType().name(), value, subFieldMapper.getLuceneFieldType())); + } + context.doc().add(new Field(prefixField.fieldType().name(), value, prefixField.getLuceneFieldType())); } - context.doc().add(new Field(prefixField.fieldType().name(), value, prefixField.getLuceneFieldType())); if (fieldType().fieldType.omitNorms()) { createFieldNamesField(context); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 3c42d0277181d..156e09c005343 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -307,13 +307,21 @@ public void testStore() throws IOException { public void testIndex() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type").field("index", false))); ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); + assertNull(doc.rootDoc().getField("field")); + } - Stream.of( - fieldType(doc, "field"), - fieldType(doc, "field._index_prefix"), - fieldType(doc, "field._2gram"), - fieldType(doc, "field._3gram") - ).forEach(ft -> assertThat(ft.indexOptions(), equalTo(IndexOptions.NONE))); + public void testStoredOnly() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { + b.field("type", "search_as_you_type"); + b.field("index", false); + b.field("store", true); + })); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); + assertTrue(fieldType(doc, "field").stored()); + assertThat(fieldType(doc, "field").indexOptions(), equalTo(IndexOptions.NONE)); + assertNull(doc.rootDoc().getField("field._index_prefix")); + assertNull(doc.rootDoc().getField("field._2gram")); + assertNull(doc.rootDoc().getField("field._3gram")); } public void testTermVectors() throws IOException { From 88ad7e5dcd9d94f28da59d0831b2e5850211b99a Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Nov 2020 15:27:05 +0000 Subject: [PATCH 13/19] Make indexAnalyzers passed via constructor --- .../index/mapper/RankFeatureFieldMapper.java | 9 +-- .../index/mapper/RankFeaturesFieldMapper.java | 8 +-- .../mapper/SearchAsYouTypeFieldMapper.java | 39 +++++------- .../join/mapper/ParentIdFieldMapper.java | 9 +-- .../join/mapper/ParentJoinFieldMapper.java | 8 +-- .../ICUCollationKeywordFieldMapper.java | 8 +-- .../AnnotatedTextFieldMapper.java | 10 +--- .../mapper/AbstractGeometryFieldMapper.java | 11 +++- .../AbstractShapeGeometryFieldMapper.java | 14 ++++- .../index/mapper/BooleanFieldMapper.java | 8 +-- .../index/mapper/CompletionFieldMapper.java | 15 ++--- .../index/mapper/DynamicKeyFieldMapper.java | 5 +- .../index/mapper/FieldMapper.java | 60 +++++++++++++------ .../index/mapper/IdFieldMapper.java | 9 +-- .../index/mapper/KeywordFieldMapper.java | 9 +-- .../mapper/LegacyGeoShapeFieldMapper.java | 8 +-- .../index/mapper/MetadataFieldMapper.java | 5 ++ .../index/mapper/RoutingFieldMapper.java | 9 +-- .../index/mapper/TextFieldMapper.java | 45 +++++++------- .../MetadataRolloverServiceTests.java | 12 ++-- .../mapper/DocumentFieldMapperTests.java | 12 +--- .../index/mapper/FakeStringFieldMapper.java | 9 +-- .../mapper/FlattenedFieldMapper.java | 9 +-- .../VersionStringFieldMapper.java | 9 +-- .../mapper/DenseVectorFieldMapper.java | 10 ---- .../wildcard/mapper/WildcardFieldMapper.java | 10 +--- 26 files changed, 146 insertions(+), 214 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 94b69330ba8c8..53d62eebce6cd 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -27,14 +27,12 @@ import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentParser.Token; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -137,7 +135,7 @@ public Query termQuery(Object value, QueryShardContext context) { private RankFeatureFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, boolean positiveScoreImpact) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); this.positiveScoreImpact = positiveScoreImpact; } @@ -184,11 +182,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); - } - public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index aedc5eb7c441d..69ca73043f7a0 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -23,7 +23,6 @@ import org.apache.lucene.search.Query; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentParser.Token; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SearchLookup; @@ -99,7 +98,7 @@ public Query termQuery(Object value, QueryShardContext context) { private RankFeaturesFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); } @Override @@ -154,9 +153,4 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); - } - } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index 429a6c4cbb956..fa42fcf360b08 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -57,6 +57,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -155,12 +156,16 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { fieldType.setStored(store.getValue()); TextParams.setTermVectorParams(termVectors.getValue(), fieldType); + Map indexAnalyzers = new HashMap<>(); + NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer(); NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer(); SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, similarity.getValue(), analyzers.getSearchAnalyzer(), analyzers.getSearchQuoteAnalyzer(), meta.getValue()); + indexAnalyzers.put(ft.name(), indexAnalyzer); + // set up the prefix field FieldType prefixft = new FieldType(fieldType); prefixft.setStoreTermVectors(false); @@ -179,7 +184,8 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { = new PrefixFieldType(fullName, prefixSearchInfo, Defaults.MIN_GRAM, Defaults.MAX_GRAM); final NamedAnalyzer prefixAnalyzer = new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, prefixIndexWrapper); - final PrefixFieldMapper prefixFieldMapper = new PrefixFieldMapper(prefixft, prefixFieldType, prefixAnalyzer); + final PrefixFieldMapper prefixFieldMapper = new PrefixFieldMapper(prefixft, prefixFieldType); + indexAnalyzers.put(prefixFieldType.name(), prefixAnalyzer); // set up the shingle fields final ShingleFieldMapper[] shingleFieldMappers = new ShingleFieldMapper[maxShingleSize.getValue() - 1]; @@ -203,11 +209,12 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { shingleFieldTypes[i] = shingleFieldType; NamedAnalyzer shingleAnalyzer = new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, shingleIndexWrapper); - shingleFieldMappers[i] = new ShingleFieldMapper(shingleft, shingleFieldType, shingleAnalyzer); + shingleFieldMappers[i] = new ShingleFieldMapper(shingleft, shingleFieldType); + indexAnalyzers.put(shingleFieldType.name(), shingleAnalyzer); } ft.setPrefixField(prefixFieldType); ft.setShingleFields(shingleFieldTypes); - return new SearchAsYouTypeFieldMapper(name, ft, copyTo.build(), prefixFieldMapper, shingleFieldMappers, this); + return new SearchAsYouTypeFieldMapper(name, ft, copyTo.build(), indexAnalyzers, prefixFieldMapper, shingleFieldMappers, this); } } @@ -396,12 +403,10 @@ public Query existsQuery(QueryShardContext context) { static final class PrefixFieldMapper extends FieldMapper { - final NamedAnalyzer analyzer; final FieldType fieldType; - PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType, NamedAnalyzer analyzer) { + PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType) { super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); - this.analyzer = analyzer; this.fieldType = fieldType; } @@ -424,11 +429,6 @@ public Builder getMergeBuilder() { return null; } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), analyzer); - } - @Override protected String contentType() { return "prefix"; @@ -442,13 +442,11 @@ public String toString() { static final class ShingleFieldMapper extends FieldMapper { - final NamedAnalyzer analyzer; private final FieldType fieldType; - ShingleFieldMapper(FieldType fieldType, ShingleFieldType mappedFieldtype, NamedAnalyzer analyzer) { + ShingleFieldMapper(FieldType fieldType, ShingleFieldType mappedFieldtype) { super(mappedFieldtype.name(), mappedFieldtype, MultiFields.empty(), CopyTo.empty()); - this.analyzer = analyzer; this.fieldType = fieldType; } @@ -461,11 +459,6 @@ public ShingleFieldType fieldType() { return (ShingleFieldType) super.fieldType(); } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), analyzer); - } - @Override protected void parseCreateField(ParseContext context) { throw new UnsupportedOperationException(); @@ -567,10 +560,11 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew public SearchAsYouTypeFieldMapper(String simpleName, SearchAsYouTypeFieldType mappedFieldType, CopyTo copyTo, + Map indexAnalyzers, PrefixFieldMapper prefixField, ShingleFieldMapper[] shingleFields, Builder builder) { - super(simpleName, mappedFieldType, MultiFields.empty(), copyTo); + super(simpleName, mappedFieldType, indexAnalyzers, MultiFields.empty(), copyTo); this.prefixField = prefixField; this.shingleFields = shingleFields; this.maxShingleSize = builder.maxShingleSize.getValue(); @@ -606,11 +600,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), analyzer); - } - public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), builder.analyzers.indexAnalyzer::getDefaultValue).init(this); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 6807a1ab2e000..9b9007769ab89 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.FieldMapper; @@ -38,7 +37,6 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; -import java.util.Map; import java.util.function.Supplier; /** @@ -92,7 +90,7 @@ public Object valueForDisplay(Object value) { } protected ParentIdFieldMapper(String name, boolean eagerGlobalOrdinals) { - super(name, new ParentIdFieldType(name, eagerGlobalOrdinals), MultiFields.empty(), CopyTo.empty()); + super(name, new ParentIdFieldType(name, eagerGlobalOrdinals), Lucene.KEYWORD_ANALYZER, MultiFields.empty(), CopyTo.empty()); } @Override @@ -112,11 +110,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); - } - @Override public Builder getMergeBuilder() { return null; // always constructed by ParentJoinFieldMapper, not through type parsers diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 7e2f2c64a14e6..6b2dea419d7d9 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.ContentPath; @@ -194,7 +193,7 @@ protected ParentJoinFieldMapper(String simpleName, MetaJoinFieldMapper uniqueFieldMapper, Map parentIdFields, boolean eagerGlobalOrdinals, List relations) { - super(simpleName, mappedFieldType, MultiFields.empty(), CopyTo.empty()); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, MultiFields.empty(), CopyTo.empty()); this.parentIdFields = parentIdFields; this.uniqueFieldMapper = uniqueFieldMapper; this.eagerGlobalOrdinals = eagerGlobalOrdinals; @@ -206,11 +205,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); - } - public JoinFieldType fieldType() { return (JoinFieldType) super.fieldType(); } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index bf7ccd1a331f0..d107107ca1120 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -36,7 +36,6 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.IndexableBinaryStringTools; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; @@ -418,7 +417,7 @@ protected ICUCollationKeywordFieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Collator collator, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); assert collator.isFrozen(); this.fieldType = fieldType; this.params = builder.collatorParams(); @@ -435,11 +434,6 @@ public CollationFieldType fieldType() { return (CollationFieldType) super.fieldType(); } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 3ce47d8179f0f..15e400ee7c2ef 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -49,7 +49,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -507,21 +506,14 @@ public String typeName() { } private final FieldType fieldType; - private final NamedAnalyzer analyzer; private final Builder builder; protected AnnotatedTextFieldMapper(String simpleName, FieldType fieldType, AnnotatedTextFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, wrapAnalyzer(builder.analyzers.getIndexAnalyzer()), multiFields, copyTo); assert fieldType.tokenized(); this.fieldType = fieldType; this.builder = builder; - this.analyzer = wrapAnalyzer(builder.analyzers.getIndexAnalyzer()); - } - - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), analyzer); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 5a7d59d2879ac..a00b4bd5d80fd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.support.MapXContentParser; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SearchLookup; @@ -151,16 +152,24 @@ protected Object parseSourceValue(Object value) { private final Parser parser; protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, + Map indexAnalyzers, Explicit ignoreMalformed, Explicit ignoreZValue, MultiFields multiFields, CopyTo copyTo, Indexer indexer, Parser parser) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo); this.ignoreMalformed = ignoreMalformed; this.ignoreZValue = ignoreZValue; this.indexer = indexer; this.parser = parser; } + protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, + Explicit ignoreMalformed, Explicit ignoreZValue, + MultiFields multiFields, CopyTo copyTo, + Indexer indexer, Parser parser) { + this(simpleName, mappedFieldType, Collections.emptyMap(), ignoreMalformed, ignoreZValue, multiFields, copyTo, indexer, parser); + } + @Override public AbstractGeometryFieldType fieldType() { return (AbstractGeometryFieldType) mappedFieldType; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java index fad7e7e7ce824..fd127919d8966 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java @@ -21,7 +21,9 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import java.util.Collections; import java.util.Map; import java.util.function.Function; @@ -61,15 +63,25 @@ protected AbstractShapeGeometryFieldType(String name, boolean isSearchable, bool protected Explicit orientation; protected AbstractShapeGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, + Map indexAnalyzers, Explicit ignoreMalformed, Explicit coerce, Explicit ignoreZValue, Explicit orientation, MultiFields multiFields, CopyTo copyTo, Indexer indexer, Parser parser) { - super(simpleName, mappedFieldType, ignoreMalformed, ignoreZValue, multiFields, copyTo, indexer, parser); + super(simpleName, mappedFieldType, indexAnalyzers, ignoreMalformed, ignoreZValue, multiFields, copyTo, indexer, parser); this.coerce = coerce; this.orientation = orientation; } + protected AbstractShapeGeometryFieldMapper(String simpleName, MappedFieldType mappedFieldType, + Explicit ignoreMalformed, Explicit coerce, + Explicit ignoreZValue, Explicit orientation, + MultiFields multiFields, CopyTo copyTo, + Indexer indexer, Parser parser) { + this(simpleName, mappedFieldType, Collections.emptyMap(), + ignoreMalformed, coerce, ignoreZValue, orientation, multiFields, copyTo, indexer, parser); + } + @Override public final boolean parsesArrayValue() { return false; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index e0845beb5c0ed..207c071b27f89 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -222,7 +221,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower protected BooleanFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); this.nullValue = builder.nullValue.getValue(); this.stored = builder.stored.getValue(); this.indexed = builder.indexed.getValue(); @@ -268,11 +267,6 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); - } - public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName()).init(this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 17e1ff8c25a8d..a6879a8d1b54b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -174,6 +174,11 @@ protected List> getParameters() { return List.of(analyzer, searchAnalyzer, preserveSeparators, preservePosInc, maxInputLength, contexts, meta); } + NamedAnalyzer buildAnalyzer() { + return new NamedAnalyzer(analyzer.get().name(), AnalyzerScope.INDEX, + new CompletionAnalyzer(analyzer.get(), preserveSeparators.get(), preservePosInc.get())); + } + @Override public CompletionFieldMapper build(BuilderContext context) { checkCompletionContextsLimit(context); @@ -303,15 +308,12 @@ protected List parseSourceValue(Object value) { private final int maxInputLength; private final Builder builder; - private final NamedAnalyzer analyzer; public CompletionFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, builder.buildAnalyzer(), multiFields, copyTo); this.builder = builder; this.maxInputLength = builder.maxInputLength.getValue(); - this.analyzer = new NamedAnalyzer(builder.analyzer.get().name(), AnalyzerScope.INDEX, - new CompletionAnalyzer(builder.analyzer.get(), builder.preserveSeparators.get(), builder.preservePosInc.get())); } @Override @@ -324,11 +326,6 @@ public boolean parsesArrayValue() { return true; } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), analyzer); - } - int getMaxInputLength() { return builder.maxInputLength.get(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java index 2811d366a13b1..2463cc0d48211 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.index.analysis.NamedAnalyzer; + /** * A field mapper that supports lookup of dynamic sub-keys. If the field mapper is named 'my_field', * then a user is able to search on the field in both of the following ways: @@ -41,8 +43,9 @@ public abstract class DynamicKeyFieldMapper extends FieldMapper { public DynamicKeyFieldMapper(String simpleName, MappedFieldType defaultFieldType, + NamedAnalyzer indexAnalyzer, CopyTo copyTo) { - super(simpleName, defaultFieldType, MultiFields.empty(), copyTo); + super(simpleName, defaultFieldType, indexAnalyzer, MultiFields.empty(), copyTo); } public abstract MappedFieldType keyedFieldType(String key); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 3ba02856e6c2c..36f45268880b8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -64,17 +64,55 @@ public abstract class FieldMapper extends Mapper implements Cloneable { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(FieldMapper.class); - protected MappedFieldType mappedFieldType; - protected MultiFields multiFields; - protected CopyTo copyTo; + protected final MappedFieldType mappedFieldType; + protected final Map indexAnalyzers; + protected final MultiFields multiFields; + protected final CopyTo copyTo; + /** + * Create a FieldMapper with no index analyzers + * @param simpleName the leaf name of the mapper + * @param mappedFieldType the MappedFieldType associated with this mapper + * @param multiFields sub fields of this mapper + * @param copyTo copyTo fields of this mapper + */ protected FieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { + this(simpleName, mappedFieldType, Collections.emptyMap(), multiFields, copyTo); + } + + /** + * Create a FieldMapper with a single associated index analyzer + * @param simpleName the leaf name of the mapper + * @param mappedFieldType the MappedFieldType associated with this mapper + * @param indexAnalyzer the index-time analyzer to use for this field + * @param multiFields sub fields of this mapper + * @param copyTo copyTo fields of this mapper + */ + protected FieldMapper(String simpleName, MappedFieldType mappedFieldType, + NamedAnalyzer indexAnalyzer, + MultiFields multiFields, CopyTo copyTo) { + this(simpleName, mappedFieldType, Collections.singletonMap(mappedFieldType.name(), indexAnalyzer), multiFields, copyTo); + } + + /** + * Create a FieldMapper that indexes into multiple analyzed fields + * @param simpleName the leaf name of the mapper + * @param mappedFieldType the MappedFieldType associated with this mapper + * @param indexAnalyzers a map of field names to analyzers, one for each analyzed field + * the mapper will add + * @param multiFields sub fields of this mapper + * @param copyTo copyTo fields of this mapper + */ + protected FieldMapper(String simpleName, MappedFieldType mappedFieldType, + Map indexAnalyzers, + MultiFields multiFields, CopyTo copyTo) { super(simpleName); if (mappedFieldType.name().isEmpty()) { throw new IllegalArgumentException("name cannot be empty string"); } this.mappedFieldType = mappedFieldType; + this.indexAnalyzers = indexAnalyzers; this.multiFields = multiFields; this.copyTo = Objects.requireNonNull(copyTo); } @@ -263,18 +301,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.endObject(); } - protected boolean indexedByDefault() { - return true; - } - - protected boolean docValuesByDefault() { - return true; - } - - protected boolean storedByDefault() { - return false; - } - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { builder.field("type", contentType()); getMergeBuilder().toXContent(builder, includeDefaults); @@ -284,8 +310,8 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, protected abstract String contentType(); - public Map indexAnalyzers() { - return Collections.emptyMap(); + public final Map indexAnalyzers() { + return indexAnalyzers; } public static class MultiFields implements Iterable { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 33b253b8dfac6..a91fcbbbf18e2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -55,7 +54,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.function.BooleanSupplier; import java.util.function.Supplier; @@ -211,11 +209,6 @@ public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, Mu } } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); - } - private static LeafFieldData wrap(LeafFieldData in) { return new LeafFieldData() { @@ -265,7 +258,7 @@ public boolean advanceExact(int doc) throws IOException { } private IdFieldMapper(BooleanSupplier fieldDataEnabled) { - super(new IdFieldType(fieldDataEnabled)); + super(new IdFieldType(fieldDataEnabled), Lucene.KEYWORD_ANALYZER); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 4ad31a63c6052..47bd3329d93f4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -314,9 +314,9 @@ public CollapseType collapseType() { private final IndexAnalyzers indexAnalyzers; - protected KeywordFieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, + protected KeywordFieldMapper(String simpleName, FieldType fieldType, KeywordFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, mappedFieldType.normalizer, multiFields, copyTo); assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0; this.indexed = builder.indexed.getValue(); this.hasDocValues = builder.hasDocValues.getValue(); @@ -343,11 +343,6 @@ public KeywordFieldType fieldType() { return (KeywordFieldType) super.fieldType(); } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), fieldType().normalizer); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { String value; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index dbcf12ba70b80..d4b4b71580f73 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -43,7 +43,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.LegacyGeoShapeQueryProcessor; import org.elasticsearch.index.query.QueryShardContext; import org.locationtech.spatial4j.shape.Shape; @@ -461,7 +460,7 @@ public LegacyGeoShapeFieldMapper(String simpleName, MappedFieldType mappedFieldT MultiFields multiFields, CopyTo copyTo, LegacyGeoShapeIndexer indexer, LegacyGeoShapeParser parser, Builder builder) { - super(simpleName, mappedFieldType, + super(simpleName, mappedFieldType, Collections.singletonMap(mappedFieldType.name(), Lucene.KEYWORD_ANALYZER), builder.ignoreMalformed.get(), builder.coerce.get(), builder.ignoreZValue.get(), builder.orientation.get(), multiFields, copyTo, indexer, parser); this.indexCreatedVersion = builder.indexCreatedVersion; @@ -477,11 +476,6 @@ String strategy() { return fieldType().strategy().getStrategyName(); } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); - } - @Override protected void addStoredFields(ParseContext context, Shape geometry) { // noop: we do not store geo_shapes; and will not store legacy geo_shape types diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 5f70a970aaace..fc76756206188 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.analysis.NamedAnalyzer; import java.io.IOException; import java.util.Map; @@ -135,6 +136,10 @@ protected MetadataFieldMapper(MappedFieldType mappedFieldType) { super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); } + protected MetadataFieldMapper(MappedFieldType mappedFieldType, NamedAnalyzer indexAnalyzer) { + super(mappedFieldType.name(), mappedFieldType, indexAnalyzer, MultiFields.empty(), CopyTo.empty()); + } + @Override public FieldMapper.Builder getMergeBuilder() { return null; // by default, things can't be configured so we have no builder diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 4dc88acd0c31f..c6367081ac8bd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -23,12 +23,10 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.search.lookup.SearchLookup; import java.util.Collections; import java.util.List; -import java.util.Map; public class RoutingFieldMapper extends MetadataFieldMapper { @@ -104,7 +102,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup looku private final boolean required; private RoutingFieldMapper(boolean required) { - super(RoutingFieldType.INSTANCE); + super(RoutingFieldType.INSTANCE, Lucene.KEYWORD_ANALYZER); this.required = required; } @@ -126,9 +124,4 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); - } - } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 076d27e45bf02..aa4edfee4e212 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -384,12 +384,34 @@ private PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType p return new PhraseFieldMapper(phraseFieldType, new PhraseFieldType(parent), a); } + public Map indexAnalyzers(String name, + PhraseFieldMapper phraseFieldMapper, + PrefixFieldMapper prefixFieldMapper) { + Map analyzers = new HashMap<>(); + NamedAnalyzer main = this.analyzers.getIndexAnalyzer(); + analyzers.put(name, main); + if (phraseFieldMapper != null) { + analyzers.put( + phraseFieldMapper.name(), + new NamedAnalyzer(main.name() + "_phrase", AnalyzerScope.INDEX, phraseFieldMapper.analyzer)); + } + if (prefixFieldMapper != null) { + analyzers.put( + prefixFieldMapper.name(), + new NamedAnalyzer(main.name() + "_prefix", AnalyzerScope.INDEX, prefixFieldMapper.analyzer)); + } + return analyzers; + } + @Override public TextFieldMapper build(BuilderContext context) { FieldType fieldType = TextParams.buildFieldType(index, store, indexOptions, norms, termVectors); TextFieldType tft = buildFieldType(fieldType, context); + PhraseFieldMapper phraseFieldMapper = buildPhraseMapper(fieldType, tft); + PrefixFieldMapper prefixFieldMapper = buildPrefixMapper(context, fieldType, tft); return new TextFieldMapper(name, fieldType, tft, - buildPrefixMapper(context, fieldType, tft), buildPhraseMapper(fieldType, tft), + indexAnalyzers(tft.name(), phraseFieldMapper, prefixFieldMapper), + prefixFieldMapper, phraseFieldMapper, multiFieldsBuilder.build(this, context), copyTo.build(), this); } } @@ -845,10 +867,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S protected TextFieldMapper(String simpleName, FieldType fieldType, TextFieldType mappedFieldType, + Map indexAnalyzers, PrefixFieldMapper prefixFieldMapper, PhraseFieldMapper phraseFieldMapper, MultiFields multiFields, CopyTo copyTo, Builder builder) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo); assert mappedFieldType.getTextSearchInfo().isTokenized(); assert mappedFieldType.hasDocValues() == false; if (fieldType.indexOptions() == IndexOptions.NONE && fieldType().fielddata()) { @@ -918,24 +941,6 @@ public TextFieldType fieldType() { return (TextFieldType) super.fieldType(); } - @Override - public Map indexAnalyzers() { - Map analyzers = new HashMap<>(); - NamedAnalyzer main = builder.analyzers.getIndexAnalyzer(); - analyzers.put(name(), main); - if (phraseFieldMapper != null) { - analyzers.put( - phraseFieldMapper.name(), - new NamedAnalyzer(main.name() + "_phrase", AnalyzerScope.INDEX, phraseFieldMapper.analyzer)); - } - if (prefixFieldMapper != null) { - analyzers.put( - prefixFieldMapper.name(), - new NamedAnalyzer(main.name() + "_prefix", AnalyzerScope.INDEX, prefixFieldMapper.analyzer)); - } - return analyzers; - } - public static Query createPhraseQuery(TokenStream stream, String field, int slop, boolean enablePositionIncrements) throws IOException { MultiPhraseQuery.Builder mpqb = new MultiPhraseQuery.Builder(); mpqb.setSlop(slop); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 0d3d745886016..1bb255bac15b5 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -58,7 +58,6 @@ import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; @@ -555,13 +554,14 @@ public void testRolloverClusterStateForDataStream() throws Exception { DateFieldMapper dateFieldMapper = new DateFieldMapper.Builder("@timestamp", DateFieldMapper.Resolution.MILLISECONDS, null, false, Version.CURRENT) .build(builderContext); - MetadataFieldMapper mockedTimestampField = mock(MetadataFieldMapper.class); - when(mockedTimestampField.name()).thenReturn("_data_stream_timestamp"); MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class); when(mockedTimestampFieldType.name()).thenReturn("_data_stream_timestamp"); - when(mockedTimestampField.fieldType()).thenReturn(mockedTimestampFieldType); - when(mockedTimestampField.copyTo()).thenReturn(FieldMapper.CopyTo.empty()); - when(mockedTimestampField.multiFields()).thenReturn(FieldMapper.MultiFields.empty()); + MetadataFieldMapper mockedTimestampField = new MetadataFieldMapper(mockedTimestampFieldType) { + @Override + protected String contentType() { + return null; + } + }; MappingLookup mappingLookup = new MappingLookup(List.of(mockedTimestampField, dateFieldMapper), List.of(), List.of(), 0); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index c3843cfff6a56..b9c86557582ba 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -32,7 +32,6 @@ import java.io.StringReader; import java.util.Arrays; import java.util.Collections; -import java.util.Map; public class DocumentFieldMapperTests extends LuceneTestCase { @@ -87,7 +86,9 @@ static class FakeFieldMapper extends FieldMapper { final String indexedValue; FakeFieldMapper(FakeFieldType fieldType, String indexedValue) { - super(fieldType.name(), fieldType, MultiFields.empty(), CopyTo.empty()); + super(fieldType.name(), fieldType, + new NamedAnalyzer("fake", AnalyzerScope.INDEX, new FakeAnalyzer(indexedValue)), + MultiFields.empty(), CopyTo.empty()); this.indexedValue = indexedValue; } @@ -95,13 +96,6 @@ static class FakeFieldMapper extends FieldMapper { protected void parseCreateField(ParseContext context) { } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap( - name(), - new NamedAnalyzer("fake", AnalyzerScope.INDEX, new FakeAnalyzer(indexedValue))); - } - @Override protected String contentType() { return null; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 9bf420cac2630..8ad5249b40d92 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -23,13 +23,11 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.Map; // Like a String mapper but with very few options. We just use it to test if highlighting on a custom string mapped field works as expected. public class FakeStringFieldMapper extends FieldMapper { @@ -84,12 +82,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc protected FakeStringFieldMapper(MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { - super(mappedFieldType.name(), mappedFieldType, multiFields, copyTo); - } - - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.STANDARD_ANALYZER); + super(mappedFieldType.name(), mappedFieldType, Lucene.STANDARD_ANALYZER, multiFields, copyTo); } @Override diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java index 03cd973216f38..06db9507ff370 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -53,7 +52,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -415,7 +413,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc private FlattenedFieldMapper(String simpleName, MappedFieldType mappedFieldType, Builder builder) { - super(simpleName, mappedFieldType, CopyTo.empty()); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, CopyTo.empty()); this.builder = builder; this.fieldParser = new FlattenedFieldParser(mappedFieldType.name(), keyedFieldName(), mappedFieldType, builder.depthLimit.get(), builder.ignoreAbove.get(), builder.nullValue.get()); @@ -448,11 +446,6 @@ public String keyedFieldName() { return mappedFieldType.name() + KEYED_FIELD_SUFFIX; } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); - } - @Override protected void parseCreateField(ParseContext context) throws IOException { if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index c2331ce203f32..05a772659bbe3 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.FieldMapper; @@ -55,7 +54,6 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -311,7 +309,7 @@ private VersionStringFieldMapper( MultiFields multiFields, CopyTo copyTo ) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); this.fieldType = fieldType; } @@ -320,11 +318,6 @@ public VersionStringFieldType fieldType() { return (VersionStringFieldType) super.fieldType(); } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), Lucene.KEYWORD_ANALYZER); - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index fedc2ed8051bd..614da8a0228c7 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -209,16 +209,6 @@ public void parse(ParseContext context) throws IOException { context.doc().addWithKey(fieldType().name(), field); } - @Override - protected boolean indexedByDefault() { - return false; - } - - @Override - protected boolean docValuesByDefault() { - return true; - } - @Override protected void parseCreateField(ParseContext context) { throw new AssertionError("parse is implemented directly"); diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 7481b64a67b46..d6b85cdf0f6ab 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -71,7 +71,6 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; @@ -931,10 +930,10 @@ protected String parseSourceValue(Object value) { private final FieldType ngramFieldType; private final Version indexVersionCreated; - private WildcardFieldMapper(String simpleName, MappedFieldType mappedFieldType, + private WildcardFieldMapper(String simpleName, WildcardFieldType mappedFieldType, int ignoreAbove, MultiFields multiFields, CopyTo copyTo, String nullValue, Version indexVersionCreated) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, mappedFieldType.analyzer, multiFields, copyTo); this.nullValue = nullValue; this.ignoreAbove = ignoreAbove; this.indexVersionCreated = indexVersionCreated; @@ -1003,11 +1002,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public Map indexAnalyzers() { - return Collections.singletonMap(name(), fieldType().analyzer); - } - public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexVersionCreated).init(this); } From 0d2a081fe06c0ccbe67f5777a829b05bbbf193d4 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 2 Nov 2020 16:59:26 +0000 Subject: [PATCH 14/19] SearchAsYouType tests --- .../SearchAsYouTypeFieldMapperTests.java | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 1b343da870dc0..d476951e36034 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -179,12 +179,12 @@ public void testDefaultConfiguration() throws IOException { assertRootFieldMapper(rootMapper, 3, "default"); PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(defaultMapper, "field._index_prefix"); - assertPrefixFieldType(prefixFieldMapper, 3, "default"); + assertPrefixFieldType(prefixFieldMapper, rootMapper.indexAnalyzers(), 3, "default"); - assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._2gram"), 2, "default", prefixFieldMapper.fieldType()); - assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._3gram"), 3, "default", prefixFieldMapper.fieldType()); + assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._2gram"), + rootMapper.indexAnalyzers(), 2, "default", prefixFieldMapper.fieldType()); + assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._3gram"), + rootMapper.indexAnalyzers(), 3, "default", prefixFieldMapper.fieldType()); } public void testConfiguration() throws IOException { @@ -200,14 +200,14 @@ public void testConfiguration() throws IOException { assertRootFieldMapper(rootMapper, maxShingleSize, analyzerName); PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(defaultMapper, "field._index_prefix"); - assertPrefixFieldType(prefixFieldMapper, maxShingleSize, analyzerName); + assertPrefixFieldType(prefixFieldMapper, rootMapper.indexAnalyzers(), maxShingleSize, analyzerName); - assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._2gram"), 2, analyzerName, prefixFieldMapper.fieldType()); - assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._3gram"), 3, analyzerName, prefixFieldMapper.fieldType()); - assertShingleFieldType( - getShingleFieldMapper(defaultMapper, "field._4gram"), 4, analyzerName, prefixFieldMapper.fieldType()); + assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._2gram"), + rootMapper.indexAnalyzers(), 2, analyzerName, prefixFieldMapper.fieldType()); + assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._3gram"), + rootMapper.indexAnalyzers(), 3, analyzerName, prefixFieldMapper.fieldType()); + assertShingleFieldType(getShingleFieldMapper(defaultMapper, "field._4gram"), + rootMapper.indexAnalyzers(), 4, analyzerName, prefixFieldMapper.fieldType()); } public void testSimpleMerge() throws IOException { @@ -603,13 +603,14 @@ private static void assertRootFieldMapper(SearchAsYouTypeFieldMapper mapper, assertThat(mapper.prefixField(), notNullValue()); assertThat(mapper.prefixField().fieldType().parentField, equalTo(mapper.name())); - assertPrefixFieldType(mapper.prefixField(), maxShingleSize, analyzerName); + assertPrefixFieldType(mapper.prefixField(), mapper.indexAnalyzers, maxShingleSize, analyzerName); for (int shingleSize = 2; shingleSize <= maxShingleSize; shingleSize++) { final ShingleFieldMapper shingleFieldMapper = mapper.shingleFields()[shingleSize - 2]; assertThat(shingleFieldMapper, notNullValue()); - assertShingleFieldType(shingleFieldMapper, shingleSize, analyzerName, mapper.prefixField().fieldType()); + assertShingleFieldType(shingleFieldMapper, mapper.indexAnalyzers, shingleSize, + analyzerName, mapper.prefixField().fieldType()); } final int numberOfShingleSubfields = (maxShingleSize - 2) + 1; @@ -623,19 +624,20 @@ private static void assertSearchAsYouTypeFieldType(SearchAsYouTypeFieldMapper ma PrefixFieldType prefixFieldType) { assertThat(fieldType.shingleFields.length, equalTo(maxShingleSize - 1)); - NamedAnalyzer indexAnalyzer = mapper.indexAnalyzers().values().iterator().next(); + NamedAnalyzer indexAnalyzer = mapper.indexAnalyzers().get(fieldType.name()); for (NamedAnalyzer analyzer : asList(indexAnalyzer, fieldType.getTextSearchInfo().getSearchAnalyzer())) { assertThat(analyzer.name(), equalTo(analyzerName)); } int shingleSize = 2; for (ShingleFieldMapper shingleField : mapper.shingleFields()) { - assertShingleFieldType(shingleField, shingleSize++, analyzerName, prefixFieldType); + assertShingleFieldType(shingleField, mapper.indexAnalyzers(), shingleSize++, analyzerName, prefixFieldType); } assertThat(fieldType.prefixField, equalTo(prefixFieldType)); } private static void assertShingleFieldType(ShingleFieldMapper mapper, + Map indexAnalyzers, int shingleSize, String analyzerName, PrefixFieldType prefixFieldType) { @@ -643,8 +645,7 @@ private static void assertShingleFieldType(ShingleFieldMapper mapper, ShingleFieldType fieldType = mapper.fieldType(); assertThat(fieldType.shingleSize, equalTo(shingleSize)); - NamedAnalyzer indexAnalyer = mapper.indexAnalyzers().values().iterator().next(); - for (NamedAnalyzer analyzer : asList(indexAnalyer, fieldType.getTextSearchInfo().getSearchAnalyzer())) { + for (NamedAnalyzer analyzer : asList(indexAnalyzers.get(fieldType.name()), fieldType.getTextSearchInfo().getSearchAnalyzer())) { assertThat(analyzer.name(), equalTo(analyzerName)); if (shingleSize > 1) { final SearchAsYouTypeAnalyzer wrappedAnalyzer = (SearchAsYouTypeAnalyzer) analyzer.analyzer(); @@ -657,9 +658,10 @@ private static void assertShingleFieldType(ShingleFieldMapper mapper, } - private static void assertPrefixFieldType(PrefixFieldMapper mapper, int shingleSize, String analyzerName) { + private static void assertPrefixFieldType(PrefixFieldMapper mapper, Map indexAnalyzers, + int shingleSize, String analyzerName) { PrefixFieldType fieldType = mapper.fieldType(); - NamedAnalyzer indexAnalyzer = mapper.indexAnalyzers().values().iterator().next(); + NamedAnalyzer indexAnalyzer = indexAnalyzers.get(fieldType.name()); for (NamedAnalyzer analyzer : asList(indexAnalyzer, fieldType.getTextSearchInfo().getSearchAnalyzer())) { assertThat(analyzer.name(), equalTo(analyzerName)); } From 5099a60e7d3bcf3068f05e388fdd474b88fc7056 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Nov 2020 11:30:07 +0000 Subject: [PATCH 15/19] Don't call FetchContext#mapperService --- .../java/org/elasticsearch/search/fetch/FetchContext.java | 8 ++++++++ .../fetch/subphase/highlight/FastVectorHighlighter.java | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java index 1071b5fcf00e7..d5dd3b0ed8c57 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java @@ -167,6 +167,14 @@ public SearchHighlightContext highlight() { return searchContext.highlight(); } + /** + * Does the index analyzer for this field have token filters that may produce + * backwards offsets in term vectors + */ + public boolean containsBrokenAnalysis(String field) { + return mapperService().containsBrokenAnalysis(field); + } + /** * Should the response include scores, even if scores were not calculated in the original query */ diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java index 39acc491a47d6..fdb49d16e2c27 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java @@ -72,7 +72,7 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc FetchSubPhase.HitContext hitContext = fieldContext.hitContext; MappedFieldType fieldType = fieldContext.fieldType; boolean forceSource = fieldContext.forceSource; - boolean fixBrokenAnalysis = fieldContext.context.mapperService().containsBrokenAnalysis(fieldContext.fieldName); + boolean fixBrokenAnalysis = fieldContext.context.containsBrokenAnalysis(fieldContext.fieldName); if (canHighlight(fieldType) == false) { throw new IllegalArgumentException("the field [" + fieldContext.fieldName + From e4f78828bba7e499c234210e48d7cbdc9e24bbc3 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Nov 2020 12:05:30 +0000 Subject: [PATCH 16/19] No really, don't use mapperservice --- .../org/elasticsearch/index/query/QueryShardContext.java | 8 ++++++++ .../java/org/elasticsearch/search/fetch/FetchContext.java | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 4a8c5c262d053..4082405d8b96d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -329,6 +329,14 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap } } + /** + * Does the index analyzer for this field have token filters that may produce + * backwards offsets in term vectors + */ + public boolean containsBrokenAnalysis(String field) { + return mapperService.containsBrokenAnalysis(field); + } + private SearchLookup lookup = null; /** diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java index d5dd3b0ed8c57..0263166059db5 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java @@ -172,7 +172,7 @@ public SearchHighlightContext highlight() { * backwards offsets in term vectors */ public boolean containsBrokenAnalysis(String field) { - return mapperService().containsBrokenAnalysis(field); + return getQueryShardContext().containsBrokenAnalysis(field); } /** From cbf355abde4b7b206fa102ff9a62bd27cd4c05ff Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Nov 2020 16:01:47 +0000 Subject: [PATCH 17/19] Don't expose TextFieldMapper subfields as mappers --- .../index/mapper/TextFieldMapper.java | 183 +++++------------- .../index/mapper/TextFieldMapperTests.java | 72 +++++-- .../index/mapper/TextFieldTypeTests.java | 12 +- .../index/mapper/MapperServiceTestCase.java | 2 + 4 files changed, 101 insertions(+), 168 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index c52062d8e4d08..34c01104dd7d3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -57,7 +57,6 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.Version; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery; @@ -80,7 +79,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -92,6 +90,7 @@ public class TextFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "text"; private static final String FAST_PHRASE_SUFFIX = "._index_phrase"; + private static final String FAST_PREFIX_SUFFIX = "._index_prefix"; public static class Defaults { public static final double FIELDDATA_MIN_FREQUENCY = 0; @@ -330,7 +329,7 @@ private TextFieldType buildFieldType(FieldType fieldType, BuilderContext context return ft; } - private PrefixFieldMapper buildPrefixMapper(BuilderContext context, FieldType fieldType, TextFieldType tft) { + private SubFieldInfo buildPrefixMapper(BuilderContext context, FieldType fieldType, TextFieldType tft) { if (indexPrefixes.get() == null) { return null; } @@ -358,16 +357,15 @@ private PrefixFieldMapper buildPrefixMapper(BuilderContext context, FieldType fi if (fieldType.storeTermVectorOffsets()) { pft.setStoreTermVectorOffsets(true); } - PrefixFieldType prefixFieldType = new PrefixFieldType(tft, fullName + "._index_prefix", indexPrefixes.get()); - tft.setPrefixFieldType(prefixFieldType); - return new PrefixFieldMapper(pft, prefixFieldType, new PrefixWrappedAnalyzer( + tft.setIndexPrefixes(indexPrefixes.get().minChars, indexPrefixes.get().maxChars); + return new SubFieldInfo(fullName + "._index_prefix", pft, new PrefixWrappedAnalyzer( analyzers.getIndexAnalyzer().analyzer(), analyzers.positionIncrementGap.get(), - prefixFieldType.minChars, - prefixFieldType.maxChars)); + indexPrefixes.get().minChars, + indexPrefixes.get().maxChars)); } - private PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType parent) { + private SubFieldInfo buildPhraseInfo(FieldType fieldType, TextFieldType parent) { if (indexPhrases.get() == false) { return null; } @@ -381,24 +379,24 @@ private PhraseFieldMapper buildPhraseMapper(FieldType fieldType, TextFieldType p parent.setIndexPhrases(); PhraseWrappedAnalyzer a = new PhraseWrappedAnalyzer(analyzers.getIndexAnalyzer().analyzer(), analyzers.positionIncrementGap.get()); - return new PhraseFieldMapper(phraseFieldType, new PhraseFieldType(parent), a); + return new SubFieldInfo(parent.name() + FAST_PHRASE_SUFFIX, phraseFieldType, a); } public Map indexAnalyzers(String name, - PhraseFieldMapper phraseFieldMapper, - PrefixFieldMapper prefixFieldMapper) { + SubFieldInfo phraseFieldInfo, + SubFieldInfo prefixFieldInfo) { Map analyzers = new HashMap<>(); NamedAnalyzer main = this.analyzers.getIndexAnalyzer(); analyzers.put(name, main); - if (phraseFieldMapper != null) { + if (phraseFieldInfo != null) { analyzers.put( - phraseFieldMapper.name(), - new NamedAnalyzer(main.name() + "_phrase", AnalyzerScope.INDEX, phraseFieldMapper.analyzer)); + phraseFieldInfo.field, + new NamedAnalyzer(main.name() + "_phrase", AnalyzerScope.INDEX, phraseFieldInfo.analyzer)); } - if (prefixFieldMapper != null) { + if (prefixFieldInfo != null) { analyzers.put( - prefixFieldMapper.name(), - new NamedAnalyzer(main.name() + "_prefix", AnalyzerScope.INDEX, prefixFieldMapper.analyzer)); + prefixFieldInfo.field, + new NamedAnalyzer(main.name() + "_prefix", AnalyzerScope.INDEX, prefixFieldInfo.analyzer)); } return analyzers; } @@ -407,12 +405,18 @@ public Map indexAnalyzers(String name, public TextFieldMapper build(BuilderContext context) { FieldType fieldType = TextParams.buildFieldType(index, store, indexOptions, norms, termVectors); TextFieldType tft = buildFieldType(fieldType, context); - PhraseFieldMapper phraseFieldMapper = buildPhraseMapper(fieldType, tft); - PrefixFieldMapper prefixFieldMapper = buildPrefixMapper(context, fieldType, tft); + SubFieldInfo phraseFieldMapper = buildPhraseInfo(fieldType, tft); + SubFieldInfo prefixFieldMapper = buildPrefixMapper(context, fieldType, tft); + MultiFields multiFields = multiFieldsBuilder.build(this, context); + for (Mapper mapper : multiFields) { + if (mapper.name().endsWith(FAST_PHRASE_SUFFIX) || mapper.name().endsWith(FAST_PREFIX_SUFFIX)) { + throw new MapperParsingException("Cannot use reserved field name [" + mapper.name() + "]"); + } + } return new TextFieldMapper(name, fieldType, tft, indexAnalyzers(tft.name(), phraseFieldMapper, prefixFieldMapper), prefixFieldMapper, phraseFieldMapper, - multiFieldsBuilder.build(this, context), copyTo.build(), this); + multiFields, copyTo.build(), this); } } @@ -478,45 +482,14 @@ protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComp } } - static final class PhraseFieldType extends StringFieldType { - - final TextFieldType parent; - - PhraseFieldType(TextFieldType parent) { - super(parent.name() + FAST_PHRASE_SUFFIX, true, false, false, parent.getTextSearchInfo(), Collections.emptyMap()); - this.parent = parent; - } - - @Override - public String typeName() { - return "phrase"; - } - - @Override - public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { - // Because this internal field is modelled as a multi-field, SourceValueFetcher will look up its - // parent field in _source. So we don't need to use the parent field name here. - return SourceValueFetcher.toString(name(), context, format); - } - - @Override - public Query existsQuery(QueryShardContext context) { - throw new UnsupportedOperationException(); - } - } - - static final class PrefixFieldType extends StringFieldType { + private static final class PrefixFieldType extends StringFieldType { final int minChars; final int maxChars; final TextFieldType parentField; - PrefixFieldType(TextFieldType parentField, String name, PrefixConfig config) { - this(parentField, name, config.minChars, config.maxChars); - } - - PrefixFieldType(TextFieldType parentField, String name, int minChars, int maxChars) { - super(name, true, false, false, parentField.getTextSearchInfo(), Collections.emptyMap()); + PrefixFieldType(TextFieldType parentField, int minChars, int maxChars) { + super(parentField.name() + FAST_PREFIX_SUFFIX, true, false, false, parentField.getTextSearchInfo(), Collections.emptyMap()); this.minChars = minChars; this.maxChars = maxChars; this.parentField = parentField; @@ -524,9 +497,7 @@ static final class PrefixFieldType extends StringFieldType { @Override public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { - // Because this internal field is modelled as a multi-field, SourceValueFetcher will look up its - // parent field in _source. So we don't need to use the parent field name here. - return SourceValueFetcher.toString(name(), context, format); + throw new UnsupportedOperationException(); } boolean accept(int length) { @@ -590,67 +561,18 @@ public Query existsQuery(QueryShardContext context) { } } - private static final class PhraseFieldMapper extends FieldMapper { + private static final class SubFieldInfo { private final Analyzer analyzer; private final FieldType fieldType; + private final String field; - PhraseFieldMapper(FieldType fieldType, PhraseFieldType mappedFieldType, PhraseWrappedAnalyzer analyzer) { - super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); + SubFieldInfo(String field, FieldType fieldType, Analyzer analyzer) { this.fieldType = fieldType; this.analyzer = analyzer; + this.field = field; } - @Override - protected void parseCreateField(ParseContext context) { - throw new UnsupportedOperationException(); - } - - @Override - public Builder getMergeBuilder() { - return null; - } - - @Override - protected String contentType() { - return "phrase"; - } - } - - private static final class PrefixFieldMapper extends FieldMapper { - - private final Analyzer analyzer; - private final FieldType fieldType; - - protected PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType, Analyzer analyzer) { - super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); - this.analyzer = analyzer; - this.fieldType = fieldType; - } - - void addField(ParseContext context, String value) { - context.doc().add(new Field(fieldType().name(), value, fieldType)); - } - - @Override - protected void parseCreateField(ParseContext context) { - throw new UnsupportedOperationException(); - } - - @Override - public Builder getMergeBuilder() { - return null; - } - - @Override - protected String contentType() { - return "prefix"; - } - - @Override - public String toString() { - return fieldType().toString(); - } } public static class TextFieldType extends StringFieldType { @@ -702,8 +624,8 @@ int fielddataMinSegmentSize() { return filter.minSegmentSize; } - void setPrefixFieldType(PrefixFieldType prefixFieldType) { - this.prefixFieldType = prefixFieldType; + void setIndexPrefixes(int minChars, int maxChars) { + this.prefixFieldType = new PrefixFieldType(this, minChars, maxChars); } void setIndexPhrases() { @@ -862,14 +784,14 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S private final Builder builder; private final FieldType fieldType; - private final PrefixFieldMapper prefixFieldMapper; - private final PhraseFieldMapper phraseFieldMapper; + private final SubFieldInfo prefixFieldInfo; + private final SubFieldInfo phraseFieldInfo; protected TextFieldMapper(String simpleName, FieldType fieldType, TextFieldType mappedFieldType, Map indexAnalyzers, - PrefixFieldMapper prefixFieldMapper, - PhraseFieldMapper phraseFieldMapper, + SubFieldInfo prefixFieldInfo, + SubFieldInfo phraseFieldInfo, MultiFields multiFields, CopyTo copyTo, Builder builder) { super(simpleName, mappedFieldType, indexAnalyzers, multiFields, copyTo); assert mappedFieldType.getTextSearchInfo().isTokenized(); @@ -878,8 +800,8 @@ protected TextFieldMapper(String simpleName, FieldType fieldType, throw new IllegalArgumentException("Cannot enable fielddata on a [text] field that is not indexed: [" + name() + "]"); } this.fieldType = fieldType; - this.prefixFieldMapper = prefixFieldMapper; - this.phraseFieldMapper = phraseFieldMapper; + this.prefixFieldInfo = prefixFieldInfo; + this.phraseFieldInfo = phraseFieldInfo; this.builder = builder; } @@ -907,30 +829,15 @@ protected void parseCreateField(ParseContext context) throws IOException { if (fieldType.omitNorms()) { createFieldNamesField(context); } - if (prefixFieldMapper != null) { - prefixFieldMapper.addField(context, value); + if (prefixFieldInfo != null) { + context.doc().add(new Field(prefixFieldInfo.field, value, prefixFieldInfo.fieldType)); } - if (phraseFieldMapper != null) { - context.doc().add(new Field(phraseFieldMapper.fieldType().name(), value, phraseFieldMapper.fieldType)); + if (phraseFieldInfo != null) { + context.doc().add(new Field(phraseFieldInfo.field, value, phraseFieldInfo.fieldType)); } } } - @Override - public Iterator iterator() { - List subIterators = new ArrayList<>(); - if (prefixFieldMapper != null) { - subIterators.add(prefixFieldMapper); - } - if (phraseFieldMapper != null) { - subIterators.add(phraseFieldMapper); - } - if (subIterators.size() == 0) { - return super.iterator(); - } - return Iterators.concat(super.iterator(), subIterators.iterator()); - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 205ea95c47dc0..b6d2ed432f8b6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -37,9 +37,12 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MultiPhraseQuery; +import org.apache.lucene.search.NormsFieldExistsQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; @@ -67,6 +70,7 @@ import org.elasticsearch.index.query.MatchPhraseQueryBuilder; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.search.MatchQuery; +import org.elasticsearch.index.search.QueryStringQueryParser; import java.io.IOException; import java.util.Arrays; @@ -564,8 +568,6 @@ public void testIndexPrefixIndexTypes() throws IOException { .field("index_options", "offsets") ) ); - FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix"); - assertEquals(prefix.name(), "field._index_prefix"); ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); IndexableField field = doc.rootDoc().getField("field._index_prefix"); assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, field.fieldType().indexOptions()); @@ -581,10 +583,8 @@ public void testIndexPrefixIndexTypes() throws IOException { .field("index_options", "freqs") ) ); - FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix"); ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); IndexableField field = doc.rootDoc().getField("field._index_prefix"); - assertEquals(prefix.name(), "field._index_prefix"); assertEquals(IndexOptions.DOCS, field.fieldType().indexOptions()); assertFalse(field.fieldType().storeTermVectors()); } @@ -599,10 +599,8 @@ public void testIndexPrefixIndexTypes() throws IOException { .field("index_options", "positions") ) ); - FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix"); ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); IndexableField field = doc.rootDoc().getField("field._index_prefix"); - assertEquals(prefix.name(), "field._index_prefix"); assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, field.fieldType().indexOptions()); assertFalse(field.fieldType().storeTermVectors()); } @@ -617,10 +615,8 @@ public void testIndexPrefixIndexTypes() throws IOException { .field("term_vector", "with_positions_offsets") ) ); - FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix"); ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); IndexableField field = doc.rootDoc().getField("field._index_prefix"); - assertEquals(prefix.name(), "field._index_prefix"); assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, field.fieldType().indexOptions()); assertTrue(field.fieldType().storeTermVectorOffsets()); } @@ -635,10 +631,8 @@ public void testIndexPrefixIndexTypes() throws IOException { .field("term_vector", "with_positions") ) ); - FieldMapper prefix = (FieldMapper) mapper.mappers().getMapper("field._index_prefix"); ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); IndexableField field = doc.rootDoc().getField("field._index_prefix"); - assertEquals(prefix.name(), "field._index_prefix"); assertEquals(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, field.fieldType().indexOptions()); assertFalse(field.fieldType().storeTermVectorOffsets()); } @@ -771,6 +765,45 @@ protected TokenStreamComponents createComponents(String fieldName) { assertThat(e.getMessage(), containsString("Cannot set index_phrases on field [field] if positions are not enabled")); } + public void testObjectExistsQuery() throws IOException, ParseException { + MapperService ms = createMapperService(mapping(b -> { + b.startObject("foo"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("bar"); + { + b.field("type", "text"); + b.field("index_phrases", true); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); + QueryShardContext qsc = createQueryShardContext(ms); + QueryStringQueryParser parser = new QueryStringQueryParser(qsc, "f"); + Query q = parser.parse("foo:*"); + assertEquals(new ConstantScoreQuery(new BooleanQuery.Builder() + .add(new NormsFieldExistsQuery("foo.bar"), BooleanClause.Occur.SHOULD) + .build()), q); + } + + private static void assertAnalyzesTo(Analyzer analyzer, String field, String input, String[] output) throws IOException { + try (TokenStream ts = analyzer.tokenStream(field, input)) { + ts.reset(); + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + for (String t : output) { + assertTrue(ts.incrementToken()); + assertEquals(t, termAtt.toString()); + } + assertFalse(ts.incrementToken()); + ts.end(); + } + } + public void testIndexPrefixMapping() throws IOException { { @@ -780,33 +813,34 @@ public void testIndexPrefixMapping() throws IOException { .field("analyzer", "standard") .startObject("index_prefixes") .field("min_chars", 2) - .field("max_chars", 10) + .field("max_chars", 6) .endObject() ) ); - assertThat(ms.documentMapper().mappers().getMapper("field._index_prefix").toString(), containsString("prefixChars=2:10")); - ParsedDocument doc = ms.documentMapper().parse(source(b -> b.field("field", "Some English text that is going to be very useful"))); IndexableField[] fields = doc.rootDoc().getFields("field._index_prefix"); assertEquals(1, fields.length); withLuceneIndex(ms, iw -> iw.addDocument(doc.rootDoc()), ir -> {}); // check we can index + + assertAnalyzesTo(ms.indexAnalyzer(), "field._index_prefix", "tweedledum", + new String[]{ "tw", "twe", "twee", "tweed", "tweedl" }); } { - DocumentMapper mapper = createDocumentMapper( + MapperService ms = createMapperService( fieldMapping(b -> b.field("type", "text").field("analyzer", "standard").startObject("index_prefixes").endObject()) ); - assertThat(mapper.mappers().getMapper("field._index_prefix").toString(), containsString("prefixChars=2:5")); - + assertAnalyzesTo(ms.indexAnalyzer(), "field._index_prefix", "tweedledum", + new String[]{ "tw", "twe", "twee", "tweed" }); } { - DocumentMapper mapper = createDocumentMapper( + MapperService ms = createMapperService( fieldMapping(b -> b.field("type", "text").nullField("index_prefixes")) ); - assertNull(mapper.mappers().getMapper("field._index_prefix")); + expectThrows(Exception.class, () -> ms.indexAnalyzer().tokenStream("field._index_prefixes", "test")); } { @@ -815,7 +849,7 @@ public void testIndexPrefixMapping() throws IOException { b.startObject("index_prefixes").field("min_chars", 1).field("max_chars", 10).endObject(); b.startObject("fields").startObject("_index_prefix").field("type", "text").endObject().endObject(); }))); - assertThat(e.getMessage(), containsString("Field [field._index_prefix] is defined more than once")); + assertThat(e.getMessage(), containsString("Cannot use reserved field name [field._index_prefix]")); } { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 2b30c2fac063e..c7d3171068ed1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -133,7 +133,7 @@ public void testFuzzyQuery() { public void testIndexPrefixes() { TextFieldType ft = createFieldType(); - ft.setPrefixFieldType(new TextFieldMapper.PrefixFieldType(ft, "field._index_prefix", 2, 10)); + ft.setIndexPrefixes(2, 10); Query q = ft.prefixQuery("goin", CONSTANT_SCORE_REWRITE, false, randomMockShardContext()); assertEquals(new ConstantScoreQuery(new TermQuery(new Term("field._index_prefix", "goin"))), q); @@ -168,15 +168,5 @@ public void testFetchSourceValue() throws IOException { assertEquals(List.of("value"), fetchSourceValue(fieldType, "value")); assertEquals(List.of("42"), fetchSourceValue(fieldType, 42L)); assertEquals(List.of("true"), fetchSourceValue(fieldType, true)); - - TextFieldMapper.PrefixFieldType prefixFieldType = new TextFieldMapper.PrefixFieldType(fieldType, "field._index_prefix", 2, 10); - assertEquals(List.of("value"), fetchSourceValue(prefixFieldType, "value")); - assertEquals(List.of("42"), fetchSourceValue(prefixFieldType, 42L)); - assertEquals(List.of("true"), fetchSourceValue(prefixFieldType, true)); - - TextFieldMapper.PhraseFieldType phraseFieldType = new TextFieldMapper.PhraseFieldType(fieldType); - assertEquals(List.of("value"), fetchSourceValue(phraseFieldType, "value")); - assertEquals(List.of("42"), fetchSourceValue(phraseFieldType, 42L)); - assertEquals(List.of("true"), fetchSourceValue(phraseFieldType, true)); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index e06722d76d0b2..5ee9d3cd66e16 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -382,6 +382,8 @@ protected QueryShardContext createQueryShardContext(MapperService mapperService) .thenAnswer(inv -> mapperService.fieldType(inv.getArguments()[0].toString()) != null); when(queryShardContext.getIndexAnalyzers()).thenReturn(mapperService.getIndexAnalyzers()); when(queryShardContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings()); + when(queryShardContext.getObjectMapper(anyString())).thenAnswer( + inv -> mapperService.getObjectMapper(inv.getArguments()[0].toString())); when(queryShardContext.simpleMatchToIndexNames(anyObject())).thenAnswer( inv -> mapperService.simpleMatchToFullName(inv.getArguments()[0].toString()) ); From 483b26c1fe454f58461b8b0bd6c9b032e1dab199 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 4 Nov 2020 16:49:52 +0000 Subject: [PATCH 18/19] FieldFetcher test --- .../search/fetch/subphase/FieldFetcherTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 181f99b982cbf..aab36d07091e7 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -384,8 +384,8 @@ public void testTextSubFields() throws IOException { .endObject(); Map fields = fetchFields(mapperService, source, "*"); - assertThat(fields.size(), equalTo(3)); - assertThat(fields.keySet(), containsInAnyOrder("field", "field._index_prefix", "field._index_phrase")); + assertThat(fields.size(), equalTo(1)); + assertThat(fields.keySet(), containsInAnyOrder("field")); for (DocumentField field : fields.values()) { assertThat(field.getValues().size(), equalTo(1)); From 6ad4804b3c21fdf91c18a5881980e2d2fe522b0b Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Thu, 5 Nov 2020 10:08:13 +0000 Subject: [PATCH 19/19] naming --- .../elasticsearch/index/mapper/TextFieldMapper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 6ca42c91042f3..beb5cdc0f955b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -329,7 +329,7 @@ private TextFieldType buildFieldType(FieldType fieldType, BuilderContext context return ft; } - private SubFieldInfo buildPrefixMapper(BuilderContext context, FieldType fieldType, TextFieldType tft) { + private SubFieldInfo buildPrefixInfo(BuilderContext context, FieldType fieldType, TextFieldType tft) { if (indexPrefixes.get() == null) { return null; } @@ -405,8 +405,8 @@ public Map indexAnalyzers(String name, public TextFieldMapper build(BuilderContext context) { FieldType fieldType = TextParams.buildFieldType(index, store, indexOptions, norms, termVectors); TextFieldType tft = buildFieldType(fieldType, context); - SubFieldInfo phraseFieldMapper = buildPhraseInfo(fieldType, tft); - SubFieldInfo prefixFieldMapper = buildPrefixMapper(context, fieldType, tft); + SubFieldInfo phraseFieldInfo = buildPhraseInfo(fieldType, tft); + SubFieldInfo prefixFieldInfo = buildPrefixInfo(context, fieldType, tft); MultiFields multiFields = multiFieldsBuilder.build(this, context); for (Mapper mapper : multiFields) { if (mapper.name().endsWith(FAST_PHRASE_SUFFIX) || mapper.name().endsWith(FAST_PREFIX_SUFFIX)) { @@ -414,8 +414,8 @@ public TextFieldMapper build(BuilderContext context) { } } return new TextFieldMapper(name, fieldType, tft, - indexAnalyzers(tft.name(), phraseFieldMapper, prefixFieldMapper), - prefixFieldMapper, phraseFieldMapper, + indexAnalyzers(tft.name(), phraseFieldInfo, prefixFieldInfo), + prefixFieldInfo, phraseFieldInfo, multiFields, copyTo.build(), this); } }