diff --git a/docs/reference/mapping/types/flattened.asciidoc b/docs/reference/mapping/types/flattened.asciidoc index f18cae2b5780a..a793f50560528 100644 --- a/docs/reference/mapping/types/flattened.asciidoc +++ b/docs/reference/mapping/types/flattened.asciidoc @@ -1,6 +1,3 @@ -[role="xpack"] -[testenv="basic"] - [[flattened]] === Flattened field type ++++ diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_flattened.yml similarity index 90% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml rename to rest-api-spec/src/main/resources/rest-api-spec/test/search/340_flattened.yml index b25da4535dbd5..bf15d8178dfdd 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/flattened/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_flattened.yml @@ -1,9 +1,10 @@ ---- -"Test exists query on flattened object field": +setup: - skip: - version: " - 7.2.99" - reason: "Flattened fields were implemented in 7.3." + version: " - 7.99.99" + reason: "Flattened fields were moved from xpack to core in 7.12." +--- +"Test exists query on flattened object field": - do: indices.create: index: flattened_test @@ -53,10 +54,6 @@ --- "Test query string query on flattened object field": - - skip: - version: " - 7.2.99" - reason: "Flattened fields were implemented in 7.3." - - do: indices.create: index: test @@ -113,10 +110,6 @@ --- "Test fields option on flattened object field": - - skip: - version: " - 7.9.99" - reason: "Fields option on search request was added in 7.10" - - do: indices.create: index: test @@ -159,9 +152,6 @@ --- "Test fields option on flattened object field with include_unmapped": - - skip: - version: ' - 7.10.99' - reason: support was introduced in 7.11 - do: indices.create: index: test diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java index 5230de8102818..60cad33afa7b4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java @@ -25,8 +25,7 @@ * sure to passes an empty multi-fields list to help prevent conflicting sub-keys from being * registered. * - * Note: we anticipate that 'flattened' fields will be the only implementation of this - * interface. Flattened object fields live in the 'mapper-flattened' module. + * Note: currently 'flattened' fields are the only implementation of this interface. */ public abstract class DynamicKeyFieldMapper extends FieldMapper { diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java similarity index 98% rename from x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java rename to server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index d1d762f5044af..4b72755ab091a 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.flattened.mapper; +package org.elasticsearch.index.mapper.flattened; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java similarity index 96% rename from x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldParser.java rename to server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index d3c931c0317ae..922d45ca7cf88 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.flattened.mapper; +package org.elasticsearch.index.mapper.flattened; import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedSetDocValuesField; diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlattenedLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java similarity index 97% rename from x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlattenedLeafFieldData.java rename to server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java index 7af2d46e39a57..6125ba3e4cbec 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlattenedLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.flattened.mapper; +package org.elasticsearch.index.mapper.flattened; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.SortedSetDocValues; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 1c74d8e9e4bbf..2e6cfe989b9d2 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -44,6 +44,7 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncAction; import org.elasticsearch.index.seqno.RetentionLeaseSyncer; @@ -113,18 +114,21 @@ public static Map getMappers(List mappe } mappers.put(BooleanFieldMapper.CONTENT_TYPE, BooleanFieldMapper.PARSER); mappers.put(BinaryFieldMapper.CONTENT_TYPE, BinaryFieldMapper.PARSER); + mappers.put(CompletionFieldMapper.CONTENT_TYPE, CompletionFieldMapper.PARSER); + DateFieldMapper.Resolution milliseconds = DateFieldMapper.Resolution.MILLISECONDS; mappers.put(milliseconds.type(), DateFieldMapper.MILLIS_PARSER); DateFieldMapper.Resolution nanoseconds = DateFieldMapper.Resolution.NANOSECONDS; mappers.put(nanoseconds.type(), DateFieldMapper.NANOS_PARSER); + + mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser()); + mappers.put(FlattenedFieldMapper.CONTENT_TYPE, FlattenedFieldMapper.PARSER); + mappers.put(GeoPointFieldMapper.CONTENT_TYPE, GeoPointFieldMapper.PARSER); mappers.put(IpFieldMapper.CONTENT_TYPE, IpFieldMapper.PARSER); - mappers.put(TextFieldMapper.CONTENT_TYPE, TextFieldMapper.PARSER); mappers.put(KeywordFieldMapper.CONTENT_TYPE, KeywordFieldMapper.PARSER); mappers.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser()); mappers.put(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser()); - mappers.put(CompletionFieldMapper.CONTENT_TYPE, CompletionFieldMapper.PARSER); - mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser()); - mappers.put(GeoPointFieldMapper.CONTENT_TYPE, GeoPointFieldMapper.PARSER); + mappers.put(TextFieldMapper.CONTENT_TYPE, TextFieldMapper.PARSER); for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index eb8327f766d39..dc2c85230a672 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -8,18 +8,24 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; public class FieldTypeLookupTests extends ESTestCase { @@ -209,4 +215,106 @@ private static int size(Iterable iterable) { } return count; } + + public void testFlattenedLookup() { + String fieldName = "object1.object2.field"; + FlattenedFieldMapper mapper = createFlattenedMapper(fieldName); + + FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), emptyList(), emptyList()); + assertEquals(mapper.fieldType(), lookup.get(fieldName)); + + String objectKey = "key1.key2"; + String searchFieldName = fieldName + "." + objectKey; + + MappedFieldType searchFieldType = lookup.get(searchFieldName); + assertEquals(mapper.keyedFieldName(), searchFieldType.name()); + assertThat(searchFieldType, Matchers.instanceOf(FlattenedFieldMapper.KeyedFlattenedFieldType.class)); + + FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) searchFieldType; + assertEquals(objectKey, keyedFieldType.key()); + } + + public void testFlattenedLookupWithAlias() { + String fieldName = "object1.object2.field"; + FlattenedFieldMapper mapper = createFlattenedMapper(fieldName); + + String aliasName = "alias"; + FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, fieldName); + + FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), singletonList(alias), emptyList()); + assertEquals(mapper.fieldType(), lookup.get(aliasName)); + + String objectKey = "key1.key2"; + String searchFieldName = aliasName + "." + objectKey; + + MappedFieldType searchFieldType = lookup.get(searchFieldName); + assertEquals(mapper.keyedFieldName(), searchFieldType.name()); + assertThat(searchFieldType, Matchers.instanceOf(FlattenedFieldMapper.KeyedFlattenedFieldType.class)); + + FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) searchFieldType; + assertEquals(objectKey, keyedFieldType.key()); + } + + public void testFlattenedLookupWithMultipleFields() { + String field1 = "object1.object2.field"; + String field2 = "object1.field"; + String field3 = "object2.field"; + + FlattenedFieldMapper mapper1 = createFlattenedMapper(field1); + FlattenedFieldMapper mapper2 = createFlattenedMapper(field2); + FlattenedFieldMapper mapper3 = createFlattenedMapper(field3); + + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2), emptyList(), emptyList()); + assertNotNull(lookup.get(field1 + ".some.key")); + assertNotNull(lookup.get(field2 + ".some.key")); + + lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2, mapper3), emptyList(), emptyList()); + assertNotNull(lookup.get(field1 + ".some.key")); + assertNotNull(lookup.get(field2 + ".some.key")); + assertNotNull(lookup.get(field3 + ".some.key")); + } + + public void testMaxDynamicKeyDepth() { + Map mappers = new HashMap<>(); + Map aliases = new HashMap<>(); + assertEquals(0, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases)); + + // Add a flattened object field. + String name = "object1.object2.field"; + FlattenedFieldMapper flattenedMapper = createFlattenedMapper(name); + mappers.put(name, flattenedMapper); + assertEquals(3, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases)); + + // Add a short alias to that field. + String aliasName = "alias"; + aliases.put(aliasName, name); + assertEquals(3, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases)); + + // Add a longer alias to that field. + String longAliasName = "object1.object2.object3.alias"; + aliases.put(longAliasName, name); + assertEquals(4, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases)); + + // Update the long alias to refer to a non-flattened object field. + String fieldName = "field"; + aliases.put(longAliasName, fieldName); + assertEquals(3, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases)); + } + + public void testFieldLookupIterator() { + MockFieldMapper mapper = new MockFieldMapper("foo"); + FlattenedFieldMapper flattenedMapper = createFlattenedMapper("object1.object2.field"); + + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper, flattenedMapper), emptyList(), emptyList()); + + Set fieldNames = new HashSet<>(); + lookup.filter(ft -> true).forEach(ft -> fieldNames.add(ft.name())); + + assertThat(fieldNames, containsInAnyOrder( + mapper.name(), flattenedMapper.name(), flattenedMapper.keyedFieldName())); + } + + private FlattenedFieldMapper createFlattenedMapper(String fieldName) { + return new FlattenedFieldMapper.Builder(fieldName).build(new ContentPath()); + } } diff --git a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java similarity index 96% rename from x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapperTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index e37c965dc53c5..1260b59f532b2 100644 --- a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.flattened.mapper; +package org.elasticsearch.index.mapper.flattened; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; @@ -22,15 +23,11 @@ import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.xpack.flattened.FlattenedMapperPlugin; -import org.elasticsearch.xpack.flattened.mapper.FlattenedFieldMapper.KeyedFlattenedFieldType; -import org.elasticsearch.xpack.flattened.mapper.FlattenedFieldMapper.RootFlattenedFieldType; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldType; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; import java.io.IOException; import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.Map; import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; @@ -67,11 +64,6 @@ protected void registerParameters(ParameterChecker checker) throws IOException { m -> assertEquals(10, ((FlattenedFieldMapper)m).depthLimit())); } - @Override - protected Collection getPlugins() { - return Collections.singleton(new FlattenedMapperPlugin()); - } - public void testDefaults() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); ParsedDocument parsedDoc = mapper.parse(source(b -> b.startObject("field").field("key", "value").endObject())); diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java similarity index 97% rename from x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldParserTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java index 150d9bd71c4be..14de011ad238a 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParserTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.flattened.mapper; +package org.elasticsearch.index.mapper.flattened; import com.fasterxml.jackson.core.JsonParseException; import org.apache.lucene.index.IndexableField; @@ -17,6 +18,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MockFieldMapper.FakeFieldType; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldParser; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.XContentTestUtils; import org.junit.Before; diff --git a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldSearchTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSearchTests.java similarity index 97% rename from x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldSearchTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSearchTests.java index 7ae22cb16fb9f..ebed0a5cd2d35 100644 --- a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlattenedFieldSearchTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSearchTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.flattened.mapper; +package org.elasticsearch.index.mapper.flattened; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -15,7 +16,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; @@ -24,13 +24,10 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.flattened.FlattenedMapperPlugin; import org.junit.Before; import java.io.IOException; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -53,10 +50,6 @@ public class FlattenedFieldSearchTests extends ESSingleNodeTestCase { - protected Collection> getPlugins() { - return pluginList(FlattenedMapperPlugin.class, LocalStateCompositeXPackPlugin.class); - } - @Before public void setUpIndex() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject() diff --git a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlattenedIndexFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java similarity index 86% rename from x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlattenedIndexFieldDataTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java index 7397718fb76f6..36dc484ad79d8 100644 --- a/x-pack/plugin/mapper-flattened/src/internalClusterTest/java/org/elasticsearch/xpack/flattened/mapper/FlattenedIndexFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.flattened.mapper; +package org.elasticsearch.index.mapper.flattened; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; @@ -23,25 +24,17 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldData; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldType; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.flattened.FlattenedMapperPlugin; -import org.elasticsearch.xpack.flattened.mapper.FlattenedFieldMapper.KeyedFlattenedFieldData; -import org.elasticsearch.xpack.flattened.mapper.FlattenedFieldMapper.KeyedFlattenedFieldType; import java.io.IOException; -import java.util.Collection; import java.util.concurrent.atomic.AtomicInteger; public class FlattenedIndexFieldDataTests extends ESSingleNodeTestCase { - @Override - protected Collection> getPlugins() { - return pluginList(FlattenedMapperPlugin.class, LocalStateCompositeXPackPlugin.class); - } public void testGlobalFieldDataCaching() throws IOException { // Set up the index service. diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java similarity index 95% rename from x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlattenedFieldTypeTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java index e0f29eb839a7f..d5c8f585705ab 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedFieldTypeTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.flattened.mapper; +package org.elasticsearch.index.mapper.flattened; import org.apache.lucene.index.Term; import org.apache.lucene.search.MultiTermQuery; @@ -19,7 +20,7 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldTypeTestCase; -import org.elasticsearch.xpack.flattened.mapper.FlattenedFieldMapper.KeyedFlattenedFieldType; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.KeyedFlattenedFieldType; import java.io.IOException; import java.util.ArrayList; diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlattenedLeafFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java similarity index 95% rename from x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlattenedLeafFieldDataTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java index 2fc0e85eeb932..9f2178a9085ae 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/KeyedFlattenedLeafFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java @@ -1,17 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.flattened.mapper; +package org.elasticsearch.index.mapper.flattened; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldParser; +import org.elasticsearch.index.mapper.flattened.KeyedFlattenedLeafFieldData; import org.elasticsearch.test.ESTestCase; import org.junit.Before; diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java similarity index 95% rename from x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlattenedFieldTypeTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java index 5d49851c55fbf..f90d4607d2b8c 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/RootFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java @@ -1,11 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.flattened.mapper; +package org.elasticsearch.index.mapper.flattened; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocValuesFieldExistsQuery; @@ -21,7 +22,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; -import org.elasticsearch.xpack.flattened.mapper.FlattenedFieldMapper.RootFlattenedFieldType; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; import java.io.IOException; import java.util.Collections; diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java index 54b5ad062c87d..c3df44c795e30 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java @@ -11,9 +11,13 @@ import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.test.ESTestCase; import org.junit.Before; +import java.util.Collections; +import java.util.function.Function; + import static org.mockito.AdditionalAnswers.returnsFirstArg; import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.doReturn; @@ -49,7 +53,38 @@ public void testFieldAliases() { assertEquals(docValues, fetchedDocValues); } - private IndexFieldData createFieldData(ScriptDocValues scriptDocValues) { + public void testFlattenedField() { + ScriptDocValues docValues1 = mock(ScriptDocValues.class); + IndexFieldData fieldData1 = createFieldData(docValues1); + + ScriptDocValues docValues2 = mock(ScriptDocValues.class); + IndexFieldData fieldData2 = createFieldData(docValues2); + + FlattenedFieldMapper.KeyedFlattenedFieldType fieldType1 + = new FlattenedFieldMapper.KeyedFlattenedFieldType("field", true, true, "key1", false, Collections.emptyMap()); + FlattenedFieldMapper.KeyedFlattenedFieldType fieldType2 + = new FlattenedFieldMapper.KeyedFlattenedFieldType( "field", true, true, "key2", false, Collections.emptyMap()); + + Function> fieldDataSupplier = fieldType -> { + FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) fieldType; + return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; + }; + + LeafDocLookup docLookup = new LeafDocLookup(field -> { + if (field.equals("json.key1")) { + return fieldType1; + } + if (field.equals("json.key2")) { + return fieldType2; + } + return null; + }, fieldDataSupplier, null); + + assertEquals(docValues1, docLookup.get("json.key1")); + assertEquals(docValues2, docLookup.get("json.key2")); + } + + private IndexFieldData createFieldData(ScriptDocValues scriptDocValues) { LeafFieldData leafFieldData = mock(LeafFieldData.class); doReturn(scriptDocValues).when(leafFieldData).getScriptValues(); diff --git a/x-pack/plugin/mapper-flattened/build.gradle b/x-pack/plugin/mapper-flattened/build.gradle deleted file mode 100644 index a69830d4ab670..0000000000000 --- a/x-pack/plugin/mapper-flattened/build.gradle +++ /dev/null @@ -1,16 +0,0 @@ -apply plugin: 'elasticsearch.esplugin' -apply plugin: 'elasticsearch.internal-cluster-test' - -esplugin { - name 'flattened' - description 'Module for the flattened field type, which allows JSON objects to be flattened into a single field.' - classname 'org.elasticsearch.xpack.flattened.FlattenedMapperPlugin' - extendedPlugins = ['x-pack-core'] -} -archivesBaseName = 'x-pack-flattened' - -dependencies { - compileOnly project(path: xpackModule('core')) - testImplementation(testArtifact(project(xpackModule('core')))) -} - diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedMapperPlugin.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedMapperPlugin.java deleted file mode 100644 index a1623bebf1a24..0000000000000 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/FlattenedMapperPlugin.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.flattened; - -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.plugins.MapperPlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.xpack.flattened.mapper.FlattenedFieldMapper; - -import java.util.Map; - -import static java.util.Collections.singletonMap; - -public class FlattenedMapperPlugin extends Plugin implements MapperPlugin { - - public FlattenedMapperPlugin() { } - - @Override - public Map getMappers() { - return singletonMap(FlattenedFieldMapper.CONTENT_TYPE, FlattenedFieldMapper.PARSER); - } -} diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java deleted file mode 100644 index c518415b2f4c6..0000000000000 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.LeafFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.search.lookup.LeafDocLookup; -import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.flattened.mapper.FlattenedFieldMapper; -import org.elasticsearch.xpack.flattened.mapper.FlattenedFieldMapper.KeyedFlattenedFieldType; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.function.BiFunction; -import java.util.function.Supplier; - -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; -import static org.mockito.Matchers.anyObject; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class FlattenedFieldLookupTests extends ESTestCase { - - public void testFieldTypeLookup() { - String fieldName = "object1.object2.field"; - FlattenedFieldMapper mapper = createFlattenedMapper(fieldName); - - FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), emptyList(), emptyList()); - assertEquals(mapper.fieldType(), lookup.get(fieldName)); - - String objectKey = "key1.key2"; - String searchFieldName = fieldName + "." + objectKey; - - MappedFieldType searchFieldType = lookup.get(searchFieldName); - assertEquals(mapper.keyedFieldName(), searchFieldType.name()); - assertThat(searchFieldType, instanceOf(KeyedFlattenedFieldType.class)); - - KeyedFlattenedFieldType keyedFieldType = (KeyedFlattenedFieldType) searchFieldType; - assertEquals(objectKey, keyedFieldType.key()); - } - - public void testFieldTypeLookupWithAlias() { - String fieldName = "object1.object2.field"; - FlattenedFieldMapper mapper = createFlattenedMapper(fieldName); - - String aliasName = "alias"; - FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, fieldName); - - FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), singletonList(alias), emptyList()); - assertEquals(mapper.fieldType(), lookup.get(aliasName)); - - String objectKey = "key1.key2"; - String searchFieldName = aliasName + "." + objectKey; - - MappedFieldType searchFieldType = lookup.get(searchFieldName); - assertEquals(mapper.keyedFieldName(), searchFieldType.name()); - assertThat(searchFieldType, instanceOf(KeyedFlattenedFieldType.class)); - - KeyedFlattenedFieldType keyedFieldType = (KeyedFlattenedFieldType) searchFieldType; - assertEquals(objectKey, keyedFieldType.key()); - } - - public void testFieldTypeLookupWithMultipleFields() { - String field1 = "object1.object2.field"; - String field2 = "object1.field"; - String field3 = "object2.field"; - - FlattenedFieldMapper mapper1 = createFlattenedMapper(field1); - FlattenedFieldMapper mapper2 = createFlattenedMapper(field2); - FlattenedFieldMapper mapper3 = createFlattenedMapper(field3); - - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2), emptyList(), emptyList()); - assertNotNull(lookup.get(field1 + ".some.key")); - assertNotNull(lookup.get(field2 + ".some.key")); - - lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2, mapper3), emptyList(), emptyList()); - assertNotNull(lookup.get(field1 + ".some.key")); - assertNotNull(lookup.get(field2 + ".some.key")); - assertNotNull(lookup.get(field3 + ".some.key")); - } - - public void testMaxDynamicKeyDepth() { - Map mappers = new HashMap<>(); - Map aliases = new HashMap<>(); - assertEquals(0, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases)); - - // Add a flattened object field. - String name = "object1.object2.field"; - FlattenedFieldMapper flattenedMapper = createFlattenedMapper(name); - mappers.put(name, flattenedMapper); - assertEquals(3, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases)); - - // Add a short alias to that field. - String aliasName = "alias"; - aliases.put(aliasName, name); - assertEquals(3, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases)); - - // Add a longer alias to that field. - String longAliasName = "object1.object2.object3.alias"; - aliases.put(longAliasName, name); - assertEquals(4, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases)); - - // Update the long alias to refer to a non-flattened object field. - String fieldName = "field"; - aliases.put(longAliasName, fieldName); - assertEquals(3, DynamicKeyFieldTypeLookup.getMaxKeyDepth(mappers, aliases)); - } - - public void testFieldLookupIterator() { - MockFieldMapper mapper = new MockFieldMapper("foo"); - FlattenedFieldMapper flattenedMapper = createFlattenedMapper("object1.object2.field"); - - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper, flattenedMapper), emptyList(), emptyList()); - - Set fieldNames = new HashSet<>(); - lookup.filter(ft -> true).forEach(ft -> fieldNames.add(ft.name())); - - assertThat(fieldNames, containsInAnyOrder( - mapper.name(), flattenedMapper.name(), flattenedMapper.keyedFieldName())); - } - - private FlattenedFieldMapper createFlattenedMapper(String fieldName) { - return new FlattenedFieldMapper.Builder(fieldName).build(new ContentPath()); - } - - public void testScriptDocValuesLookup() { - ScriptDocValues docValues1 = mock(ScriptDocValues.class); - IndexFieldData fieldData1 = createFieldData(docValues1); - - ScriptDocValues docValues2 = mock(ScriptDocValues.class); - IndexFieldData fieldData2 = createFieldData(docValues2); - - KeyedFlattenedFieldType fieldType1 - = new KeyedFlattenedFieldType("field", true, true, "key1", false, Collections.emptyMap()); - KeyedFlattenedFieldType fieldType2 - = new KeyedFlattenedFieldType( "field", true, true, "key2", false, Collections.emptyMap()); - - BiFunction, IndexFieldData> fieldDataSupplier = (fieldType, searchLookup) -> { - KeyedFlattenedFieldType keyedFieldType = (KeyedFlattenedFieldType) fieldType; - return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; - }; - - SearchLookup searchLookup = new SearchLookup(field -> { - if (field.equals("json.key1")) { - return fieldType1; - } - if (field.equals("json.key2")) { - return fieldType2; - } - return null; - }, fieldDataSupplier); - LeafDocLookup docLookup = searchLookup.getLeafSearchLookup(null).doc(); - - assertEquals(docValues1, docLookup.get("json.key1")); - assertEquals(docValues2, docLookup.get("json.key2")); - } - - private IndexFieldData createFieldData(ScriptDocValues scriptDocValues) { - LeafFieldData leafFieldData = mock(LeafFieldData.class); - doReturn(scriptDocValues).when(leafFieldData).getScriptValues(); - - IndexFieldData fieldData = mock(IndexFieldData.class); - when(fieldData.getFieldName()).thenReturn("field"); - doReturn(leafFieldData).when(fieldData).load(anyObject()); - - return fieldData; - } -}