From fcce7af424fabf07a91ed22479d19d34901521ce Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 19 Jul 2021 13:57:55 +0200 Subject: [PATCH 01/16] [Rest Api Compatibility] Typed query Type information is no longer available, thereofre type query in v7 compatibility is working as match_all query previously removed in #47207 relates main meta issue #51816 relates types removal meta #54160 --- rest-api-spec/build.gradle | 5 +- .../index/query/TypeQueryV7Builder.java | 89 +++++++++++++++++++ .../elasticsearch/search/SearchModule.java | 2 + 3 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 56aabfc0092ab..e1475dbb6beee 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -94,11 +94,10 @@ tasks.named("yamlRestCompatTest").configure { 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', // not fixing this in #70966 'indices.put_template/11_basic_with_types/Put template with empty mappings', - 'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers', + 'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers', //#42809 nested path and nested filter 'search.aggregation/51_filter_with_types/Filter aggs with terms lookup and ensure it\'s cached', 'search/150_rewrite_on_coordinator/Ensure that we fetch the document only once', //terms_lookup 'search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception', //cutoff_frequency - 'search/340_type_query/type query', // type_query - probably should behave like match_all ] + v7compatibilityNotSupportedTests()) .join(',') @@ -226,6 +225,8 @@ tasks.named("transformV7RestTests").configure({ task -> task.removeWarningForTest("the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; " + "specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour" , "?wait_for_active_shards default is deprecated") + + task.replaceValueInMatch("hits.total", 1, "type query") // 340_type_query.yml }) tasks.register('enforceYamlTestConvention').configure { diff --git a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java new file mode 100644 index 0000000000000..e502d10d06ce7 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.query; + +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ParseField; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.index.mapper.MapperService; + +import java.io.IOException; + +public class TypeQueryV7Builder extends AbstractQueryBuilder { + public static final String NAME = "type"; + private static final ParseField VALUE_FIELD = new ParseField("value"); + private static final ObjectParser PARSER = new ObjectParser<>(NAME, TypeQueryV7Builder::new); + + static { + PARSER.declareString(QueryBuilder::queryName, + AbstractQueryBuilder.NAME_FIELD.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7))); + PARSER.declareFloat(QueryBuilder::boost, + AbstractQueryBuilder.BOOST_FIELD.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7))); + PARSER.declareString((queryBuilder, value) -> { + }, + VALUE_FIELD.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7))); + } + + public TypeQueryV7Builder() { + } + + /** + * Read from a stream. + */ + public TypeQueryV7Builder(StreamInput in) throws IOException { + super(in); + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.field(VALUE_FIELD.getPreferredName(), MapperService.SINGLE_MAPPING_NAME); + printBoostAndQueryName(builder); + builder.endObject(); + } + + @Override + protected Query doToQuery(SearchExecutionContext context) throws IOException { + return new MatchAllDocsQuery(); + } + + @Override + protected boolean doEquals(TypeQueryV7Builder other) { + return true; + } + + @Override + protected int doHashCode() { + return 0; + } + + public static TypeQueryV7Builder fromXContent(XContentParser parser) throws IOException { + try { + return PARSER.apply(parser, null); + } catch (IllegalArgumentException e) { + throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e); + } + } + + @Override + public String getWriteableName() { + return NAME; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 927e0e42544a5..671983ab4bb1d 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -67,6 +67,7 @@ import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.query.TermsSetQueryBuilder; +import org.elasticsearch.index.query.TypeQueryV7Builder; import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.index.query.WrapperQueryBuilder; import org.elasticsearch.index.query.functionscore.ExponentialDecayFunctionBuilder; @@ -837,6 +838,7 @@ private void registerQueryParsers(List plugins) { if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) { registerQuery(new QuerySpec<>(GeoShapeQueryBuilder.NAME, GeoShapeQueryBuilder::new, GeoShapeQueryBuilder::fromXContent)); } + registerQuery(new QuerySpec<>(TypeQueryV7Builder.NAME, TypeQueryV7Builder::new, TypeQueryV7Builder::fromXContent)); registerFromPlugin(plugins, SearchPlugin::getQueries, this::registerQuery); } From be5f38be81c8bc9cce72256b96fc518fb454c0f0 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 19 Jul 2021 14:54:50 +0200 Subject: [PATCH 02/16] register in named xcontent for compatibility --- .../index/query/TypeQueryV7Builder.java | 4 ++-- .../main/java/org/elasticsearch/node/Node.java | 10 ++++++---- .../org/elasticsearch/search/SearchModule.java | 11 ++++++++++- .../java/org/elasticsearch/node/NodeTests.java | 16 +++++++++------- 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java index e502d10d06ce7..68ce551a70d4e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java @@ -24,6 +24,7 @@ public class TypeQueryV7Builder extends AbstractQueryBuilder { public static final String NAME = "type"; + public static final ParseField NAME_V7 = new ParseField("type").forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)); private static final ParseField VALUE_FIELD = new ParseField("value"); private static final ObjectParser PARSER = new ObjectParser<>(NAME, TypeQueryV7Builder::new); @@ -32,8 +33,7 @@ public class TypeQueryV7Builder extends AbstractQueryBuilder AbstractQueryBuilder.NAME_FIELD.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7))); PARSER.declareFloat(QueryBuilder::boost, AbstractQueryBuilder.BOOST_FIELD.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7))); - PARSER.declareString((queryBuilder, value) -> { - }, + PARSER.declareString((queryBuilder, value) -> {}, VALUE_FIELD.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7))); } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 66cdea8bdd9b5..f4c4869cd5b03 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -418,7 +418,7 @@ protected Node(final Environment initialEnvironment, .flatMap(p -> p.getNamedXContent().stream()), ClusterModule.getNamedXWriteables().stream()) .flatMap(Function.identity()).collect(toList()), - getCompatibleNamedXContents() + getCompatibleNamedXContents(searchModule) ); final Map featuresMap = pluginsService .filterPlugins(SystemIndexPlugin.class) @@ -738,9 +738,11 @@ protected Node(final Environment initialEnvironment, } // package scope for testing - List getCompatibleNamedXContents() { - return pluginsService.filterPlugins(Plugin.class).stream() - .flatMap(p -> p.getNamedXContentForCompatibility().stream()).collect(toList()); + List getCompatibleNamedXContents(SearchModule searchModule) { + return Stream.of(pluginsService.filterPlugins(Plugin.class).stream() + .flatMap(p -> p.getNamedXContentForCompatibility().stream()), + searchModule.getNamedXContentForCompatibility().stream()) + .flatMap(Function.identity()).collect(toList()); } protected TransportService newTransportService(Settings settings, Transport transport, ThreadPool threadPool, diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 671983ab4bb1d..c016e5a282462 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.CombinedFieldsQueryBuilder; @@ -256,6 +257,7 @@ import java.util.function.Consumer; import java.util.function.Function; +import static java.util.Collections.emptyList; import static java.util.Collections.unmodifiableMap; import static java.util.Objects.requireNonNull; @@ -838,7 +840,6 @@ private void registerQueryParsers(List plugins) { if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) { registerQuery(new QuerySpec<>(GeoShapeQueryBuilder.NAME, GeoShapeQueryBuilder::new, GeoShapeQueryBuilder::fromXContent)); } - registerQuery(new QuerySpec<>(TypeQueryV7Builder.NAME, TypeQueryV7Builder::new, TypeQueryV7Builder::fromXContent)); registerFromPlugin(plugins, SearchPlugin::getQueries, this::registerQuery); } @@ -895,4 +896,12 @@ private void registerBoolQuery(ParseField name, Writeable.Reader reader) { public FetchPhase getFetchPhase() { return new FetchPhase(fetchSubPhases); } + + public List getNamedXContentForCompatibility() { + if (RestApiVersion.minimumSupported() == RestApiVersion.V_7) { + return List.of( + new NamedXContentRegistry.Entry(QueryBuilder.class, TypeQueryV7Builder.NAME_V7, TypeQueryV7Builder::fromXContent)); + } + return emptyList(); + } } diff --git a/server/src/test/java/org/elasticsearch/node/NodeTests.java b/server/src/test/java/org/elasticsearch/node/NodeTests.java index b76b95280cdb6..422c722f4c7b3 100644 --- a/server/src/test/java/org/elasticsearch/node/NodeTests.java +++ b/server/src/test/java/org/elasticsearch/node/NodeTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.plugins.CircuitBreakerPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.MockHttpTransport; @@ -55,6 +56,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; @LuceneTestCase.SuppressFileSystems(value = "ExtrasFS") public class NodeTests extends ESTestCase { @@ -340,25 +342,25 @@ interface MockRestApiVersion { RestApiVersion minimumRestCompatibilityVersion(); } - static MockRestApiVersion MockCompatibleVersion = Mockito.mock(MockRestApiVersion.class); + static MockRestApiVersion MockCompatibleVersion = mock(MockRestApiVersion.class); static NamedXContentRegistry.Entry v7CompatibleEntries = new NamedXContentRegistry.Entry(Integer.class, - new ParseField("name"), Mockito.mock(ContextParser.class)); + new ParseField("name"), mock(ContextParser.class)); static NamedXContentRegistry.Entry v8CompatibleEntries = new NamedXContentRegistry.Entry(Integer.class, - new ParseField("name2"), Mockito.mock(ContextParser.class)); + new ParseField("name2"), mock(ContextParser.class)); public static class TestRestCompatibility1 extends Plugin { @Override public List getNamedXContentForCompatibility() { // real plugin will use CompatibleVersion.minimumRestCompatibilityVersion() - if (/*CompatibleVersion.minimumRestCompatibilityVersion()*/ + if (/*RestApiVersion.minimumSupported() == */ MockCompatibleVersion.minimumRestCompatibilityVersion().equals(RestApiVersion.V_7)) { //return set of N-1 entries return List.of(v7CompatibleEntries); } // after major release, new compatible apis can be added before the old ones are removed. - if (/*CompatibleVersion.minimumRestCompatibilityVersion()*/ + if (/*RestApiVersion.minimumSupported() == */ MockCompatibleVersion.minimumRestCompatibilityVersion().equals(RestApiVersion.V_8)) { return List.of(v8CompatibleEntries); @@ -381,7 +383,7 @@ public void testLoadingMultipleRestCompatibilityPlugins() throws IOException { plugins.add(TestRestCompatibility1.class); try (Node node = new MockNode(settings.build(), plugins)) { - List compatibleNamedXContents = node.getCompatibleNamedXContents(); + List compatibleNamedXContents = node.getCompatibleNamedXContents(mock(SearchModule.class)); assertThat(compatibleNamedXContents, contains(v7CompatibleEntries)); } } @@ -396,7 +398,7 @@ public void testLoadingMultipleRestCompatibilityPlugins() throws IOException { plugins.add(TestRestCompatibility1.class); try (Node node = new MockNode(settings.build(), plugins)) { - List compatibleNamedXContents = node.getCompatibleNamedXContents(); + List compatibleNamedXContents = node.getCompatibleNamedXContents(mock(SearchModule.class)); assertThat(compatibleNamedXContents, contains(v8CompatibleEntries)); } } From ea9236fea9e9b8a7595b3e1a36a632bbe8fa027e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 20 Jul 2021 17:51:15 +0200 Subject: [PATCH 03/16] rework named xcontent registry --- .../xcontent/NamedXContentRegistry.java | 104 ++++++++---------- .../elasticsearch/index/mapper/Mapping.java | 2 +- .../index/query/SearchExecutionContext.java | 4 + .../index/query/TypeQueryV7Builder.java | 19 +++- .../java/org/elasticsearch/node/Node.java | 14 +-- .../elasticsearch/search/SearchModule.java | 16 +-- .../CompatibleNamedXContentRegistryTests.java | 22 ++-- .../org/elasticsearch/node/NodeTests.java | 12 +- 8 files changed, 100 insertions(+), 93 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java index 5f5f59a21db24..36b8c64cb3d9b 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java @@ -12,12 +12,11 @@ import org.elasticsearch.core.RestApiVersion; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -43,92 +42,84 @@ public static class Entry { /** A name for the entry which is unique within the {@link #categoryClass}. */ public final ParseField name; + public final Function restApiCompatibility; + /** A parser capability of parser the entry's class. */ private final ContextParser parser; - /** Creates a new entry which can be stored by the registry. */ + /** + * Creates a new entry which can be stored by the registry. + */ public Entry(Class categoryClass, ParseField name, CheckedFunction parser) { - this.categoryClass = Objects.requireNonNull(categoryClass); - this.name = Objects.requireNonNull(name); - this.parser = Objects.requireNonNull((p, c) -> parser.apply(p)); + this(categoryClass, name, (p, c) -> parser.apply(p), + RestApiVersion.onOrAfter(RestApiVersion.minimumSupported())); + } + + public Entry(Class categoryClass, ParseField name, CheckedFunction parser, + Function restApiCompatibility) { + this(categoryClass, name, (p, c) -> parser.apply(p), restApiCompatibility); } /** * Creates a new entry which can be stored by the registry. * Prefer {@link Entry#Entry(Class, ParseField, CheckedFunction)} unless you need a context to carry around while parsing. */ public Entry(Class categoryClass, ParseField name, ContextParser parser) { + this(categoryClass, name, parser, + RestApiVersion.onOrAfter(RestApiVersion.minimumSupported())); + } + + public Entry(Class categoryClass, ParseField name, ContextParser parser, + Function restApiCompatibility) { this.categoryClass = Objects.requireNonNull(categoryClass); this.name = Objects.requireNonNull(name); this.parser = Objects.requireNonNull(parser); + this.restApiCompatibility = restApiCompatibility; } } - private final Map, Map> registry; - private final Map, Map> compatibleRegistry; + private final Map, Map>> registry; - public NamedXContentRegistry(List entries){ - this(entries, Collections.emptyList()); - } - public NamedXContentRegistry(List entries, List compatibleEntries) { + public NamedXContentRegistry(List entries) { this.registry = unmodifiableMap(getRegistry(entries)); - this.compatibleRegistry = unmodifiableMap(getCompatibleRegistry(compatibleEntries)); } - private Map, Map> getCompatibleRegistry(List compatibleEntries) { - Map, Map> compatibleRegistry = new HashMap<>(registry); - List unseenEntries = new ArrayList<>(); - compatibleEntries.forEach(entry -> { - Map parsers = compatibleRegistry.get(entry.categoryClass); - if (parsers == null) { - unseenEntries.add(entry); - } else { - Map parsersCopy = new HashMap<>(parsers); - for (String name : entry.name.getAllNamesIncludedDeprecated()) { - parsersCopy.put(name, entry); //override the parser for the given name - } - compatibleRegistry.put(entry.categoryClass, parsersCopy); - } - } - ); - compatibleRegistry.putAll(getRegistry(unseenEntries)); - return compatibleRegistry; - } - private Map, Map> getRegistry(List entries){ + private Map, Map>> getRegistry(List entries){ if (entries.isEmpty()) { return emptyMap(); } - entries = new ArrayList<>(entries); - entries.sort((e1, e2) -> e1.categoryClass.getName().compareTo(e2.categoryClass.getName())); - Map, Map> registry = new HashMap<>(); - Map parsers = null; - Class currentCategory = null; + Map, Map>> registry = new HashMap<>(); for (Entry entry : entries) { - if (currentCategory != entry.categoryClass) { - if (currentCategory != null) { - // we've seen the last of this category, put it into the big map - registry.put(currentCategory, unmodifiableMap(parsers)); - } - parsers = new HashMap<>(); - currentCategory = entry.categoryClass; - } - for (String name : entry.name.getAllNamesIncludedDeprecated()) { - Object old = parsers.put(name, entry); - if (old != null) { - throw new IllegalArgumentException("NamedXContent [" + currentCategory.getName() + "][" + entry.name + "]" + - " is already registered for [" + old.getClass().getName() + "]," + - " cannot register [" + entry.parser.getClass().getName() + "]"); + if (RestApiVersion.minimumSupported().matches(entry.restApiCompatibility)) { + registerParsers(registry, entry, name, RestApiVersion.minimumSupported()); + } + if (RestApiVersion.current().matches(entry.restApiCompatibility)) { + registerParsers(registry, entry, name, RestApiVersion.current()); } } } - // handle the last category - registry.put(currentCategory, unmodifiableMap(parsers)); return registry; } + private void registerParsers(Map, Map>> registry, + Entry entry, + String name, + RestApiVersion restApiVersion) { + final Map, Map> classRegistry = + registry.computeIfAbsent(restApiVersion, (v) -> new HashMap<>()); + final Map parsers = + classRegistry.computeIfAbsent(entry.categoryClass, (v) -> new HashMap<>()); + Object old = parsers.put(name, entry); + if (old != null) { + throw new IllegalArgumentException("NamedXContent [" + entry.categoryClass.getName() + "][" + entry.name + "]" + + " is already registered for [" + old.getClass().getName() + "]," + + " cannot register [" + entry.parser.getClass().getName() + "]"); + } + } + /** * Parse a named object, throwing an exception if the parser isn't found. Throws an {@link NamedObjectNotFoundException} if the * {@code categoryClass} isn't registered because this is almost always a bug. Throws an {@link NamedObjectNotFoundException} if the @@ -137,9 +128,8 @@ private Map, Map> getRegistry(List entries){ * @throws NamedObjectNotFoundException if the categoryClass or name is not registered */ public T parseNamedObject(Class categoryClass, String name, XContentParser parser, C context) throws IOException { - - Map parsers = parser.getRestApiVersion() == RestApiVersion.minimumSupported() ? - compatibleRegistry.get(categoryClass) : registry.get(categoryClass); + Map parsers = registry.getOrDefault(parser.getRestApiVersion(), emptyMap()) + .get(categoryClass); if (parsers == null) { if (registry.isEmpty()) { // The "empty" registry will never work so we throw a better exception as a hint. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java index 4e1c0037e9148..91321c4907480 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -80,7 +80,7 @@ public CompressedXContent toCompressedXContent() { /** * Returns the root object for the current mapping */ - RootObjectMapper getRoot() { + public RootObjectMapper getRoot() { return root; } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index a518ee74a797e..c413b0fe44b46 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -648,4 +648,8 @@ public String getNestedParent(String nestedPath) { public NestedDocuments getNestedDocuments() { return new NestedDocuments(mappingLookup, bitsetFilterCache::getBitSetProducer); } + + public MappingLookup getMappingLookup() { + return mappingLookup; + } } diff --git a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java index 68ce551a70d4e..563fd3aa91f5e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; @@ -23,8 +24,8 @@ import java.io.IOException; public class TypeQueryV7Builder extends AbstractQueryBuilder { - public static final String NAME = "type"; - public static final ParseField NAME_V7 = new ParseField("type").forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)); + private static final String NAME = "type"; + public static final ParseField NAME_V7 = new ParseField(NAME).forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)); private static final ParseField VALUE_FIELD = new ParseField("value"); private static final ObjectParser PARSER = new ObjectParser<>(NAME, TypeQueryV7Builder::new); @@ -33,10 +34,12 @@ public class TypeQueryV7Builder extends AbstractQueryBuilder AbstractQueryBuilder.NAME_FIELD.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7))); PARSER.declareFloat(QueryBuilder::boost, AbstractQueryBuilder.BOOST_FIELD.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7))); - PARSER.declareString((queryBuilder, value) -> {}, + PARSER.declareString(TypeQueryV7Builder::setValue, VALUE_FIELD.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7))); } + private String value; + public TypeQueryV7Builder() { } @@ -61,7 +64,11 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep @Override protected Query doToQuery(SearchExecutionContext context) throws IOException { - return new MatchAllDocsQuery(); + final String typeName = context.getMappingLookup().getMapping().getRoot().name(); + if (value.equals(typeName)) { + return new MatchAllDocsQuery(); + } + return new MatchNoDocsQuery(); } @Override @@ -86,4 +93,8 @@ public static TypeQueryV7Builder fromXContent(XContentParser parser) throws IOEx public String getWriteableName() { return NAME; } + + public void setValue(String value){ + this.value = value; + } } diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index f4c4869cd5b03..a19b5920cc622 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -416,9 +416,9 @@ protected Node(final Environment initialEnvironment, searchModule.getNamedXContents().stream(), pluginsService.filterPlugins(Plugin.class).stream() .flatMap(p -> p.getNamedXContent().stream()), - ClusterModule.getNamedXWriteables().stream()) - .flatMap(Function.identity()).collect(toList()), - getCompatibleNamedXContents(searchModule) + ClusterModule.getNamedXWriteables().stream(), + getCompatibleNamedXContents()) + .flatMap(Function.identity()).collect(toList()) ); final Map featuresMap = pluginsService .filterPlugins(SystemIndexPlugin.class) @@ -738,11 +738,9 @@ protected Node(final Environment initialEnvironment, } // package scope for testing - List getCompatibleNamedXContents(SearchModule searchModule) { - return Stream.of(pluginsService.filterPlugins(Plugin.class).stream() - .flatMap(p -> p.getNamedXContentForCompatibility().stream()), - searchModule.getNamedXContentForCompatibility().stream()) - .flatMap(Function.identity()).collect(toList()); + Stream getCompatibleNamedXContents() { + return pluginsService.filterPlugins(Plugin.class).stream() + .flatMap(p -> p.getNamedXContentForCompatibility().stream()); } protected TransportService newTransportService(Settings settings, Transport transport, ThreadPool threadPool, diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index c016e5a282462..44fca8396525c 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -11,7 +11,6 @@ import org.apache.lucene.search.BooleanQuery; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.NamedRegistry; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.geo.GeoShapeType; import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -21,6 +20,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; @@ -257,7 +257,6 @@ import java.util.function.Consumer; import java.util.function.Function; -import static java.util.Collections.emptyList; import static java.util.Collections.unmodifiableMap; import static java.util.Objects.requireNonNull; @@ -842,6 +841,10 @@ private void registerQueryParsers(List plugins) { } registerFromPlugin(plugins, SearchPlugin::getQueries, this::registerQuery); + + if(RestApiVersion.minimumSupported() == RestApiVersion.V_7) { + registerQuery(new QuerySpec<>(TypeQueryV7Builder.NAME_V7, TypeQueryV7Builder::new, TypeQueryV7Builder::fromXContent)); + } } private void registerIntervalsSourceProviders() { @@ -884,7 +887,7 @@ public static List getIntervalsSourceProviderNamed private void registerQuery(QuerySpec spec) { namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, spec.getName().getPreferredName(), spec.getReader())); namedXContents.add(new NamedXContentRegistry.Entry(QueryBuilder.class, spec.getName(), - (p, c) -> spec.getParser().fromXContent(p))); + (p, c) -> spec.getParser().fromXContent(p), spec.getName().getForRestApiVersion())); } private void registerBoolQuery(ParseField name, Writeable.Reader reader) { @@ -897,11 +900,4 @@ public FetchPhase getFetchPhase() { return new FetchPhase(fetchSubPhases); } - public List getNamedXContentForCompatibility() { - if (RestApiVersion.minimumSupported() == RestApiVersion.V_7) { - return List.of( - new NamedXContentRegistry.Entry(QueryBuilder.class, TypeQueryV7Builder.NAME_V7, TypeQueryV7Builder::fromXContent)); - } - return emptyList(); - } } diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java index 72d6cf0797552..a44f994c61fe6 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.test.rest.FakeRestRequest; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -57,6 +56,8 @@ static class SubObject { private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "parser1", false, a -> new SubObject((String) a[0])); + static final ParseField NAME = new ParseField("namedObjectName1") + .forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.current())); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("new_field")); @@ -77,7 +78,8 @@ static class OldSubObject extends SubObject { private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "parser2", false, a -> new SubObject((String) a[0])); - + static final ParseField NAME = new ParseField("namedObjectName1") + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.minimumSupported())); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("old_field")); } @@ -93,8 +95,10 @@ public static SubObject parse(XContentParser parser) { public void testNotCompatibleRequest() throws IOException { NamedXContentRegistry registry = new NamedXContentRegistry( - Arrays.asList(new NamedXContentRegistry.Entry(SubObject.class, new ParseField("namedObjectName1"), SubObject::parse)), - Arrays.asList(new NamedXContentRegistry.Entry(SubObject.class, new ParseField("namedObjectName1"), OldSubObject::parse))); + List.of(new NamedXContentRegistry.Entry(SubObject.class, SubObject.NAME, SubObject::parse, + SubObject.NAME.getForRestApiVersion()), + new NamedXContentRegistry.Entry(SubObject.class, OldSubObject.NAME, OldSubObject::parse, + OldSubObject.NAME.getForRestApiVersion()))); XContentBuilder b = XContentBuilder.builder(XContentType.JSON.xContent()); b.startObject(); @@ -124,9 +128,11 @@ public void testNotCompatibleRequest() throws IOException { } public void testCompatibleRequest() throws IOException { - NamedXContentRegistry compatibleRegistry = new NamedXContentRegistry( - Arrays.asList(new NamedXContentRegistry.Entry(SubObject.class, new ParseField("namedObjectName1"), SubObject::parse)), - Arrays.asList(new NamedXContentRegistry.Entry(SubObject.class, new ParseField("namedObjectName1"), OldSubObject::parse))); + NamedXContentRegistry registry = new NamedXContentRegistry( + List.of(new NamedXContentRegistry.Entry(SubObject.class, SubObject.NAME, SubObject::parse, + SubObject.NAME.getForRestApiVersion()), + new NamedXContentRegistry.Entry(SubObject.class, OldSubObject.NAME, OldSubObject::parse, + OldSubObject.NAME.getForRestApiVersion()))); XContentBuilder b = XContentBuilder.builder(XContentType.JSON.xContent()); b.startObject(); @@ -141,7 +147,7 @@ public void testCompatibleRequest() throws IOException { String.valueOf(RestApiVersion.minimumSupported().major))); List mediaTypeList = Collections.singletonList(mediaType); - RestRequest restRequest2 = new FakeRestRequest.Builder(compatibleRegistry) + RestRequest restRequest2 = new FakeRestRequest.Builder(registry) .withContent(BytesReference.bytes(b), RestRequest.parseContentType(mediaTypeList)) .withPath("/foo") .withHeaders(Map.of("Content-Type", mediaTypeList, "Accept", mediaTypeList)) diff --git a/server/src/test/java/org/elasticsearch/node/NodeTests.java b/server/src/test/java/org/elasticsearch/node/NodeTests.java index 422c722f4c7b3..b97b8dc33fe40 100644 --- a/server/src/test/java/org/elasticsearch/node/NodeTests.java +++ b/server/src/test/java/org/elasticsearch/node/NodeTests.java @@ -12,14 +12,14 @@ import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.bootstrap.BootstrapContext; import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.xcontent.ContextParser; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ParseField; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine.Searcher; @@ -29,7 +29,6 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.plugins.CircuitBreakerPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.MockHttpTransport; @@ -45,6 +44,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; @@ -383,7 +383,8 @@ public void testLoadingMultipleRestCompatibilityPlugins() throws IOException { plugins.add(TestRestCompatibility1.class); try (Node node = new MockNode(settings.build(), plugins)) { - List compatibleNamedXContents = node.getCompatibleNamedXContents(mock(SearchModule.class)); + List compatibleNamedXContents = node.getCompatibleNamedXContents() + .collect(Collectors.toList()); assertThat(compatibleNamedXContents, contains(v7CompatibleEntries)); } } @@ -398,7 +399,8 @@ public void testLoadingMultipleRestCompatibilityPlugins() throws IOException { plugins.add(TestRestCompatibility1.class); try (Node node = new MockNode(settings.build(), plugins)) { - List compatibleNamedXContents = node.getCompatibleNamedXContents(mock(SearchModule.class)); + List compatibleNamedXContents = node.getCompatibleNamedXContents() + .collect(Collectors.toList());; assertThat(compatibleNamedXContents, contains(v8CompatibleEntries)); } } From 58e29abc70a0c11b138ecff26841f0e62e5b6da5 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 22 Jul 2021 10:48:08 +0200 Subject: [PATCH 04/16] failing test --- rest-api-spec/build.gradle | 1 + .../340_type_query_indices_created_on_8.yml | 139 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index ad75cd14f122e..11c78475e18a0 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -94,6 +94,7 @@ tasks.named("yamlRestCompatTest").configure { 'indices.flush/10_basic/Index synced flush rest test', 'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers',//#42809 nested path and nested filter 'search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception', //cutoff_frequency + 'search/340_type_query/type query' // type_query - probably should behave like match_all ] + v7compatibilityNotSupportedTests()) .join(',') diff --git a/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml b/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml new file mode 100644 index 0000000000000..3aceb9868765d --- /dev/null +++ b/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml @@ -0,0 +1,139 @@ +--- +setup: + - skip: + features: + - "headers" + - "allowed_warnings_regex" +--- +type query: + - do: + index: + index: "test1" + id: 1 + type: "cat" + refresh: true + body: + foo: "bar" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - do: + index: + index: "test2" + id: 1 + type: "_doc" + refresh: true + body: + foo: "bar" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - do: + search: + rest_total_hits_as_int: true + index: "test1" + body: + query: + type: + value: "cat" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 1 + - do: + search: + rest_total_hits_as_int: true + index: "test1" + body: + query: + type: + value: "dog" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 10 + - do: + search: + rest_total_hits_as_int: true + index: "test1" + body: + query: + type: + value: "_doc" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 1 + - do: + search: + rest_total_hits_as_int: true + index: "test1" + body: + query: + type: + value: "_default_" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 0 + - do: + search: + rest_total_hits_as_int: true + index: "test2" + body: + query: + type: + value: "_doc" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 1 + - do: + search: + rest_total_hits_as_int: true + index: "test2" + body: + query: + type: + value: "dog" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 2 + - do: + search: + rest_total_hits_as_int: true + index: "test2" + body: + query: + type: + value: "_default_" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 2 From a4f506a619b98a34e33c244bead1ae253ab99354 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 22 Jul 2021 11:23:41 +0200 Subject: [PATCH 05/16] rolling upgraded failing --- .../test/old_cluster/20_type_query.yml | 34 ++++++ .../test/upgraded_cluster/20_type_query.yml | 113 ++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_type_query.yml create mode 100644 qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_type_query.yml diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_type_query.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_type_query.yml new file mode 100644 index 0000000000000..109e1674a12c6 --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_type_query.yml @@ -0,0 +1,34 @@ +--- +setup: + - skip: + features: + - "headers" + - "allowed_warnings_regex" +--- +create indices for type query: + - do: + index: + index: "test1" + id: 1 + type: "cat" + refresh: true + body: + foo: "bar" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - do: + index: + index: "test2" + id: 1 + type: "_doc" + refresh: true + body: + foo: "bar" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_type_query.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_type_query.yml new file mode 100644 index 0000000000000..8ceed7786e5c4 --- /dev/null +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_type_query.yml @@ -0,0 +1,113 @@ +--- +setup: + - skip: + features: + - "headers" + - "allowed_warnings_regex" +--- +run type query: + - do: + search: + rest_total_hits_as_int: true + index: "test1" + body: + query: + type: + value: "cat" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 1 + - do: + search: + rest_total_hits_as_int: true + index: "test1" + body: + query: + type: + value: "dog" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 0 + - do: + search: + rest_total_hits_as_int: true + index: "test1" + body: + query: + type: + value: "_doc" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 0 + - do: + search: + rest_total_hits_as_int: true + index: "test1" + body: + query: + type: + value: "_default_" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 0 + - do: + search: + rest_total_hits_as_int: true + index: "test2" + body: + query: + type: + value: "_doc" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 1 + - do: + search: + rest_total_hits_as_int: true + index: "test2" + body: + query: + type: + value: "dog" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 0 + - do: + search: + rest_total_hits_as_int: true + index: "test2" + body: + query: + type: + value: "_default_" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - match: + hits.total: 1 From 4b23f2809050740b73d2c26ba4e79b3acd9a69e3 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 22 Jul 2021 12:16:26 +0200 Subject: [PATCH 06/16] full cluster test --- .../upgrades/FullClusterRestartIT.java | 34 ++++++ .../test/old_cluster/20_type_query.yml | 34 ------ .../test/upgraded_cluster/20_type_query.yml | 113 ------------------ .../340_type_query_indices_created_on_8.yml | 8 +- 4 files changed, 38 insertions(+), 151 deletions(-) delete mode 100644 qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_type_query.yml delete mode 100644 qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_type_query.yml diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 1c4c972b83347..e44d16bfca8ef 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -1625,6 +1625,40 @@ public void testTransportCompressionSetting() throws IOException { } } + + public void testTypeQuery() throws IOException { + if (isRunningAgainstOldCluster()) { + String doc = "{\"foo\": \"bar\"}"; + Request createDoc = new Request("PUT", "/test_type_query_1/cat/1"); + createDoc.addParameter("refresh", "true"); + createDoc.setJsonEntity(doc); + client().performRequest(createDoc); + + createDoc = new Request("PUT", "/test_type_query_2/_doc/1"); + createDoc.addParameter("refresh", "true"); + createDoc.setJsonEntity(doc); + client().performRequest(createDoc); + } else { + assertTypeQueryHits("test_type_query_1", "cat", 1); + assertTypeQueryHits("test_type_query_1", "dog", 0); + assertTypeQueryHits("test_type_query_1", "_doc", 0); + assertTypeQueryHits("test_type_query_1", "_default_", 0); + + assertTypeQueryHits("test_type_query_2", "_doc", 1); + assertTypeQueryHits("test_type_query_2", "dog", 0); + assertTypeQueryHits("test_type_query_2", "_default_", 0); + + } + } + + private void assertTypeQueryHits(String index, String type, int numHits) throws IOException { + Request searchRequest = new Request("GET", "/" + index + "/_search?rest_total_hits_as_int=true"); + searchRequest.setJsonEntity("{ \"query\": { \"type\" : {\"value\": \"" + type + "\"} }}"); + Map response = entityAsMap(client().performRequest(searchRequest)); + assertNoFailures(response); + assertThat(extractTotalHits(response), equalTo(numHits)); + } + public static void assertNumHits(String index, int numHits, int totalShards) throws IOException { Map resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))); assertNoFailures(resp); diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_type_query.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_type_query.yml deleted file mode 100644 index 109e1674a12c6..0000000000000 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/20_type_query.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -setup: - - skip: - features: - - "headers" - - "allowed_warnings_regex" ---- -create indices for type query: - - do: - index: - index: "test1" - id: 1 - type: "cat" - refresh: true - body: - foo: "bar" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - do: - index: - index: "test2" - id: 1 - type: "_doc" - refresh: true - body: - foo: "bar" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_type_query.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_type_query.yml deleted file mode 100644 index 8ceed7786e5c4..0000000000000 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/20_type_query.yml +++ /dev/null @@ -1,113 +0,0 @@ ---- -setup: - - skip: - features: - - "headers" - - "allowed_warnings_regex" ---- -run type query: - - do: - search: - rest_total_hits_as_int: true - index: "test1" - body: - query: - type: - value: "cat" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 1 - - do: - search: - rest_total_hits_as_int: true - index: "test1" - body: - query: - type: - value: "dog" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 0 - - do: - search: - rest_total_hits_as_int: true - index: "test1" - body: - query: - type: - value: "_doc" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 0 - - do: - search: - rest_total_hits_as_int: true - index: "test1" - body: - query: - type: - value: "_default_" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 0 - - do: - search: - rest_total_hits_as_int: true - index: "test2" - body: - query: - type: - value: "_doc" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 1 - - do: - search: - rest_total_hits_as_int: true - index: "test2" - body: - query: - type: - value: "dog" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 0 - - do: - search: - rest_total_hits_as_int: true - index: "test2" - body: - query: - type: - value: "_default_" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 1 diff --git a/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml b/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml index 3aceb9868765d..d2736064ab016 100644 --- a/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml +++ b/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml @@ -46,7 +46,7 @@ type query: allowed_warnings_regex: - "\\[types removal\\].*" - match: - hits.total: 1 + hits.total: 0 - do: search: rest_total_hits_as_int: true @@ -61,7 +61,7 @@ type query: allowed_warnings_regex: - "\\[types removal\\].*" - match: - hits.total: 10 + hits.total: 0 - do: search: rest_total_hits_as_int: true @@ -121,7 +121,7 @@ type query: allowed_warnings_regex: - "\\[types removal\\].*" - match: - hits.total: 2 + hits.total: 0 - do: search: rest_total_hits_as_int: true @@ -136,4 +136,4 @@ type query: allowed_warnings_regex: - "\\[types removal\\].*" - match: - hits.total: 2 + hits.total: 0 From 3b11f6e29d1887747384c8a54b989287aca48ab7 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 22 Jul 2021 16:13:33 +0200 Subject: [PATCH 07/16] tests --- .../upgrades/FullClusterRestartIT.java | 25 +++++++++++++++++-- .../index/query/TypeQueryV7Builder.java | 6 +++++ .../rest/action/document/RestIndexAction.java | 2 +- .../search/SearchModuleTests.java | 7 ++++++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index e44d16bfca8ef..89ed12b3e63fc 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -8,6 +8,9 @@ package org.elasticsearch.upgrades; +import org.apache.http.entity.ContentType; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.nio.entity.NStringEntity; import org.apache.http.util.EntityUtils; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsResponse; @@ -19,14 +22,19 @@ import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ParsedMediaType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.query.TypeQueryV7Builder; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; +import org.elasticsearch.rest.action.document.RestIndexAction; import org.elasticsearch.test.NotEqualMessageBuilder; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.test.rest.ESRestTestCase; @@ -1632,11 +1640,13 @@ public void testTypeQuery() throws IOException { Request createDoc = new Request("PUT", "/test_type_query_1/cat/1"); createDoc.addParameter("refresh", "true"); createDoc.setJsonEntity(doc); + createDoc.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); client().performRequest(createDoc); createDoc = new Request("PUT", "/test_type_query_2/_doc/1"); createDoc.addParameter("refresh", "true"); createDoc.setJsonEntity(doc); + createDoc.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); client().performRequest(createDoc); } else { assertTypeQueryHits("test_type_query_1", "cat", 1); @@ -1652,8 +1662,19 @@ public void testTypeQuery() throws IOException { } private void assertTypeQueryHits(String index, String type, int numHits) throws IOException { - Request searchRequest = new Request("GET", "/" + index + "/_search?rest_total_hits_as_int=true"); - searchRequest.setJsonEntity("{ \"query\": { \"type\" : {\"value\": \"" + type + "\"} }}"); + final ParsedMediaType parsedMediaType = XContentType.VND_JSON.toParsedMediaType(); + final String v7MediaType = parsedMediaType + .responseContentTypeHeader(Map.of("compatible-with", String.valueOf(RestApiVersion.V_7.major))); + + final ContentType contentType = ContentType.create(parsedMediaType.mediaTypeWithoutParameters(), + new BasicNameValuePair("compatible-with", String.valueOf(RestApiVersion.V_7.major))); + + Request searchRequest = new Request("GET", "/" + index + "/_search"); + + searchRequest.setOptions(expectWarnings(TypeQueryV7Builder.TYPES_DEPRECATION_MESSAGE).toBuilder() + .addHeader("Accept", v7MediaType)); + searchRequest.setEntity(new NStringEntity("{ \"query\": { \"type\" : {\"value\": \"" + type + "\"} }}", contentType)); + Map response = entityAsMap(client().performRequest(searchRequest)); assertNoFailures(response); assertThat(extractTotalHits(response), equalTo(numHits)); diff --git a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java index 563fd3aa91f5e..374a501eb31a8 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -24,6 +25,10 @@ import java.io.IOException; public class TypeQueryV7Builder extends AbstractQueryBuilder { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(TypeQueryV7Builder.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Type queries are deprecated, " + + "prefer to filter on a field instead."; + private static final String NAME = "type"; public static final ParseField NAME_V7 = new ParseField(NAME).forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)); private static final ParseField VALUE_FIELD = new ParseField("value"); @@ -82,6 +87,7 @@ protected int doHashCode() { } public static TypeQueryV7Builder fromXContent(XContentParser parser) throws IOException { + deprecationLogger.compatibleApiWarning("type_query", TYPES_DEPRECATION_MESSAGE); try { return PARSER.apply(parser, null); } catch (IllegalArgumentException e) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index 021749bd0c0ee..10d2aefc5ce83 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -30,7 +30,7 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT; public class RestIndexAction extends BaseRestHandler { - static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " + "index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, " + "or /{index}/_create/{id})."; diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java index 05c965855a7fd..e16769ec9b3dc 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java @@ -17,10 +17,12 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TypeQueryV7Builder; import org.elasticsearch.index.query.functionscore.GaussDecayFunctionBuilder; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.AggregationBuilder; @@ -237,11 +239,13 @@ public void testRegisteredQueries() { List allSupportedQueries = new ArrayList<>(); Collections.addAll(allSupportedQueries, NON_DEPRECATED_QUERIES); Collections.addAll(allSupportedQueries, DEPRECATED_QUERIES); + Collections.addAll(allSupportedQueries, REST_COMPATIBLE_QUERIES); SearchModule module = new SearchModule(Settings.EMPTY, emptyList()); Set registeredNonDeprecated = module.getNamedXContents().stream() .filter(e -> e.categoryClass.equals(QueryBuilder.class)) .filter(e -> e.name.getAllReplacedWith() == null) + .filter(e -> RestApiVersion.current().matches(e.restApiCompatibility)) .map(e -> e.name.getPreferredName()) .collect(toSet()); Set registeredAll = module.getNamedXContents().stream() @@ -386,6 +390,9 @@ public CheckedBiConsumer getReque //add here deprecated queries to make sure we log a deprecation warnings when they are used private static final String[] DEPRECATED_QUERIES = new String[] {"field_masking_span", "geo_polygon"}; + //add here compatible queries + private static final String[] REST_COMPATIBLE_QUERIES = new String[] {TypeQueryV7Builder.NAME_V7.getPreferredName()}; + /** * Dummy test {@link AggregationBuilder} used to test registering aggregation builders. */ From 63e113bb110862b4d2452d513a3a0097e8f465da Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 22 Jul 2021 17:17:02 +0200 Subject: [PATCH 08/16] new test --- .../upgrades/FullClusterRestartIT.java | 61 --------------- .../upgrades/RestApiCompatibleUpgradeIT.java | 75 +++++++++++++++++++ .../AbstractFullClusterRestartTestCase.java | 15 ++++ 3 files changed, 90 insertions(+), 61 deletions(-) create mode 100644 qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 89ed12b3e63fc..bfe13b6e47d49 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -8,9 +8,6 @@ package org.elasticsearch.upgrades; -import org.apache.http.entity.ContentType; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.nio.entity.NStringEntity; import org.apache.http.util.EntityUtils; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsResponse; @@ -22,19 +19,14 @@ import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ParsedMediaType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.query.TypeQueryV7Builder; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; -import org.elasticsearch.rest.action.document.RestIndexAction; import org.elasticsearch.test.NotEqualMessageBuilder; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.test.rest.ESRestTestCase; @@ -607,20 +599,11 @@ static String toStr(Response response) throws IOException { return EntityUtils.toString(response.getEntity()); } - static void assertNoFailures(Map response) { - int failed = (int) XContentMapValues.extractValue("_shards.failed", response); - assertEquals(0, failed); - } - void assertTotalHits(int expectedTotalHits, Map response) { int actualTotalHits = extractTotalHits(response); assertEquals(response.toString(), expectedTotalHits, actualTotalHits); } - static int extractTotalHits(Map response) { - return (Integer) XContentMapValues.extractValue("hits.total.value", response); - } - /** * Tests that a single document survives. Super basic smoke test. */ @@ -1634,51 +1617,7 @@ public void testTransportCompressionSetting() throws IOException { } - public void testTypeQuery() throws IOException { - if (isRunningAgainstOldCluster()) { - String doc = "{\"foo\": \"bar\"}"; - Request createDoc = new Request("PUT", "/test_type_query_1/cat/1"); - createDoc.addParameter("refresh", "true"); - createDoc.setJsonEntity(doc); - createDoc.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); - client().performRequest(createDoc); - - createDoc = new Request("PUT", "/test_type_query_2/_doc/1"); - createDoc.addParameter("refresh", "true"); - createDoc.setJsonEntity(doc); - createDoc.setOptions(expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE)); - client().performRequest(createDoc); - } else { - assertTypeQueryHits("test_type_query_1", "cat", 1); - assertTypeQueryHits("test_type_query_1", "dog", 0); - assertTypeQueryHits("test_type_query_1", "_doc", 0); - assertTypeQueryHits("test_type_query_1", "_default_", 0); - - assertTypeQueryHits("test_type_query_2", "_doc", 1); - assertTypeQueryHits("test_type_query_2", "dog", 0); - assertTypeQueryHits("test_type_query_2", "_default_", 0); - - } - } - - private void assertTypeQueryHits(String index, String type, int numHits) throws IOException { - final ParsedMediaType parsedMediaType = XContentType.VND_JSON.toParsedMediaType(); - final String v7MediaType = parsedMediaType - .responseContentTypeHeader(Map.of("compatible-with", String.valueOf(RestApiVersion.V_7.major))); - - final ContentType contentType = ContentType.create(parsedMediaType.mediaTypeWithoutParameters(), - new BasicNameValuePair("compatible-with", String.valueOf(RestApiVersion.V_7.major))); - Request searchRequest = new Request("GET", "/" + index + "/_search"); - - searchRequest.setOptions(expectWarnings(TypeQueryV7Builder.TYPES_DEPRECATION_MESSAGE).toBuilder() - .addHeader("Accept", v7MediaType)); - searchRequest.setEntity(new NStringEntity("{ \"query\": { \"type\" : {\"value\": \"" + type + "\"} }}", contentType)); - - Map response = entityAsMap(client().performRequest(searchRequest)); - assertNoFailures(response); - assertThat(extractTotalHits(response), equalTo(numHits)); - } public static void assertNumHits(String index, int numHits, int totalShards) throws IOException { Map resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))); diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java new file mode 100644 index 0000000000000..c52beb4f50091 --- /dev/null +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.upgrades; + +import org.apache.http.entity.ContentType; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.nio.entity.NStringEntity; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.common.xcontent.ParsedMediaType; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.index.query.TypeQueryV7Builder; +import org.elasticsearch.rest.action.document.RestIndexAction; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class RestApiCompatibleUpgradeIT extends AbstractFullClusterRestartTestCase { + final ParsedMediaType parsedMediaType = XContentType.VND_JSON.toParsedMediaType(); + final String v7MediaType = parsedMediaType + .responseContentTypeHeader(Map.of("compatible-with", String.valueOf(RestApiVersion.V_7.major))); + final ContentType v7ContentType = ContentType.create(parsedMediaType.mediaTypeWithoutParameters(), + new BasicNameValuePair("compatible-with", String.valueOf(RestApiVersion.V_7.major))); + + + public void testTypeQuery() throws IOException { + if (isRunningAgainstOldCluster()) { + String doc = "{\"foo\": \"bar\"}"; + createIndexAndPostDoc("/test_type_query_1/cat/1", doc, true); + createIndexAndPostDoc("/test_type_query_2/_doc/1", doc, false); + } else { + assertTypeQueryHits("test_type_query_1", "cat", 1); + assertTypeQueryHits("test_type_query_1", "dog", 0); + assertTypeQueryHits("test_type_query_1", "_doc", 0); + assertTypeQueryHits("test_type_query_1", "_default_", 0); + + assertTypeQueryHits("test_type_query_2", "_doc", 1); + assertTypeQueryHits("test_type_query_2", "dog", 0); + assertTypeQueryHits("test_type_query_2", "_default_", 0); + + } + } + + private void createIndexAndPostDoc(String endpoint, String doc, boolean expectWarning) throws IOException { + Request createDoc = new Request("PUT", endpoint); + createDoc.addParameter("refresh", "true"); + RequestOptions options = expectWarning ? expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE) : RequestOptions.DEFAULT; + createDoc.setOptions(options.toBuilder() + .addHeader("Accept", v7MediaType)); + createDoc.setEntity(new NStringEntity(doc, v7ContentType)); + client().performRequest(createDoc); + } + + private void assertTypeQueryHits(String index, String type, int numHits) throws IOException { + Request searchRequest = new Request("GET", "/" + index + "/_search"); + + searchRequest.setOptions(expectWarnings(TypeQueryV7Builder.TYPES_DEPRECATION_MESSAGE).toBuilder() + .addHeader("Accept", v7MediaType)); + searchRequest.setEntity(new NStringEntity("{ \"query\": { \"type\" : {\"value\": \"" + type + "\"} }}", v7ContentType)); + + Map response = entityAsMap(client().performRequest(searchRequest)); + assertNoFailures(response); + assertThat(extractTotalHits(response), equalTo(numHits)); + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java index bd03a55a2eb07..f0fd56447d34c 100644 --- a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java @@ -9,9 +9,12 @@ package org.elasticsearch.upgrades; import org.elasticsearch.Version; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Booleans; import org.elasticsearch.test.rest.ESRestTestCase; +import java.util.Map; + public abstract class AbstractFullClusterRestartTestCase extends ESRestTestCase { private final boolean runningAgainstOldCluster = Booleans.parseBoolean(System.getProperty("tests.is_old_cluster")); @@ -70,4 +73,16 @@ protected boolean preserveSLMPoliciesUponCompletion() { protected boolean preserveDataStreamsUponCompletion() { return true; } + + + protected static void assertNoFailures(Map response) { + int failed = (int) XContentMapValues.extractValue("_shards.failed", response); + assertEquals(0, failed); + } + + + protected static int extractTotalHits(Map response) { + return (Integer) XContentMapValues.extractValue("hits.total.value", response); + } + } From f7bc8e4c75b8c68151869c2d494debb50032a43b Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 26 Jul 2021 12:43:06 +0200 Subject: [PATCH 09/16] test skip --- .../elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java index c52beb4f50091..dc5a594f1bc26 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java @@ -11,6 +11,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.message.BasicNameValuePair; import org.apache.http.nio.entity.NStringEntity; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.common.xcontent.ParsedMediaType; @@ -32,7 +33,9 @@ public class RestApiCompatibleUpgradeIT extends AbstractFullClusterRestartTestCa new BasicNameValuePair("compatible-with", String.valueOf(RestApiVersion.V_7.major))); - public void testTypeQuery() throws IOException { + public void testTypeQueryReturnsTypeWhen7xCreatedIndices() throws IOException { + assumeTrue("Old cluster has to be in 7.x", getOldClusterVersion().before(Version.V_8_0_0)); + if (isRunningAgainstOldCluster()) { String doc = "{\"foo\": \"bar\"}"; createIndexAndPostDoc("/test_type_query_1/cat/1", doc, true); From 5f2f3cb977e5520d9683c509f4b63c353caed7bb Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 26 Jul 2021 13:58:32 +0200 Subject: [PATCH 10/16] cleanup --- .../elasticsearch/upgrades/FullClusterRestartIT.java | 3 --- .../upgrades/RestApiCompatibleUpgradeIT.java | 2 -- rest-api-spec/build.gradle | 10 ++++++++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index bfe13b6e47d49..738bc6a1a6789 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -1616,9 +1616,6 @@ public void testTransportCompressionSetting() throws IOException { } } - - - public static void assertNumHits(String index, int numHits, int totalShards) throws IOException { Map resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))); assertNoFailures(resp); diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java index dc5a594f1bc26..31f3515f59a82 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java @@ -49,7 +49,6 @@ public void testTypeQueryReturnsTypeWhen7xCreatedIndices() throws IOException { assertTypeQueryHits("test_type_query_2", "_doc", 1); assertTypeQueryHits("test_type_query_2", "dog", 0); assertTypeQueryHits("test_type_query_2", "_default_", 0); - } } @@ -74,5 +73,4 @@ private void assertTypeQueryHits(String index, String type, int numHits) throws assertNoFailures(response); assertThat(extractTotalHits(response), equalTo(numHits)); } - } diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 11c78475e18a0..615a6069e8e40 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -80,7 +80,14 @@ def v7compatibilityNotSupportedTests = { 'field_caps/30_filter/Field caps with index filter', //behaviour change after #63692 4digits dates are parsed as epoch and in quotes as year - 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', //#44761 bug fix + 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', //#44761 bug fix, + + /* + This test relies on indices created in 7.x hence won't work in 8.x cluster. + There is a new RestApiCompatibleUpgradeIT#testTypeQueryReturnsTypeWhen7xCreatedIndices that covers it + */ + 'search/340_type_query/type query' + ] } tasks.named("yamlRestCompatTest").configure { @@ -94,7 +101,6 @@ tasks.named("yamlRestCompatTest").configure { 'indices.flush/10_basic/Index synced flush rest test', 'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers',//#42809 nested path and nested filter 'search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception', //cutoff_frequency - 'search/340_type_query/type query' // type_query - probably should behave like match_all ] + v7compatibilityNotSupportedTests()) .join(',') From 0b425a65b066887015cc60dfba9f2bb5558eebdf Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 27 Jul 2021 10:06:43 +0200 Subject: [PATCH 11/16] exception --- .../org/elasticsearch/index/query/TypeQueryV7Builder.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java index 374a501eb31a8..fcc4b5692aa4d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java @@ -88,11 +88,7 @@ protected int doHashCode() { public static TypeQueryV7Builder fromXContent(XContentParser parser) throws IOException { deprecationLogger.compatibleApiWarning("type_query", TYPES_DEPRECATION_MESSAGE); - try { - return PARSER.apply(parser, null); - } catch (IllegalArgumentException e) { - throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e); - } + throw new ParsingException(parser.getTokenLocation(), TYPES_DEPRECATION_MESSAGE); } @Override From 36fc16c1124c5e8937a06ad5bc96ca52f2037a3e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 27 Jul 2021 15:19:32 +0200 Subject: [PATCH 12/16] tests --- .../upgrades/RestApiCompatibleUpgradeIT.java | 76 --------------- .../340_type_query_indices_created_on_8.yml | 97 +------------------ .../index/query/TypeQueryV7Builder.java | 4 - 3 files changed, 5 insertions(+), 172 deletions(-) delete mode 100644 qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java deleted file mode 100644 index 31f3515f59a82..0000000000000 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/RestApiCompatibleUpgradeIT.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.upgrades; - -import org.apache.http.entity.ContentType; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.nio.entity.NStringEntity; -import org.elasticsearch.Version; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.common.xcontent.ParsedMediaType; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.index.query.TypeQueryV7Builder; -import org.elasticsearch.rest.action.document.RestIndexAction; - -import java.io.IOException; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; - -public class RestApiCompatibleUpgradeIT extends AbstractFullClusterRestartTestCase { - final ParsedMediaType parsedMediaType = XContentType.VND_JSON.toParsedMediaType(); - final String v7MediaType = parsedMediaType - .responseContentTypeHeader(Map.of("compatible-with", String.valueOf(RestApiVersion.V_7.major))); - final ContentType v7ContentType = ContentType.create(parsedMediaType.mediaTypeWithoutParameters(), - new BasicNameValuePair("compatible-with", String.valueOf(RestApiVersion.V_7.major))); - - - public void testTypeQueryReturnsTypeWhen7xCreatedIndices() throws IOException { - assumeTrue("Old cluster has to be in 7.x", getOldClusterVersion().before(Version.V_8_0_0)); - - if (isRunningAgainstOldCluster()) { - String doc = "{\"foo\": \"bar\"}"; - createIndexAndPostDoc("/test_type_query_1/cat/1", doc, true); - createIndexAndPostDoc("/test_type_query_2/_doc/1", doc, false); - } else { - assertTypeQueryHits("test_type_query_1", "cat", 1); - assertTypeQueryHits("test_type_query_1", "dog", 0); - assertTypeQueryHits("test_type_query_1", "_doc", 0); - assertTypeQueryHits("test_type_query_1", "_default_", 0); - - assertTypeQueryHits("test_type_query_2", "_doc", 1); - assertTypeQueryHits("test_type_query_2", "dog", 0); - assertTypeQueryHits("test_type_query_2", "_default_", 0); - } - } - - private void createIndexAndPostDoc(String endpoint, String doc, boolean expectWarning) throws IOException { - Request createDoc = new Request("PUT", endpoint); - createDoc.addParameter("refresh", "true"); - RequestOptions options = expectWarning ? expectWarnings(RestIndexAction.TYPES_DEPRECATION_MESSAGE) : RequestOptions.DEFAULT; - createDoc.setOptions(options.toBuilder() - .addHeader("Accept", v7MediaType)); - createDoc.setEntity(new NStringEntity(doc, v7ContentType)); - client().performRequest(createDoc); - } - - private void assertTypeQueryHits(String index, String type, int numHits) throws IOException { - Request searchRequest = new Request("GET", "/" + index + "/_search"); - - searchRequest.setOptions(expectWarnings(TypeQueryV7Builder.TYPES_DEPRECATION_MESSAGE).toBuilder() - .addHeader("Accept", v7MediaType)); - searchRequest.setEntity(new NStringEntity("{ \"query\": { \"type\" : {\"value\": \"" + type + "\"} }}", v7ContentType)); - - Map response = entityAsMap(client().performRequest(searchRequest)); - assertNoFailures(response); - assertThat(extractTotalHits(response), equalTo(numHits)); - } -} diff --git a/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml b/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml index d2736064ab016..b81c060ed38d7 100644 --- a/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml +++ b/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml @@ -19,20 +19,9 @@ type query: Accept: "application/vnd.elasticsearch+json;compatible-with=7" allowed_warnings_regex: - "\\[types removal\\].*" + - do: - index: - index: "test2" - id: 1 - type: "_doc" - refresh: true - body: - foo: "bar" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - do: + catch: /\[types removal\] Type queries are deprecated, prefer to filter on a field instead./ search: rest_total_hits_as_int: true index: "test1" @@ -45,24 +34,9 @@ type query: Accept: "application/vnd.elasticsearch+json;compatible-with=7" allowed_warnings_regex: - "\\[types removal\\].*" - - match: - hits.total: 0 - - do: - search: - rest_total_hits_as_int: true - index: "test1" - body: - query: - type: - value: "dog" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 0 + - do: + catch: /\[types removal\] Type queries are deprecated, prefer to filter on a field instead./ search: rest_total_hits_as_int: true index: "test1" @@ -75,65 +49,4 @@ type query: Accept: "application/vnd.elasticsearch+json;compatible-with=7" allowed_warnings_regex: - "\\[types removal\\].*" - - match: - hits.total: 1 - - do: - search: - rest_total_hits_as_int: true - index: "test1" - body: - query: - type: - value: "_default_" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 0 - - do: - search: - rest_total_hits_as_int: true - index: "test2" - body: - query: - type: - value: "_doc" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 1 - - do: - search: - rest_total_hits_as_int: true - index: "test2" - body: - query: - type: - value: "dog" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 0 - - do: - search: - rest_total_hits_as_int: true - index: "test2" - body: - query: - type: - value: "_default_" - headers: - Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" - Accept: "application/vnd.elasticsearch+json;compatible-with=7" - allowed_warnings_regex: - - "\\[types removal\\].*" - - match: - hits.total: 0 + diff --git a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java index fcc4b5692aa4d..0ddf0a2400373 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java @@ -69,10 +69,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep @Override protected Query doToQuery(SearchExecutionContext context) throws IOException { - final String typeName = context.getMappingLookup().getMapping().getRoot().name(); - if (value.equals(typeName)) { - return new MatchAllDocsQuery(); - } return new MatchNoDocsQuery(); } From 682a2e71dc8d545600fc7e882ddefbd0cc66a496 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 27 Jul 2021 16:10:24 +0200 Subject: [PATCH 13/16] precommit --- .../java/org/elasticsearch/index/query/TypeQueryV7Builder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java index 0ddf0a2400373..915662f4e9893 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TypeQueryV7Builder.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.query; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.ParsingException; From e07570722525f4d71b34c9940962a9dcaf5d5cf4 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 3 Aug 2021 11:20:10 +0200 Subject: [PATCH 14/16] remove unused code --- .../upgrades/FullClusterRestartIT.java | 9 +++++++++ ...indices_created_on_8.yml => 10_type_query.yml} | 2 +- .../org/elasticsearch/index/mapper/Mapping.java | 2 +- .../index/query/SearchExecutionContext.java | 4 ---- .../org/elasticsearch/search/SearchModule.java | 2 +- .../AbstractFullClusterRestartTestCase.java | 15 --------------- 6 files changed, 12 insertions(+), 22 deletions(-) rename rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/{340_type_query_indices_created_on_8.yml => 10_type_query.yml} (97%) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 7a8810ebc2856..d6dcd01a3e053 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -599,11 +599,20 @@ static String toStr(Response response) throws IOException { return EntityUtils.toString(response.getEntity()); } + static void assertNoFailures(Map response) { + int failed = (int) XContentMapValues.extractValue("_shards.failed", response); + assertEquals(0, failed); + } + void assertTotalHits(int expectedTotalHits, Map response) { int actualTotalHits = extractTotalHits(response); assertEquals(response.toString(), expectedTotalHits, actualTotalHits); } + static int extractTotalHits(Map response) { + return (Integer) XContentMapValues.extractValue("hits.total.value", response); + } + /** * Tests that a single document survives. Super basic smoke test. */ diff --git a/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml b/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/10_type_query.yml similarity index 97% rename from rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml rename to rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/10_type_query.yml index b81c060ed38d7..fa4e20fdfa6fe 100644 --- a/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/340_type_query_indices_created_on_8.yml +++ b/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/10_type_query.yml @@ -5,7 +5,7 @@ setup: - "headers" - "allowed_warnings_regex" --- -type query: +type query throws exception when used: - do: index: index: "test1" diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java index 91321c4907480..4e1c0037e9148 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -80,7 +80,7 @@ public CompressedXContent toCompressedXContent() { /** * Returns the root object for the current mapping */ - public RootObjectMapper getRoot() { + RootObjectMapper getRoot() { return root; } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 5489fdf13fe4a..6eb8abe7dcdca 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -661,8 +661,4 @@ public String getNestedParent(String nestedPath) { public NestedDocuments getNestedDocuments() { return new NestedDocuments(mappingLookup, bitsetFilterCache::getBitSetProducer); } - - public MappingLookup getMappingLookup() { - return mappingLookup; - } } diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index e6b454d71134d..b1ecd5f61460d 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -842,7 +842,7 @@ private void registerQueryParsers(List plugins) { registerFromPlugin(plugins, SearchPlugin::getQueries, this::registerQuery); - if(RestApiVersion.minimumSupported() == RestApiVersion.V_7) { + if (RestApiVersion.minimumSupported() == RestApiVersion.V_7) { registerQuery(new QuerySpec<>(TypeQueryV7Builder.NAME_V7, TypeQueryV7Builder::new, TypeQueryV7Builder::fromXContent)); } } diff --git a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java index f0fd56447d34c..bd03a55a2eb07 100644 --- a/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/upgrades/AbstractFullClusterRestartTestCase.java @@ -9,12 +9,9 @@ package org.elasticsearch.upgrades; import org.elasticsearch.Version; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Booleans; import org.elasticsearch.test.rest.ESRestTestCase; -import java.util.Map; - public abstract class AbstractFullClusterRestartTestCase extends ESRestTestCase { private final boolean runningAgainstOldCluster = Booleans.parseBoolean(System.getProperty("tests.is_old_cluster")); @@ -73,16 +70,4 @@ protected boolean preserveSLMPoliciesUponCompletion() { protected boolean preserveDataStreamsUponCompletion() { return true; } - - - protected static void assertNoFailures(Map response) { - int failed = (int) XContentMapValues.extractValue("_shards.failed", response); - assertEquals(0, failed); - } - - - protected static int extractTotalHits(Map response) { - return (Integer) XContentMapValues.extractValue("hits.total.value", response); - } - } From aec8676d06e804af03d94cd9049ad83fe1798456 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 3 Aug 2021 11:23:29 +0200 Subject: [PATCH 15/16] scope and javadoc --- rest-api-spec/build.gradle | 6 +----- .../elasticsearch/rest/action/document/RestIndexAction.java | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 7144a444ffa48..2e80aa2b16218 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -82,11 +82,7 @@ def v7compatibilityNotSupportedTests = { 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', //#44761 bug fix, - /* - This test relies on indices created in 7.x hence won't work in 8.x cluster. - There is a new RestApiCompatibleUpgradeIT#testTypeQueryReturnsTypeWhen7xCreatedIndices that covers it - */ - 'search/340_type_query/type query' + 'search/340_type_query/type query' //#47207 type query throws exception in compatible mode ] } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index 10d2aefc5ce83..021749bd0c0ee 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -30,7 +30,7 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT; public class RestIndexAction extends BaseRestHandler { - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " + static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in document " + "index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, " + "or /{index}/_create/{id})."; From 0ee3b1cbfefea840a95f6b2fc5ca50af6ddbafbe Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 3 Aug 2021 11:59:22 +0200 Subject: [PATCH 16/16] add type query to searchmodule tests --- .../org/elasticsearch/search/SearchModuleTests.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java index 9241b803acabe..e74dd18e767b3 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.TypeQueryV7Builder; import org.elasticsearch.index.query.functionscore.GaussDecayFunctionBuilder; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.AggregationBuilder; @@ -241,11 +242,14 @@ public void testRegisteredQueries() { List allSupportedQueries = new ArrayList<>(); Collections.addAll(allSupportedQueries, NON_DEPRECATED_QUERIES); Collections.addAll(allSupportedQueries, DEPRECATED_QUERIES); + Collections.addAll(allSupportedQueries, REST_COMPATIBLE_QUERIES); + SearchModule module = new SearchModule(Settings.EMPTY, emptyList()); Set registeredNonDeprecated = module.getNamedXContents().stream() .filter(e -> e.categoryClass.equals(QueryBuilder.class)) .filter(e -> e.name.getAllReplacedWith() == null) + .filter(e -> RestApiVersion.current().matches(e.restApiCompatibility)) .map(e -> e.name.getPreferredName()) .collect(toSet()); Set registeredAll = module.getNamedXContents().stream() @@ -389,6 +393,7 @@ public CheckedBiConsumer getReque //add here deprecated queries to make sure we log a deprecation warnings when they are used private static final String[] DEPRECATED_QUERIES = new String[] {"field_masking_span", "geo_polygon"}; + private static final String[] REST_COMPATIBLE_QUERIES = new String[] {TypeQueryV7Builder.NAME_V7.getPreferredName()}; /** * Dummy test {@link AggregationBuilder} used to test registering aggregation builders. @@ -669,14 +674,15 @@ public List> getQueries() { .filter(e -> RestApiVersion.minimumSupported().matches(e.restApiCompatibility)) .filter(e -> RestApiVersion.current().matches(e.restApiCompatibility)) .collect(toSet()), - hasSize(searchModule.getNamedXContents().size() - 1)); + hasSize(searchModule.getNamedXContents().size()- REST_COMPATIBLE_QUERIES.length - 1 )); final List compatEntry = searchModule.getNamedXContents().stream() .filter(e -> e.categoryClass.equals(QueryBuilder.class) && - e.name.match(CompatQueryBuilder.NAME_OLD.getPreferredName(), LoggingDeprecationHandler.INSTANCE)) + RestApiVersion.minimumSupported().matches(e.name.getForRestApiVersion()) // v7 compatbile + && RestApiVersion.current().matches(e.name.getForRestApiVersion()) == false) // but not v8 compatible .collect(toList()); - assertThat(compatEntry, hasSize(1)); + assertThat(compatEntry, hasSize(REST_COMPATIBLE_QUERIES.length + 1));//+1 because of registered in the test assertTrue(RestApiVersion.minimumSupported().matches(compatEntry.get(0).restApiCompatibility)); assertFalse(RestApiVersion.current().matches(compatEntry.get(0).restApiCompatibility)); }