diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle index b1d72e79d19e2..a174356eaa533 100644 --- a/modules/analysis-common/build.gradle +++ b/modules/analysis-common/build.gradle @@ -30,13 +30,11 @@ def v7compatibilityNotSupportedTests = { //marked as not needing compatible api 'indices.analyze/10_analyze/htmlStrip_deprecated', // Cleanup versioned deprecations in analysis #41560 'analysis-common/40_token_filters/delimited_payload_filter_error', //Remove preconfigured delimited_payload_filter #43686 - 'analysis-common/20_analyzers/standard_html_strip' // Cleanup versioned deprecations in analysis #41560 + 'analysis-common/20_analyzers/standard_html_strip', // Cleanup versioned deprecations in analysis #41560 + 'search.query/50_queries_with_synonyms/Test common terms query with stacked tokens', // #42654 - `common` query throws an exception ] } tasks.named("yamlRestCompatTest").configure { - systemProperty 'tests.rest.blacklist', ([ - 'search.query/50_queries_with_synonyms/Test common terms query with stacked tokens', // #42654 - not sure, remove `common` query - ]+ v7compatibilityNotSupportedTests()) - .join(',') + systemProperty 'tests.rest.blacklist', v7compatibilityNotSupportedTests().join(',') } diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index e8d32117a175a..fe1ca824479e2 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -84,6 +84,8 @@ def v7compatibilityNotSupportedTests = { 'search/340_type_query/type query', //#47207 type query throws exception in compatible mode 'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers', // #42809 the use nested path and filter sort throws an exception + 'search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception', //#42654 cutoff_frequency, common terms are not supported. Throwing an exception + ] } @@ -92,11 +94,7 @@ tasks.named("yamlRestCompatTest").configure { // Skip these tests on Windows since the blacklist exceeds Windows CLI limits OS.current() != OS.WINDOWS } - systemProperty 'tests.rest.blacklist', ([ - 'search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception', //cutoff_frequency - ] + v7compatibilityNotSupportedTests()) - .join(',') - + systemProperty 'tests.rest.blacklist', v7compatibilityNotSupportedTests().join(',') } tasks.named("transformV7RestTests").configure({ task -> diff --git a/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/10_cutoff_frequency.yml b/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/10_cutoff_frequency.yml new file mode 100644 index 0000000000000..2d645a9419171 --- /dev/null +++ b/rest-api-spec/src/yamlRestCompatTest/resources/rest-api-spec/test/v7compat/search/10_cutoff_frequency.yml @@ -0,0 +1,105 @@ +--- +setup: + - skip: + version: "9.0.0 - " + reason: "compatible from 8.x to 7.x" + features: + - "headers" + - "allowed_warnings_regex" + - do: + indices.create: + index: "test" + body: + mappings: + properties: + my_field1: + type: "text" + my_field2: + type: "text" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - do: + index: + index: "test" + id: 1 + body: + my_field1: "brown fox jump" + my_field2: "xylophone" + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + - do: + indices.refresh: {} + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + +--- +multi_match multiple fields with cutoff_frequency throws exception: +- do: + catch: "/cutoff_freqency is not supported. The \\[multi_match\\] query can skip block of documents efficiently if the total number of hits is not tracked/" + search: + rest_total_hits_as_int: true + index: "test" + body: + query: + multi_match: + query: "brown" + type: "bool_prefix" + fields: + - "my_field1" + - "my_field2" + cutoff_frequency: 0.001 + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + +--- +match with cutoff_frequency throws exception: + - do: + catch: "/cutoff_freqency is not supported. The \\[match\\] query can skip block of documents efficiently if the total number of hits is not tracked/" + search: + rest_total_hits_as_int: true + index: "test" + body: + query: + match: + my_field1: + query: "brown" + type: "bool_prefix" + cutoff_frequency: 0.001 + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" + +--- +common querythrows exception: + - do: + catch: "/Common Terms Query usage is not supported. Use \\[match\\] query which can efficiently skip blocks of documents if the total number of hits is not tracked./" + search: + rest_total_hits_as_int: true + index: "test" + body: + query: + common: + my_field1: + query: "brown" + type: "bool_prefix" + cutoff_frequency: 0.001 + low_freq_operator: or + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + allowed_warnings_regex: + - "\\[types removal\\].*" diff --git a/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java new file mode 100644 index 0000000000000..300a66b0e93c2 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.query; + +import org.apache.lucene.search.Query; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.xcontent.ParseField; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.core.RestApiVersion; + +import java.io.IOException; + +public class CommonTermsQueryBuilder extends AbstractQueryBuilder { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(CommonTermsQueryBuilder.class); + public static final String COMMON_TERMS_QUERY_DEPRECATION_MSG = "Common Terms Query usage is not supported. " + + "Use [match] query which can efficiently skip blocks of documents if the total number of hits is not tracked."; + + public static ParseField NAME_V7 = new ParseField("common") + .withAllDeprecated(COMMON_TERMS_QUERY_DEPRECATION_MSG) + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)); + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + throw new UnsupportedOperationException("common_term_query is not meant to be serialized."); + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + } + + @Override + protected Query doToQuery(SearchExecutionContext context) throws IOException { + return null; + } + + @Override + protected boolean doEquals(CommonTermsQueryBuilder other) { + return false; + } + + @Override + protected int doHashCode() { + return 0; + } + + @Override + public String getWriteableName() { + return null; + } + + public static CommonTermsQueryBuilder fromXContent(XContentParser parser) throws IOException { + deprecationLogger.compatibleApiWarning("common_term_query", COMMON_TERMS_QUERY_DEPRECATION_MSG); + throw new ParsingException(parser.getTokenLocation(), COMMON_TERMS_QUERY_DEPRECATION_MSG); + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java index dcc124b86a868..724932175914d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.query.support.QueryParsers; import org.elasticsearch.index.search.MatchQueryParser; @@ -31,7 +32,12 @@ * result of the analysis. */ public class MatchQueryBuilder extends AbstractQueryBuilder { - + private static final String CUTOFF_FREQUENCY_DEPRECATION_MSG = "cutoff_freqency is not supported. " + + "The [match] query can skip block of documents efficiently if the total number of hits is not tracked"; + public static final ParseField CUTOFF_FREQUENCY_FIELD = + new ParseField("cutoff_frequency") + .withAllDeprecated(CUTOFF_FREQUENCY_DEPRECATION_MSG) + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)); public static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query"); public static final ParseField LENIENT_FIELD = new ParseField("lenient"); public static final ParseField FUZZY_TRANSPOSITIONS_FIELD = new ParseField("fuzzy_transpositions"); @@ -464,6 +470,9 @@ public static MatchQueryBuilder fromXContent(XContentParser parser) throws IOExc queryName = parser.text(); } else if (GENERATE_SYNONYMS_PHRASE_QUERY.match(currentFieldName, parser.getDeprecationHandler())) { autoGenerateSynonymsPhraseQuery = parser.booleanValue(); + } else if (parser.getRestApiVersion() == RestApiVersion.V_7 && + CUTOFF_FREQUENCY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + throw new ParsingException(parser.getTokenLocation(), CUTOFF_FREQUENCY_DEPRECATION_MSG); } else { throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] query does not support [" + currentFieldName + "]"); diff --git a/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java index b1b00114fe5f8..7f7ca2c3d86b1 100644 --- a/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.query.support.QueryParsers; import org.elasticsearch.index.search.MatchQueryParser; import org.elasticsearch.index.search.MultiMatchQueryParser; @@ -42,7 +43,12 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder { public static final String NAME = "multi_match"; - + private static final String CUTOFF_FREQUENCY_DEPRECATION_MSG = "cutoff_freqency is not supported." + + " The [multi_match] query can skip block of documents efficiently if the total number of hits is not tracked"; + private static final ParseField CUTOFF_FREQUENCY_FIELD = + new ParseField("cutoff_frequency") + .withAllDeprecated(CUTOFF_FREQUENCY_DEPRECATION_MSG) + .forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)); public static final MultiMatchQueryBuilder.Type DEFAULT_TYPE = MultiMatchQueryBuilder.Type.BEST_FIELDS; public static final Operator DEFAULT_OPERATOR = Operator.OR; public static final int DEFAULT_PHRASE_SLOP = MatchQueryParser.DEFAULT_PHRASE_SLOP; @@ -570,7 +576,6 @@ public static MultiMatchQueryBuilder fromXContent(XContentParser parser) throws ZeroTermsQueryOption zeroTermsQuery = DEFAULT_ZERO_TERMS_QUERY; boolean autoGenerateSynonymsPhraseQuery = true; boolean fuzzyTranspositions = DEFAULT_FUZZY_TRANSPOSITIONS; - float boost = AbstractQueryBuilder.DEFAULT_BOOST; String queryName = null; @@ -633,6 +638,9 @@ public static MultiMatchQueryBuilder fromXContent(XContentParser parser) throws autoGenerateSynonymsPhraseQuery = parser.booleanValue(); } else if (FUZZY_TRANSPOSITIONS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { fuzzyTranspositions = parser.booleanValue(); + } else if (parser.getRestApiVersion() == RestApiVersion.V_7 && + CUTOFF_FREQUENCY_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + throw new ParsingException(parser.getTokenLocation(), CUTOFF_FREQUENCY_DEPRECATION_MSG); } else { throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] query does not support [" + currentFieldName + "]"); diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index b1ecd5f61460d..b8e223879c79c 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.CombinedFieldsQueryBuilder; +import org.elasticsearch.index.query.CommonTermsQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.DisMaxQueryBuilder; import org.elasticsearch.index.query.DistanceFeatureQueryBuilder; @@ -482,6 +483,12 @@ private ValuesSourceRegistry registerAggregations(List plugins) { .setAggregatorRegistrar(CompositeAggregationBuilder::registerAggregators), builder ); + + if(RestApiVersion.minimumSupported() == RestApiVersion.V_7) { + registerQuery(new QuerySpec<>(CommonTermsQueryBuilder.NAME_V7, + (streamInput) -> new CommonTermsQueryBuilder(), CommonTermsQueryBuilder::fromXContent)); + } + registerFromPlugin(plugins, SearchPlugin::getAggregations, (agg) -> this.registerAggregation(agg, builder)); // after aggs have been registered, see if there are any new VSTypes that need to be linked to core fields diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java index e74dd18e767b3..2c2d4c6381876 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.CommonTermsQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.SearchExecutionContext; @@ -393,7 +394,10 @@ public CheckedBiConsumer getReque //add here deprecated queries to make sure we log a deprecation warnings when they are used private static final String[] DEPRECATED_QUERIES = new String[] {"field_masking_span", "geo_polygon"}; - private static final String[] REST_COMPATIBLE_QUERIES = new String[] {TypeQueryV7Builder.NAME_V7.getPreferredName()}; + private static final String[] REST_COMPATIBLE_QUERIES = new String[] { + TypeQueryV7Builder.NAME_V7.getPreferredName(), + CommonTermsQueryBuilder.NAME_V7.getPreferredName() + }; /** * Dummy test {@link AggregationBuilder} used to test registering aggregation builders. @@ -674,7 +678,8 @@ public List> getQueries() { .filter(e -> RestApiVersion.minimumSupported().matches(e.restApiCompatibility)) .filter(e -> RestApiVersion.current().matches(e.restApiCompatibility)) .collect(toSet()), - hasSize(searchModule.getNamedXContents().size()- REST_COMPATIBLE_QUERIES.length - 1 )); + // -1 because of the registered in the test + hasSize(searchModule.getNamedXContents().size() - REST_COMPATIBLE_QUERIES.length -1 )); final List compatEntry = searchModule.getNamedXContents().stream()