Add warning about upcoming expanded fields limit (#34906)

Starting with Elasticsearch 7.0, we limit the number of automatically expanded
fields for the "all fields" mode ("default_field": "*") for the query_string and
simple_query_string queries to 1024 fields (see #26541). This change adds a
deprecation warning to the QueryParserHelper where in the next major version we
will throw an error to warn users of the upcoming change.

Relates to #26541

Author: Christoph Büscher

docs/reference/migration/index.asciidoc

@@ -33,3 +33,5 @@ include::migrate_6_3.asciidoc[]
 include::migrate_6_4.asciidoc[]
 
 include::migrate_6_5.asciidoc[]
+
+include::migrate_6_6.asciidoc[]

docs/reference/migration/migrate_6_6.asciidoc

@@ -0,0 +1,24 @@
+[[breaking-changes-6.6]]
+== Breaking changes in 6.6
+++++
+<titleabbrev>6.6</titleabbrev>
+++++
+
+This section discusses the changes that you need to be aware of when migrating
+your application to Elasticsearch 6.6.
+
+* <<breaking_66_search_changes>>
+
+See also <<release-highlights>> and <<es-release-notes>>.
+
+[float]
+[[breaking_66_search_changes]]
+=== Search changes
+
+[float]
+==== `query_string`, `multi_match` and `simple_query_string` query
+
+Using automatically expanded fields for the "all fields" mode ("default_field": "*")
+for the `query_string`, `multi_match` and `simple_query_string` now raises a warning and
+a deprecation notice to be logged for queries beyond 1024 fields. This limit will be
+enforced with a hard error starting in 7.0.

docs/reference/query-dsl/query-string-query.asciidoc

@@ -63,6 +63,10 @@ settings, which in turn defaults to `*`.
 and filters the metadata fields. All extracted fields are then combined
 to build a query when no prefix field is provided.
 
+WARNING The amount of fields being allowed to be queries at once will be limited
+to 1024 in the next major version or Elasticsearch. Currently this will be raised
+and logged as a Warning only.
+
 |`default_operator` |The default operator used if no explicit operator
 is specified. For example, with a default operator of `OR`, the query
 `capital of Hungary` is translated to `capital OR of OR Hungary`, and

docs/reference/query-dsl/simple-query-string-query.asciidoc

@@ -33,6 +33,10 @@ The `simple_query_string` top level parameters include:
 `*` extracts all fields in the mapping that are eligible to term queries
 and filters the metadata fields.
 
+WARNING The amount of fields being allowed to be queries at once will be limited
+to 1024 in the next major version or Elasticsearch. Currently this will be raised
+and logged as a Warning only.
+
 |`default_operator` |The default operator used if no explicit operator
 is specified. For example, with a default operator of `OR`, the query
 `capital of Hungary` is translated to `capital OR of OR Hungary`, and

server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java

@@ -19,7 +19,9 @@
 
 package org.elasticsearch.index.search;
 
+import org.apache.logging.log4j.LogManager;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -34,6 +36,9 @@
  * Helpers to extract and expand field names and boosts
  */
 public final class QueryParserHelper {
+
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(QueryParserHelper.class));
+
     private QueryParserHelper() {}
 
     /**
@@ -85,6 +90,7 @@ public static Map<String, Float> resolveMappingFields(QueryShardContext context,
                 !multiField, !allField, fieldSuffix);
             resolvedFields.putAll(fieldMap);
         }
+        checkForTooManyFields(resolvedFields);
         return resolvedFields;
     }
 
@@ -149,6 +155,16 @@ public static Map<String, Float> resolveMappingField(QueryShardContext context,
             }
             fields.put(fieldName, weight);
         }
+        checkForTooManyFields(fields);
         return fields;
     }
+
+    private static void checkForTooManyFields(Map<String, Float> fields) {
+        if (fields.size() > 1024) {
+            DEPRECATION_LOGGER.deprecatedAndMaybeLog("field_expansion_limit",
+                    "Field expansion matches too many fields, got: {}. A limit of 1024 will be enforced starting "
+                            + "with version 7.0 of Elasticsearch. Lowering the number of fields will be necessary before upgrading.",
+                    fields.size());
+        }
+    }
 }

server/src/test/java/org/elasticsearch/index/search/QueryParserHelperTests.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.search;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.util.Collections;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+public class QueryParserHelperTests extends ESSingleNodeTestCase {
+
+    /**
+     * Test that when {@link QueryParserHelper#resolveMappingFields(QueryShardContext, java.util.Map, String)} exceeds
+     * the limit of 1024 fields, we emit a warning
+     */
+    public void testLimitOnExpandedFields() throws Exception {
+
+        XContentBuilder builder = jsonBuilder();
+        builder.startObject();
+        builder.startObject("type1");
+        builder.startObject("properties");
+        for (int i = 0; i < 1025; i++) {
+            builder.startObject("field" + i).field("type", "text").endObject();
+        }
+        builder.endObject(); // properties
+        builder.endObject(); // type1
+        builder.endObject();
+        IndexService indexService = createIndex("toomanyfields", Settings.builder()
+                .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1200).build(),
+                "type1", builder);
+        client().prepareIndex("toomanyfields", "type1", "1").setSource("field171", "foo bar baz").get();
+
+        QueryShardContext queryShardContext = indexService.newQueryShardContext(
+                randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null);
+
+        QueryParserHelper.resolveMappingField(queryShardContext, "*", 1.0f, true, false);
+        assertWarnings("Field expansion matches too many fields, got: 1025. A limit of 1024 will be enforced starting with "
+                + "version 7.0 of Elasticsearch. Lowering the number of fields will be necessary before upgrading.");
+
+        QueryParserHelper.resolveMappingFields(queryShardContext,Collections.singletonMap("*", 1.0f));
+        assertWarnings("Field expansion matches too many fields, got: 1025. A limit of 1024 will be enforced starting with "
+                + "version 7.0 of Elasticsearch. Lowering the number of fields will be necessary before upgrading.");
+    }
+}