Refactor field expansion for match, multi_match and query_string query (#25726)

This commit changes the way we handle field expansion in `match`, `multi_match` and `query_string` query.
 The main changes are:

- For exact field name, the new behavior is to rewrite to a matchnodocs query when the field name is not found in the mapping.

- For partial field names (with `*` suffix), the expansion is done only on `keyword`, `text`, `date`, `ip` and `number` field types. Other field types are simply ignored.

- For all fields (`*`), the expansion is done on accepted field types only (see above) and metadata fields are also filtered.

- The `*` notation can also be used to set `default_field` option on`query_string` query. This should replace the needs for the extra option `use_all_fields` which is deprecated in this change.

This commit also rewrites simple `*` query to matchalldocs query when all fields are requested (Fixes #25556). 

The same change should be done on `simple_query_string` for completeness.

`use_all_fields` option in `query_string` is also deprecated in this change, `default_field` should be set to `*` instead.

Relates #25551

Author: jimczi

core/src/main/java/org/elasticsearch/common/lucene/search/Queries.java

@@ -30,6 +30,7 @@
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.mapper.TypeFieldMapper;
 
@@ -47,6 +48,16 @@ public static Query newMatchNoDocsQuery(String reason) {
         return new MatchNoDocsQuery(reason);
     }
 
+
+    public static Query newUnmappedFieldQuery(String field) {
+        return Queries.newMatchNoDocsQuery("unmapped field [" + (field != null ? field : "null") + "]");
+    }
+
+    public static Query newLenientFieldQuery(String field, RuntimeException e) {
+        String message = ElasticsearchException.getExceptionName(e) + ":[" + e.getMessage() + "]";
+        return Queries.newMatchNoDocsQuery("failed [" + field + "] query, caused by " + message);
+    }
+
     public static Query newNestedFilter() {
         return new PrefixQuery(new Term(TypeFieldMapper.NAME, new BytesRef("__")));
     }

core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.index.query.support.QueryParsers;
 import org.elasticsearch.index.search.MatchQuery;
 import org.elasticsearch.index.search.MultiMatchQuery;
+import org.elasticsearch.index.search.QueryStringQueryParser;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -739,27 +740,10 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             }
         }
 
-        Map<String, Float> newFieldsBoosts = handleFieldsMatchPattern(context.getMapperService(), fieldsBoosts);
-
+        Map<String, Float> newFieldsBoosts = QueryStringQueryParser.resolveMappingFields(context, fieldsBoosts);
         return multiMatchQuery.parse(type, newFieldsBoosts, value, minimumShouldMatch);
     }
 
-    private static Map<String, Float> handleFieldsMatchPattern(MapperService mapperService, Map<String, Float> fieldsBoosts) {
-        Map<String, Float> newFieldsBoosts = new TreeMap<>();
-        for (Map.Entry<String, Float> fieldBoost : fieldsBoosts.entrySet()) {
-            String fField = fieldBoost.getKey();
-            Float fBoost = fieldBoost.getValue();
-            if (Regex.isSimpleMatchPattern(fField)) {
-                for (String field : mapperService.simpleMatchToIndexNames(fField)) {
-                    newFieldsBoosts.put(field, fBoost);
-                }
-            } else {
-                newFieldsBoosts.put(fField, fBoost);
-            }
-        }
-        return newFieldsBoosts;
-    }
-
     @Override
     protected int doHashCode() {
         return Objects.hash(value, fieldsBoosts, type, operator, analyzer, slop, fuzziness,

core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java

@@ -34,28 +34,17 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.mapper.DateFieldMapper;
-import org.elasticsearch.index.mapper.IpFieldMapper;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.NumberFieldMapper;
-import org.elasticsearch.index.mapper.ScaledFloatFieldMapper;
-import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.query.support.QueryParsers;
 import org.elasticsearch.index.search.QueryStringQueryParser;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.TreeMap;
 
 /**
@@ -110,24 +99,10 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
     private static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone");
     private static final ParseField SPLIT_ON_WHITESPACE = new ParseField("split_on_whitespace")
             .withAllDeprecated("This setting is ignored, the parser always splits on logical operator");
-    private static final ParseField ALL_FIELDS_FIELD = new ParseField("all_fields");
+    private static final ParseField ALL_FIELDS_FIELD = new ParseField("all_fields")
+            .withAllDeprecated("Set [default_field] to `*` instead");
     private static final ParseField TYPE_FIELD = new ParseField("type");
 
-    // Mapping types the "all-ish" query can be executed against
-    public static final Set<String> ALLOWED_QUERY_MAPPER_TYPES;
-
-    static {
-        ALLOWED_QUERY_MAPPER_TYPES = new HashSet<>();
-        ALLOWED_QUERY_MAPPER_TYPES.add(DateFieldMapper.CONTENT_TYPE);
-        ALLOWED_QUERY_MAPPER_TYPES.add(IpFieldMapper.CONTENT_TYPE);
-        ALLOWED_QUERY_MAPPER_TYPES.add(KeywordFieldMapper.CONTENT_TYPE);
-        for (NumberFieldMapper.NumberType nt : NumberFieldMapper.NumberType.values()) {
-            ALLOWED_QUERY_MAPPER_TYPES.add(nt.typeName());
-        }
-        ALLOWED_QUERY_MAPPER_TYPES.add(ScaledFloatFieldMapper.CONTENT_TYPE);
-        ALLOWED_QUERY_MAPPER_TYPES.add(TextFieldMapper.CONTENT_TYPE);
-    }
-
     private final String queryString;
 
     private String defaultField;
@@ -179,8 +154,6 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
 
     private DateTimeZone timeZone;
 
-    private Boolean useAllFields;
-
     /** To limit effort spent determinizing regexp queries. */
     private int maxDeterminizedStates = DEFAULT_MAX_DETERMINED_STATES;
 
@@ -240,8 +213,11 @@ public QueryStringQueryBuilder(StreamInput in) throws IOException {
         if (in.getVersion().onOrAfter(Version.V_5_1_1)) {
             if (in.getVersion().before(Version.V_6_0_0_beta1)) {
                 in.readBoolean(); // split_on_whitespace
+                Boolean useAllField = in.readOptionalBoolean();
+                if (useAllField != null && useAllField) {
+                    defaultField = "*";
+                }
             }
-            useAllFields = in.readOptionalBoolean();
         }
     }
 
@@ -291,8 +267,9 @@ protected void doWriteTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_5_1_1)) {
             if (out.getVersion().before(Version.V_6_0_0_beta1)) {
                 out.writeBoolean(false); // split_on_whitespace
+                Boolean useAllFields = defaultField == null ? null : Regex.isMatchAllPattern(defaultField);
+                out.writeOptionalBoolean(useAllFields);
             }
-            out.writeOptionalBoolean(this.useAllFields);
         }
     }
 
@@ -314,17 +291,19 @@ public String defaultField() {
     }
 
     /**
-     * Tell the query_string query to use all fields explicitly, even if _all is
-     * enabled. If the "default_field" parameter or "fields" are specified, they
-     * will be ignored.
+     * This setting is deprecated, set {@link #defaultField(String)} to "*" instead.
      */
+    @Deprecated
     public QueryStringQueryBuilder useAllFields(Boolean useAllFields) {
-        this.useAllFields = useAllFields;
+        if (useAllFields) {
+            this.defaultField = "*";
+        }
         return this;
     }
 
+    @Deprecated
     public Boolean useAllFields() {
-        return this.useAllFields;
+        return defaultField == null ? null : Regex.isMatchAllPattern(defaultField);
     }
 
     /**
@@ -703,9 +682,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
             builder.field(TIME_ZONE_FIELD.getPreferredName(), this.timeZone.getID());
         }
         builder.field(ESCAPE_FIELD.getPreferredName(), this.escape);
-        if (this.useAllFields != null) {
-            builder.field(ALL_FIELDS_FIELD.getPreferredName(), this.useAllFields);
-        }
         printBoostAndQueryName(builder);
         builder.endObject();
     }
@@ -737,7 +713,6 @@ public static QueryStringQueryBuilder fromXContent(XContentParser parser) throws
         Fuzziness fuzziness = QueryStringQueryBuilder.DEFAULT_FUZZINESS;
         String fuzzyRewrite = null;
         String rewrite = null;
-        Boolean useAllFields = null;
         Map<String, Float> fieldsAndWeights = new HashMap<>();
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
@@ -812,7 +787,7 @@ public static QueryStringQueryBuilder fromXContent(XContentParser parser) throws
                 } else if (LENIENT_FIELD.match(currentFieldName)) {
                     lenient = parser.booleanValue();
                 } else if (ALL_FIELDS_FIELD.match(currentFieldName)) {
-                    useAllFields = parser.booleanValue();
+                    defaultField = "*";
                 } else if (MAX_DETERMINIZED_STATES_FIELD.match(currentFieldName)) {
                     maxDeterminizedStates = parser.intValue();
                 } else if (TIME_ZONE_FIELD.match(currentFieldName)) {
@@ -847,12 +822,6 @@ public static QueryStringQueryBuilder fromXContent(XContentParser parser) throws
             throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME + "] must be provided with a [query]");
         }
 
-        if ((useAllFields != null && useAllFields) &&
-                (defaultField != null || fieldsAndWeights.size() != 0)) {
-            throw new ParsingException(parser.getTokenLocation(),
-                    "cannot use [all_fields] parameter in conjunction with [default_field] or [fields]");
-        }
-
         QueryStringQueryBuilder queryStringQuery = new QueryStringQueryBuilder(queryString);
         queryStringQuery.fields(fieldsAndWeights);
         queryStringQuery.defaultField(defaultField);
@@ -880,7 +849,6 @@ public static QueryStringQueryBuilder fromXContent(XContentParser parser) throws
         queryStringQuery.timeZone(timeZone);
         queryStringQuery.boost(boost);
         queryStringQuery.queryName(queryName);
-        queryStringQuery.useAllFields(useAllFields);
         return queryStringQuery;
     }
 
@@ -914,8 +882,7 @@ protected boolean doEquals(QueryStringQueryBuilder other) {
                 timeZone == null ? other.timeZone == null : other.timeZone != null &&
                 Objects.equals(timeZone.getID(), other.timeZone.getID()) &&
                 Objects.equals(escape, other.escape) &&
-                Objects.equals(maxDeterminizedStates, other.maxDeterminizedStates) &&
-                Objects.equals(useAllFields, other.useAllFields);
+                Objects.equals(maxDeterminizedStates, other.maxDeterminizedStates);
     }
 
     @Override
@@ -924,72 +891,37 @@ protected int doHashCode() {
                 quoteFieldSuffix, allowLeadingWildcard, analyzeWildcard,
                 enablePositionIncrements, fuzziness, fuzzyPrefixLength,
                 fuzzyMaxExpansions, fuzzyRewrite, phraseSlop, type, tieBreaker, rewrite, minimumShouldMatch, lenient,
-                timeZone == null ? 0 : timeZone.getID(), escape, maxDeterminizedStates, useAllFields);
-    }
-
-    /**
-     * Given a shard context, return a map of all fields in the mappings that
-     * can be queried. The map will be field name to a float of 1.0f.
-     */
-    public static Map<String, Float> allQueryableDefaultFields(QueryShardContext context) {
-        Collection<String> allFields = context.simpleMatchToIndexNames("*");
-        Map<String, Float> fields = new HashMap<>();
-        for (String fieldName : allFields) {
-            if (MapperService.isMetadataField(fieldName)) {
-                // Ignore our metadata fields
-                continue;
-            }
-            MappedFieldType mft = context.fieldMapper(fieldName);
-            assert mft != null : "should never have a null mapper for an existing field";
-
-            // Ignore fields that are not in the allowed mapper types. Some
-            // types do not support term queries, and thus we cannot generate
-            // a special query for them.
-            String mappingType = mft.typeName();
-            if (ALLOWED_QUERY_MAPPER_TYPES.contains(mappingType)) {
-                fields.put(fieldName, 1.0f);
-            }
-        }
-        return fields;
+                timeZone == null ? 0 : timeZone.getID(), escape, maxDeterminizedStates);
     }
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         String rewrittenQueryString = escape ? org.apache.lucene.queryparser.classic.QueryParser.escape(this.queryString) : queryString;
-        if ((useAllFields != null && useAllFields) && (fieldsAndWeights.size() != 0 || this.defaultField != null)) {
-            throw addValidationError("cannot use [all_fields] parameter in conjunction with [default_field] or [fields]", null);
+        if (fieldsAndWeights.size() > 0 && this.defaultField != null) {
+            throw addValidationError("cannot use [fields] parameter in conjunction with [default_field]", null);
         }
 
         QueryStringQueryParser queryParser;
         boolean isLenient = lenient == null ? context.queryStringLenient() : lenient;
         if (defaultField != null) {
-            queryParser = new QueryStringQueryParser(context, defaultField, isLenient);
-        } else if (fieldsAndWeights.size() > 0) {
-            final Map<String, Float> resolvedFields = new TreeMap<>();
-            for (Map.Entry<String, Float> fieldsEntry : fieldsAndWeights.entrySet()) {
-                String fieldName = fieldsEntry.getKey();
-                Float weight = fieldsEntry.getValue();
-                if (Regex.isSimpleMatchPattern(fieldName)) {
-                    for (String resolvedFieldName : context.getMapperService().simpleMatchToIndexNames(fieldName)) {
-                        resolvedFields.put(resolvedFieldName, weight);
-                    }
-                } else {
-                    resolvedFields.put(fieldName, weight);
-                }
+            if (Regex.isMatchAllPattern(defaultField)) {
+                queryParser = new QueryStringQueryParser(context, lenient == null ? true : lenient);
+            } else {
+                queryParser = new QueryStringQueryParser(context, defaultField, isLenient);
             }
+        } else if (fieldsAndWeights.size() > 0) {
+            final Map<String, Float> resolvedFields = QueryStringQueryParser.resolveMappingFields(context, fieldsAndWeights);
             queryParser = new QueryStringQueryParser(context, resolvedFields, isLenient);
         } else {
-            // If explicitly required to use all fields, use all fields, OR:
-            // Automatically determine the fields (to replace the _all field) if all of the following are true:
-            // - The _all field is disabled,
-            // - and the default_field has not been changed in the settings
-            // - and default_field is not specified in the request
-            // - and no fields are specified in the request
-            if ((useAllFields != null && useAllFields) ||
+            // Expand to all fields if:
+            // - The index default search field is "*"
+            // - The index default search field is "_all" and _all is disabled
+            // TODO the index default search field should be "*" for new indices.
+            if (Regex.isMatchAllPattern(context.defaultField()) ||
                     (context.getMapperService().allEnabled() == false && "_all".equals(context.defaultField()))) {
                 // Automatically determine the fields from the index mapping.
                 // Automatically set leniency to "true" if unset so mismatched fields don't cause exceptions;
-                queryParser = new QueryStringQueryParser(context, allQueryableDefaultFields(context), lenient == null ? true : lenient);
+                queryParser = new QueryStringQueryParser(context, lenient == null ? true : lenient);
             } else {
                 queryParser = new QueryStringQueryParser(context, context.defaultField(), isLenient);
             }

core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.query.SimpleQueryParser.Settings;
+import org.elasticsearch.index.search.QueryStringQueryParser;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -376,7 +377,8 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
                 (context.getMapperService().allEnabled() == false &&
                         "_all".equals(context.defaultField()) &&
                         this.fieldsAndWeights.isEmpty())) {
-            resolvedFieldsAndWeights = QueryStringQueryBuilder.allQueryableDefaultFields(context);
+            resolvedFieldsAndWeights = QueryStringQueryParser.resolveMappingField(context, "*", 1.0f,
+                false, false);
             // Need to use lenient mode when using "all-mode" so exceptions aren't thrown due to mismatched types
             newSettings.lenient(lenientSet ? settings.lenient() : true);
         } else {