Merge pull request #20026 from jimferenczi/disable_stored_fields

Add the ability to disable the retrieval of the stored fields entirely

Author: jimczi

core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java

@@ -249,14 +249,6 @@ public SearchRequestBuilder setStats(List<String> statsGroups) {
         return this;
     }
 
-    /**
-     * Sets no fields to be loaded, resulting in only id and type to be returned per field.
-     */
-    public SearchRequestBuilder setNoStoredFields() {
-        sourceBuilder().noStoredFields();
-        return this;
-    }
-
     /**
      * Indicates whether the response should contain the stored _source for every hit
      */
@@ -302,7 +294,6 @@ public SearchRequestBuilder addDocValueField(String name) {
 
     /**
      * Adds a stored field to load and return (note, it must be stored) as part of the search request.
-     * If none are specified, the source of the document will be return.
      */
     public SearchRequestBuilder addStoredField(String field) {
         sourceBuilder().storedField(field);
@@ -380,9 +371,8 @@ public SearchRequestBuilder setTrackScores(boolean trackScores) {
     }
 
     /**
-     * Sets the stored fields to load and return as part of the search request. If none
-     * are specified, the source of the document will be returned.
-     *
+     * Adds stored fields to load and return (note, it must be stored) as part of the search request.
+     * To disable the stored fields entirely (source and metadata fields) use {@code storedField("_none_")}.
      * @deprecated Use {@link SearchRequestBuilder#storedFields(String...)} instead.
      */
     @Deprecated
@@ -392,8 +382,8 @@ public SearchRequestBuilder fields(String... fields) {
     }
 
     /**
-     * Sets the fields to load and return as part of the search request. If none
-     * are specified, the source of the document will be returned.
+     * Adds stored fields to load and return (note, it must be stored) as part of the search request.
+     * To disable the stored fields entirely (source and metadata fields) use {@code storedField("_none_")}.
      */
     public SearchRequestBuilder storedFields(String... fields) {
         sourceBuilder().storedFields(Arrays.asList(fields));

core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField;
+import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
@@ -137,7 +138,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     private boolean version;
     private boolean trackScores;
 
-    private List<String> storedFieldNames;
+    private StoredFieldsContext storedFieldsContext;
     private QueryBuilder query = DEFAULT_INNER_HIT_QUERY;
     private List<SortBuilder<?>> sorts;
     private List<String> docValueFields;
@@ -156,14 +157,14 @@ private InnerHitBuilder(InnerHitBuilder other) {
         explain = other.explain;
         version = other.version;
         trackScores = other.trackScores;
-        if (other.storedFieldNames != null) {
-            storedFieldNames = new ArrayList<>(other.storedFieldNames);
+        if (other.storedFieldsContext != null) {
+            storedFieldsContext = new StoredFieldsContext(other.storedFieldsContext);
         }
         if (other.docValueFields != null) {
-            docValueFields = new ArrayList<>(other.docValueFields);
+            docValueFields = new ArrayList<> (other.docValueFields);
         }
         if (other.scriptFields != null) {
-            scriptFields = new HashSet<>(other.scriptFields);
+            scriptFields = new HashSet<> (other.scriptFields);
         }
         if (other.fetchSourceContext != null) {
             fetchSourceContext = new FetchSourceContext(
@@ -210,7 +211,7 @@ public InnerHitBuilder(StreamInput in) throws IOException {
         explain = in.readBoolean();
         version = in.readBoolean();
         trackScores = in.readBoolean();
-        storedFieldNames = (List<String>) in.readGenericValue();
+        storedFieldsContext = in.readOptionalWriteable(StoredFieldsContext::new);
         docValueFields = (List<String>) in.readGenericValue();
         if (in.readBoolean()) {
             int size = in.readVInt();
@@ -248,14 +249,14 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(explain);
         out.writeBoolean(version);
         out.writeBoolean(trackScores);
-        out.writeGenericValue(storedFieldNames);
+        out.writeOptionalWriteable(storedFieldsContext);
         out.writeGenericValue(docValueFields);
         boolean hasScriptFields = scriptFields != null;
         out.writeBoolean(hasScriptFields);
         if (hasScriptFields) {
             out.writeVInt(scriptFields.size());
             for (ScriptField scriptField : scriptFields) {
-                scriptField.writeTo(out);;
+                scriptField.writeTo(out);
             }
         }
         out.writeOptionalStreamable(fetchSourceContext);
@@ -343,39 +344,42 @@ public InnerHitBuilder setTrackScores(boolean trackScores) {
     /**
      * Gets the stored fields to load and return.
      *
-     * @deprecated Use {@link InnerHitBuilder#getStoredFieldNames()} instead.
+     * @deprecated Use {@link InnerHitBuilder#getStoredFieldsContext()} instead.
      */
     @Deprecated
     public List<String> getFieldNames() {
-        return storedFieldNames;
+        return storedFieldsContext == null ? null : storedFieldsContext.fieldNames();
     }
 
     /**
-     * Sets the stored fields to load and return. If none
-     * are specified, the source of the document will be returned.
+     * Sets the stored fields to load and return.
+     * If none are specified, the source of the document will be returned.
      *
      * @deprecated Use {@link InnerHitBuilder#setStoredFieldNames(List)} instead.
      */
     @Deprecated
     public InnerHitBuilder setFieldNames(List<String> fieldNames) {
-        this.storedFieldNames = fieldNames;
-        return this;
+        return setStoredFieldNames(fieldNames);
     }
 
 
     /**
-     * Gets the stored fields to load and return.
+     * Gets the stored fields context.
      */
-    public List<String> getStoredFieldNames() {
-        return storedFieldNames;
+    public StoredFieldsContext getStoredFieldsContext() {
+        return storedFieldsContext;
     }
 
     /**
-     * Sets the stored fields to load and return. If none
-     * are specified, the source of the document will be returned.
+     * Sets the stored fields to load and return.
+     * If none are specified, the source of the document will be returned.
      */
     public InnerHitBuilder setStoredFieldNames(List<String> fieldNames) {
-        this.storedFieldNames = fieldNames;
+        if (storedFieldsContext == null) {
+            storedFieldsContext = StoredFieldsContext.fromList(fieldNames);
+        } else {
+            storedFieldsContext.addFieldNames(fieldNames);
+        }
         return this;
     }
 
@@ -564,14 +568,8 @@ private void setupInnerHitsContext(QueryShardContext context, InnerHitsContext.B
         innerHitsContext.explain(explain);
         innerHitsContext.version(version);
         innerHitsContext.trackScores(trackScores);
-        if (storedFieldNames != null) {
-            if (storedFieldNames.isEmpty()) {
-                innerHitsContext.emptyFieldNames();
-            } else {
-                for (String fieldName : storedFieldNames) {
-                    innerHitsContext.fieldNames().add(fieldName);
-                }
-            }
+        if (storedFieldsContext != null) {
+            innerHitsContext.storedFieldsContext(storedFieldsContext);
         }
         if (docValueFields != null) {
             DocValueFieldsContext docValueFieldsContext = innerHitsContext
@@ -633,16 +631,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (fetchSourceContext != null) {
             builder.field(SearchSourceBuilder._SOURCE_FIELD.getPreferredName(), fetchSourceContext, params);
         }
-        if (storedFieldNames != null) {
-            if (storedFieldNames.size() == 1) {
-                builder.field(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), storedFieldNames.get(0));
-            } else {
-                builder.startArray(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName());
-                for (String fieldName : storedFieldNames) {
-                    builder.value(fieldName);
-                }
-                builder.endArray();
-            }
+        if (storedFieldsContext != null) {
+            storedFieldsContext.toXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), builder);
         }
         if (docValueFields != null) {
             builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
@@ -693,7 +683,7 @@ public boolean equals(Object o) {
                 Objects.equals(explain, that.explain) &&
                 Objects.equals(version, that.version) &&
                 Objects.equals(trackScores, that.trackScores) &&
-                Objects.equals(storedFieldNames, that.storedFieldNames) &&
+                Objects.equals(storedFieldsContext, that.storedFieldsContext) &&
                 Objects.equals(docValueFields, that.docValueFields) &&
                 Objects.equals(scriptFields, that.scriptFields) &&
                 Objects.equals(fetchSourceContext, that.fetchSourceContext) &&
@@ -705,7 +695,7 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, nestedPath, parentChildType, from, size, explain, version, trackScores, storedFieldNames,
+        return Objects.hash(name, nestedPath, parentChildType, from, size, explain, version, trackScores, storedFieldsContext,
             docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits);
     }
 

core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java

@@ -24,7 +24,6 @@
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
@@ -33,7 +32,6 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
@@ -42,13 +40,12 @@
 import org.elasticsearch.rest.action.RestStatusToXContentListener;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.SearchRequestParsers;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.StoredFieldsContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.search.suggest.SuggestBuilder;
-import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
 
 import java.io.IOException;
@@ -178,18 +175,11 @@ private static void parseSearchSource(final SearchSourceBuilder searchSourceBuil
                 "if the field is not stored");
         }
 
-        String sField = request.param("stored_fields");
-        if (sField != null) {
-            if (!Strings.hasText(sField)) {
-                searchSourceBuilder.noStoredFields();
-            } else {
-                String[] sFields = Strings.splitStringByCommaToArray(sField);
-                if (sFields != null) {
-                    for (String field : sFields) {
-                        searchSourceBuilder.storedField(field);
-                    }
-                }
-            }
+
+        StoredFieldsContext storedFieldsContext =
+            StoredFieldsContext.fromRestRequest(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), request);
+        if (storedFieldsContext != null) {
+            searchSourceBuilder.storedFields(storedFieldsContext);
         }
         String sDocValueFields = request.param("docvalue_fields");
         if (sDocValueFields == null) {

core/src/main/java/org/elasticsearch/search/SearchService.java

@@ -67,8 +67,8 @@
 import org.elasticsearch.search.fetch.ScrollQueryFetchSearchResult;
 import org.elasticsearch.search.fetch.ShardFetchRequest;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
-import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext.DocValueField;
+import org.elasticsearch.search.fetch.subphase.DocValueFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.DefaultSearchContext;
@@ -729,9 +729,6 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
                 throw new SearchContextException(context, "failed to create RescoreSearchContext", e);
             }
         }
-        if (source.storedFields() != null) {
-            context.fieldNames().addAll(source.storedFields());
-        }
         if (source.explain() != null) {
             context.explain(source.explain());
         }
@@ -823,6 +820,18 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
             }
             context.sliceBuilder(source.slice());
         }
+
+        if (source.storedFields() != null) {
+            if (source.storedFields().fetchFields() == false) {
+                if (context.version()) {
+                    throw new SearchContextException(context, "`stored_fields` cannot be disabled if version is requested");
+                }
+                if (context.sourceRequested()) {
+                    throw new SearchContextException(context, "`stored_fields` cannot be disabled if _source is requested");
+                }
+            }
+            context.storedFieldsContext(source.storedFields());
+        }
     }
 
     /**