Integrate "fields" API into QL (#68467)

Author: astefan

docs/reference/sql/endpoints/translate.asciidoc

@@ -22,20 +22,22 @@ Which returns:
 --------------------------------------------------
 {
   "size": 10,
-  "docvalue_fields": [
+  "_source": false,
+  "fields": [
+    {
+      "field": "author"
+    },
+    {
+      "field": "name"
+    },
+    {
+      "field": "page_count"
+    },
     {
       "field": "release_date",
       "format": "epoch_millis"
     }
   ],
-  "_source": {
-    "includes": [
-      "author",
-      "name",
-      "page_count"
-    ],
-    "excludes": []
-  },
   "sort": [
     {
       "page_count": {

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java

@@ -117,7 +117,7 @@ public static List<HitExtractor> createExtractor(List<FieldExtraction> fields, E
     public static HitExtractor createExtractor(FieldExtraction ref, EqlConfiguration cfg) {
         if (ref instanceof SearchHitFieldRef) {
             SearchHitFieldRef f = (SearchHitFieldRef) ref;
-            return new FieldHitExtractor(f.name(), f.fullFieldName(), f.getDataType(), cfg.zoneId(), f.useDocValue(), f.hitName(), false);
+            return new FieldHitExtractor(f.name(), f.getDataType(), cfg.zoneId(), f.hitName(), false);
         }
 
         if (ref instanceof ComputedRef) {
@@ -150,11 +150,11 @@ public static SearchRequest prepareRequest(Client client,
                                                boolean includeFrozen,
                                                String... indices) {
         return client.prepareSearch(indices)
-                .setSource(source)
-                .setAllowPartialSearchResults(false)
-                .setIndicesOptions(
-                        includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS)
-                .request();
+            .setSource(source)
+            .setAllowPartialSearchResults(false)
+            .setIndicesOptions(
+                    includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS)
+            .request();
     }
 
     public static List<SearchHit> searchHits(SearchResponse response) {

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java

@@ -6,10 +6,8 @@
  */
 package org.elasticsearch.xpack.eql.execution.search;
 
-import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
@@ -60,13 +58,8 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB
 
         sorting(container, source);
 
-        // disable the source if there are no includes
-        if (source.fetchSource() == null || CollectionUtils.isEmpty(source.fetchSource().includes())) {
-            source.fetchSource(FetchSourceContext.DO_NOT_FETCH_SOURCE);
-        } else {
-            // use true to fetch only the needed bits from the source
-            source.fetchSource(true);
-        }
+        // disable the source, as we rely on "fields" API
+        source.fetchSource(false);
 
         if (container.limit() != null) {
             // add size and from

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/extractor/FieldHitExtractor.java

@@ -29,9 +29,9 @@ public FieldHitExtractor(StreamInput in) throws IOException {
         super(in);
     }
 
-    public FieldHitExtractor(String name, String fullFieldName, DataType dataType, ZoneId zoneId, boolean useDocValue, String hitName,
+    public FieldHitExtractor(String name, DataType dataType, ZoneId zoneId, String hitName,
                              boolean arrayLeniency) {
-        super(name, fullFieldName, dataType, zoneId, useDocValue, hitName, arrayLeniency);
+        super(name, dataType, zoneId, hitName, arrayLeniency);
     }
 
     @Override

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/extractor/TimestampFieldHitExtractor.java

@@ -10,7 +10,7 @@
 public class TimestampFieldHitExtractor extends FieldHitExtractor {
 
     public TimestampFieldHitExtractor(FieldHitExtractor target) {
-        super(target.fieldName(), target.fullFieldName(), target.dataType(), target.zoneId(), target.useDocValues(), target.hitName(),
+        super(target.fieldName(), target.dataType(), target.zoneId(), target.hitName(),
                 target.arrayLeniency());
     }
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/FieldExtractorRegistry.java

@@ -13,7 +13,6 @@
 import org.elasticsearch.xpack.ql.expression.Expressions;
 import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.gen.pipeline.ConstantInput;
-import org.elasticsearch.xpack.ql.type.DataTypes;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -46,39 +45,6 @@ private FieldExtraction createFieldExtractionFor(Expression expression) {
     }
 
     private FieldExtraction topHitFieldExtractor(FieldAttribute fieldAttr) {
-        FieldAttribute actualField = fieldAttr;
-        FieldAttribute rootField = fieldAttr;
-        StringBuilder fullFieldName = new StringBuilder(fieldAttr.field().getName());
-
-        // Only if the field is not an alias (in which case it will be taken out from docvalue_fields if it's isAggregatable()),
-        // go up the tree of parents until a non-object (and non-nested) type of field is found and use that specific parent
-        // as the field to extract data from, from _source. We do it like this because sub-fields are not in the _source, only
-        // the root field to which those sub-fields belong to, are. Instead of "text_field.keyword_subfield" for _source extraction,
-        // we use "text_field", because there is no source for "keyword_subfield".
-        /*
-         *    "text_field": {
-         *       "type": "text",
-         *       "fields": {
-         *         "keyword_subfield": {
-         *           "type": "keyword"
-         *         }
-         *       }
-         *     }
-         */
-        if (fieldAttr.field().isAlias() == false) {
-            while (actualField.parent() != null
-                    && actualField.parent().field().getDataType() != DataTypes.OBJECT
-                    && actualField.parent().field().getDataType() != DataTypes.NESTED
-                    && actualField.field().getDataType().hasDocValues() == false) {
-                actualField = actualField.parent();
-            }
-        }
-        while (rootField.parent() != null) {
-            fullFieldName.insert(0, ".").insert(0, rootField.parent().field().getName());
-            rootField = rootField.parent();
-        }
-
-        return new SearchHitFieldRef(actualField.name(), fullFieldName.toString(), fieldAttr.field().getDataType(),
-                                     fieldAttr.field().isAggregatable(), fieldAttr.field().isAlias());
+        return new SearchHitFieldRef(fieldAttr.name(), fieldAttr.field().getDataType(), fieldAttr.field().isAlias());
     }
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/SearchHitFieldRef.java

@@ -13,31 +13,23 @@
 
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME_NANOS;
-import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 
 // NB: this class is taken from SQL - it hasn't been ported over to QL
 // since at this stage is unclear whether the whole FieldExtraction infrastructure
 // needs porting or just the field extraction
 public class SearchHitFieldRef implements FieldExtraction {
 
     private final String name;
-    private final String fullFieldName; // path included. If field full path is a.b.c, full field name is "a.b.c" and name is "c"
     private final DataType dataType;
-    private final boolean docValue;
     private final String hitName;
 
-    public SearchHitFieldRef(String name, String fullFieldName, DataType dataType, boolean useDocValueInsteadOfSource, boolean isAlias) {
-        this(name, fullFieldName, dataType, useDocValueInsteadOfSource, isAlias, null);
+    public SearchHitFieldRef(String name, DataType dataType, boolean isAlias) {
+        this(name, dataType, isAlias, null);
     }
 
-    public SearchHitFieldRef(String name, String fullFieldName, DataType dataType, boolean useDocValueInsteadOfSource, boolean isAlias,
-                             String hitName) {
+    public SearchHitFieldRef(String name, DataType dataType, boolean isAlias, String hitName) {
         this.name = name;
-        this.fullFieldName = fullFieldName;
         this.dataType = dataType;
-        // these field types can only be extracted from docvalue_fields (ie, values already computed by Elasticsearch)
-        // because, for us to be able to extract them from _source, we would need the mapping of those fields (which we don't have)
-        this.docValue = isAlias ? useDocValueInsteadOfSource : (hasDocValues(dataType) ? useDocValueInsteadOfSource : false);
         this.hitName = hitName;
     }
 
@@ -49,29 +41,17 @@ public String name() {
         return name;
     }
 
-    public String fullFieldName() {
-        return fullFieldName;
-    }
-
     public DataType getDataType() {
         return dataType;
     }
 
-    public boolean useDocValue() {
-        return docValue;
-    }
-
     @Override
     public void collectFields(QlSourceBuilder sourceBuilder) {
         // nested fields are handled by inner hits
         if (hitName != null) {
             return;
         }
-        if (docValue) {
-            sourceBuilder.addDocField(name, format(dataType));
-        } else {
-            sourceBuilder.addSourceField(name);
-        }
+        sourceBuilder.addFetchField(name, format(dataType));
     }
 
     @Override
@@ -84,10 +64,6 @@ public String toString() {
         return name;
     }
 
-    private static boolean hasDocValues(DataType dataType) {
-        return dataType == KEYWORD || dataType == DATETIME || dataType == DATETIME_NANOS;
-    }
-
     private static String format(DataType dataType) {
         if (dataType == DATETIME_NANOS) {
             return "strict_date_optional_time_nanos";

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java

@@ -30,8 +30,8 @@ public class CriterionOrdinalExtractionTests extends ESTestCase {
     private String tsField = "timestamp";
     private String tbField = "tiebreaker";
 
-    private HitExtractor tsExtractor = new FieldHitExtractor(tsField, tsField, DataTypes.LONG, null, true, null, false);
-    private HitExtractor tbExtractor = new FieldHitExtractor(tbField, tbField, DataTypes.LONG, null, true, null, false);
+    private HitExtractor tsExtractor = new FieldHitExtractor(tsField, DataTypes.LONG, null, null, false);
+    private HitExtractor tbExtractor = new FieldHitExtractor(tbField, DataTypes.LONG, null, null, false);
 
     public void testTimeOnly() throws Exception {
         long time = randomLong();
@@ -56,7 +56,7 @@ public void testTimeAndTiebreakerNull() throws Exception {
     }
 
     public void testTimeNotComparable() throws Exception {
-        HitExtractor badExtractor = new FieldHitExtractor(tsField, tsField, DataTypes.BINARY, null, true, null, false);
+        HitExtractor badExtractor = new FieldHitExtractor(tsField, DataTypes.BINARY, null, null, false);
         SearchHit hit = searchHit(randomAlphaOfLength(10), null);
         Criterion<BoxedQueryRequest> criterion = new Criterion<BoxedQueryRequest>(0, null, emptyList(), badExtractor, null, false);
         EqlIllegalArgumentException exception = expectThrows(EqlIllegalArgumentException.class, () -> criterion.ordinal(hit));

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java

@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.ql.execution.search;
 
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
@@ -23,8 +22,7 @@
  */
 public class QlSourceBuilder {
     // The LinkedHashMaps preserve the order of the fields in the response
-    private final Set<String> sourceFields = new LinkedHashSet<>();
-    private final Set<FieldAndFormat> docFields = new LinkedHashSet<>();
+    private final Set<FieldAndFormat> fetchFields = new LinkedHashSet<>();
     private final Map<String, Script> scriptFields = new LinkedHashMap<>();
 
     boolean trackScores = false;
@@ -40,17 +38,10 @@ public void trackScores() {
     }
 
     /**
-     * Retrieve the requested field from the {@code _source} of the document
+     * Retrieve the requested field using the "fields" API
      */
-    public void addSourceField(String field) {
-        sourceFields.add(field);
-    }
-
-    /**
-     * Retrieve the requested field from doc values (or fielddata) of the document
-     */
-    public void addDocField(String field, String format) {
-        docFields.add(new FieldAndFormat(field, format));
+    public void addFetchField(String field, String format) {
+        fetchFields.add(new FieldAndFormat(field, format));
     }
 
     /**
@@ -66,14 +57,7 @@ public void addScriptField(String name, Script script) {
      */
     public void build(SearchSourceBuilder sourceBuilder) {
         sourceBuilder.trackScores(this.trackScores);
-        if (!sourceFields.isEmpty()) {
-            sourceBuilder.fetchSource(sourceFields.toArray(Strings.EMPTY_ARRAY), null);
-        }
-        docFields.forEach(field -> sourceBuilder.docValueField(field.field, field.format));
+        fetchFields.forEach(field -> sourceBuilder.fetchField(new FieldAndFormat(field.field, field.format, null)));
         scriptFields.forEach(sourceBuilder::scriptField);
     }
-
-    public boolean noSource() {
-        return sourceFields.isEmpty();
-    }
 }