[7.5][ML] Prevent fetching multi-field from source (#48770) (#48798)

Aggregatable mutli-fields are at the moment wrongly mapped
as normal doc_value fields and thus they support fetching from
source. However, they do not exist in the source. This results
to failure to extract such fields.

This commit fixes this bug. While a fix could be worked out
on top of the existing code, it is evident the extraction logic
has become difficult to understand and maintain. As we also
want to deduplicate multi-fields for data frame analytics,
it seemed appropriate to refactor the code to simplify and
better handle the extraction of multi-fields.

Relates #48756

Backport of #48770

Author: dimitris-athanasiou

x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java

@@ -55,7 +55,7 @@ protected boolean preserveTemplatesUponCompletion() {
         return true;
     }
 
-    private void setupDataAccessRole(String index) throws IOException {
+    private static void setupDataAccessRole(String index) throws IOException {
         Request request = new Request("PUT", "/_security/role/test_data_access");
         request.setJsonEntity("{"
                 + "  \"indices\" : ["
@@ -283,10 +283,12 @@ public void testLookbackOnlyWithSourceDisabled() throws Exception {
         new LookbackOnlyTestHelper("test-lookback-only-with-source-disabled", "airline-data-disabled-source").execute();
     }
 
-    @AwaitsFix(bugUrl = "This test uses painless which is not available in the integTest phase")
     public void testLookbackOnlyWithScriptFields() throws Exception {
-        new LookbackOnlyTestHelper("test-lookback-only-with-script-fields", "airline-data-disabled-source")
-                .setAddScriptedFields(true).execute();
+        new LookbackOnlyTestHelper("test-lookback-only-with-script-fields", "airline-data")
+                .setScriptedFields(
+                    "{\"scripted_airline\":{\"script\":{\"lang\":\"painless\",\"source\":\"doc['airline.keyword'].value\"}}}")
+            .setAirlineVariant("scripted_airline")
+            .execute();
     }
 
     public void testLookbackOnlyWithNestedFields() throws Exception {
@@ -1088,7 +1090,7 @@ private class LookbackOnlyTestHelper {
         private String jobId;
         private String airlineVariant;
         private String dataIndex;
-        private boolean addScriptedFields;
+        private String scriptedFields;
         private boolean shouldSucceedInput;
         private boolean shouldSucceedProcessing;
 
@@ -1100,8 +1102,8 @@ private class LookbackOnlyTestHelper {
             this.airlineVariant = "airline";
         }
 
-        public LookbackOnlyTestHelper setAddScriptedFields(boolean value) {
-            addScriptedFields = value;
+        public LookbackOnlyTestHelper setScriptedFields(String scriptFields) {
+            this.scriptedFields = scriptFields;
             return this;
         }
 
@@ -1124,10 +1126,7 @@ public LookbackOnlyTestHelper setShouldSucceedProcessing(boolean value) {
         public void execute() throws Exception {
             createJob(jobId, airlineVariant);
             String datafeedId = "datafeed-" + jobId;
-            new DatafeedBuilder(datafeedId, jobId, dataIndex)
-                    .setScriptedFields(addScriptedFields ?
-                            "{\"airline\":{\"script\":{\"lang\":\"painless\",\"inline\":\"doc['airline'].value\"}}}" : null)
-                    .build();
+            new DatafeedBuilder(datafeedId, jobId, dataIndex).setScriptedFields(scriptedFields).build();
             openJob(client(), jobId);
 
             startDatafeedAndWaitUntilStopped(datafeedId);

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java

@@ -130,7 +130,7 @@ private SearchRequestBuilder buildSearchRequest(long start) {
                         context.query, context.extractedFields.timeField(), start, context.end));
 
         for (ExtractedField docValueField : context.extractedFields.getDocValueFields()) {
-            searchRequestBuilder.addDocValueField(docValueField.getName(), docValueField.getDocValueFormat());
+            searchRequestBuilder.addDocValueField(docValueField.getSearchField(), docValueField.getDocValueFormat());
         }
         String[] sourceFields = context.extractedFields.getSourceFields();
         if (sourceFields.length == 0) {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessor.java

@@ -29,7 +29,7 @@ class SearchHitToJsonProcessor implements Releasable {
     public void process(SearchHit hit) throws IOException {
         jsonBuilder.startObject();
         for (ExtractedField field : fields.getAllFields()) {
-            writeKeyValue(field.getAlias(), field.value(hit));
+            writeKeyValue(field.getName(), field.value(hit));
         }
         jsonBuilder.endObject();
     }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/TimeBasedExtractedFields.java

@@ -14,7 +14,6 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -42,13 +41,13 @@ public String timeField() {
     public Long timeFieldValue(SearchHit hit) {
         Object[] value = timeField.value(hit);
         if (value.length != 1) {
-            throw new RuntimeException("Time field [" + timeField.getAlias() + "] expected a single value; actual was: "
+            throw new RuntimeException("Time field [" + timeField.getName() + "] expected a single value; actual was: "
                     + Arrays.toString(value));
         }
         if (value[0] instanceof Long) {
             return (Long) value[0];
         }
-        throw new RuntimeException("Time field [" + timeField.getAlias() + "] expected a long value; actual was: " + value[0]);
+        throw new RuntimeException("Time field [" + timeField.getName() + "] expected a long value; actual was: " + value[0]);
     }
 
     public static TimeBasedExtractedFields build(Job job, DatafeedConfig datafeed, FieldCapabilitiesResponse fieldsCapabilities) {
@@ -58,20 +57,18 @@ public static TimeBasedExtractedFields build(Job job, DatafeedConfig datafeed, F
         if (scriptFields.contains(timeField) == false && extractionMethodDetector.isAggregatable(timeField) == false) {
             throw new IllegalArgumentException("cannot retrieve time field [" + timeField + "] because it is not aggregatable");
         }
-        ExtractedField timeExtractedField = extractedTimeField(timeField, scriptFields, fieldsCapabilities);
+        ExtractedField timeExtractedField = extractedTimeField(timeField, scriptFields);
         List<String> remainingFields = job.allInputFields().stream().filter(f -> !f.equals(timeField)).collect(Collectors.toList());
         List<ExtractedField> allExtractedFields = new ArrayList<>(remainingFields.size() + 1);
         allExtractedFields.add(timeExtractedField);
         remainingFields.stream().forEach(field -> allExtractedFields.add(extractionMethodDetector.detect(field)));
+
         return new TimeBasedExtractedFields(timeExtractedField, allExtractedFields);
     }
 
-    private static ExtractedField extractedTimeField(String timeField, Set<String> scriptFields,
-                                                     FieldCapabilitiesResponse fieldCapabilities) {
-        if (scriptFields.contains(timeField)) {
-            return ExtractedField.newTimeField(timeField, Collections.emptySet(), ExtractedField.ExtractionMethod.SCRIPT_FIELD);
-        }
-        return ExtractedField.newTimeField(timeField, fieldCapabilities.getField(timeField).keySet(),
-            ExtractedField.ExtractionMethod.DOC_VALUE);
+    private static ExtractedField extractedTimeField(String timeField, Set<String> scriptFields) {
+        ExtractedField.Method method = scriptFields.contains(timeField) ? ExtractedField.Method.SCRIPT_FIELD
+            : ExtractedField.Method.DOC_VALUE;
+        return ExtractedFields.newTimeField(timeField, method);
     }
 }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/DataFrameDataExtractor.java

@@ -24,8 +24,8 @@
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis;
-import org.elasticsearch.xpack.ml.extractor.ExtractedField;
 import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsIndex;
+import org.elasticsearch.xpack.ml.extractor.ExtractedField;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -138,7 +138,7 @@ private SearchRequestBuilder buildSearchRequest() {
         setFetchSource(searchRequestBuilder);
 
         for (ExtractedField docValueField : context.extractedFields.getDocValueFields()) {
-            searchRequestBuilder.addDocValueField(docValueField.getName(), docValueField.getDocValueFormat());
+            searchRequestBuilder.addDocValueField(docValueField.getSearchField(), docValueField.getDocValueFormat());
         }
 
         return searchRequestBuilder;
@@ -231,7 +231,7 @@ private void clearScroll(String scrollId) {
     }
 
     public List<String> getFieldNames() {
-        return context.extractedFields.getAllFields().stream().map(ExtractedField::getAlias).collect(Collectors.toList());
+        return context.extractedFields.getAllFields().stream().map(ExtractedField::getName).collect(Collectors.toList());
     }
 
     public DataSummary collectDataSummary() {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetector.java

@@ -11,11 +11,9 @@
 import org.elasticsearch.action.fieldcaps.FieldCapabilities;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.BooleanFieldMapper;
-import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
 import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest;
@@ -24,9 +22,9 @@
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
 import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
 import org.elasticsearch.xpack.core.ml.utils.NameResolver;
+import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsIndex;
 import org.elasticsearch.xpack.ml.extractor.ExtractedField;
 import org.elasticsearch.xpack.ml.extractor.ExtractedFields;
-import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsIndex;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -264,13 +262,13 @@ private ExtractedFields fetchBooleanFieldsAsIntegers(ExtractedFields extractedFi
         List<ExtractedField> adjusted = new ArrayList<>(extractedFields.getAllFields().size());
         for (ExtractedField field : extractedFields.getAllFields()) {
             if (isBoolean(field.getTypes())) {
-                if (config.getAnalysis().getAllowedCategoricalTypes(field.getAlias()).contains(BooleanFieldMapper.CONTENT_TYPE)) {
+                if (config.getAnalysis().getAllowedCategoricalTypes(field.getName()).contains(BooleanFieldMapper.CONTENT_TYPE)) {
                     // We convert boolean field to string if it is a categorical dependent variable
-                    adjusted.add(new BooleanMapper<>(field, Boolean.TRUE.toString(), Boolean.FALSE.toString()));
+                    adjusted.add(ExtractedFields.applyBooleanMapping(field, Boolean.TRUE.toString(), Boolean.FALSE.toString()));
                 } else {
                     // We convert boolean fields to integers with values 0, 1 as this is the preferred
                     // way to consume such features in the analytics process.
-                    adjusted.add(new BooleanMapper<>(field, 1, 0));
+                    adjusted.add(ExtractedFields.applyBooleanMapping(field, 1, 0));
                 }
             } else {
                 adjusted.add(field);
@@ -282,33 +280,4 @@ private ExtractedFields fetchBooleanFieldsAsIntegers(ExtractedFields extractedFi
     private static boolean isBoolean(Set<String> types) {
         return types.size() == 1 && types.contains(BooleanFieldMapper.CONTENT_TYPE);
     }
-
-    /**
-     * {@link BooleanMapper} makes boolean field behave as a field of different type.
-     */
-    private static final class BooleanMapper<T> extends ExtractedField {
-
-        private final T trueValue;
-        private final T falseValue;
-
-        BooleanMapper(ExtractedField field, T trueValue, T falseValue) {
-            super(field.getAlias(), field.getName(), Collections.singleton(BooleanFieldMapper.CONTENT_TYPE), ExtractionMethod.DOC_VALUE);
-            this.trueValue = trueValue;
-            this.falseValue = falseValue;
-        }
-
-        @Override
-        public Object[] value(SearchHit hit) {
-            DocumentField keyValue = hit.field(name);
-            if (keyValue != null) {
-                return keyValue.getValues().stream().map(v -> Boolean.TRUE.equals(v) ? trueValue : falseValue).toArray();
-            }
-            return new Object[0];
-        }
-
-        @Override
-        public boolean supportsFromSource() {
-            return false;
-        }
-    }
 }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/AbstractField.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.extractor;
+
+import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.search.SearchHit;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+abstract class AbstractField implements ExtractedField {
+
+    private final String name;
+
+    private final Set<String> types;
+
+    AbstractField(String name, Set<String> types) {
+        this.name = Objects.requireNonNull(name);
+        this.types = Objects.requireNonNull(types);
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public String getSearchField() {
+        return name;
+    }
+
+    @Override
+    public Set<String> getTypes() {
+        return types;
+    }
+
+    protected Object[] getFieldValue(SearchHit hit) {
+        DocumentField keyValue = hit.field(getSearchField());
+        if (keyValue != null) {
+            List<Object> values = keyValue.getValues();
+            return values.toArray(new Object[0]);
+        }
+        return new Object[0];
+    }
+}

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/DocValueField.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.ml.extractor;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.search.SearchHit;
+
+import java.util.Set;
+
+public class DocValueField extends AbstractField {
+
+    public DocValueField(String name, Set<String> types) {
+        super(name, types);
+    }
+
+    @Override
+    public Method getMethod() {
+        return Method.DOC_VALUE;
+    }
+
+    @Override
+    public Object[] value(SearchHit hit) {
+        return getFieldValue(hit);
+    }
+
+    @Override
+    public boolean supportsFromSource() {
+        return true;
+    }
+
+    @Override
+    public ExtractedField newFromSource() {
+        return new SourceField(getSearchField(), getTypes());
+    }
+
+    @Override
+    public boolean isMultiField() {
+        return false;
+    }
+
+    @Nullable
+    public String getDocValueFormat() {
+        return null;
+    }
+}