Allow field mappers to retrieve fields from source. (#56928)

This PR adds new method `FieldMapper#lookupValues(SourceLookup)` that extracts
and parses the source values. This lets us return values like numbers and dates
in a consistent format, and also handle special data types like
`constant_keyword`. The `lookupValues` method calls into `parseSourceValue`,
which mappers can override to specify how values should be parsed.

Author: jtibshirani

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java

@@ -162,11 +162,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
         float value;
         if (context.externalValueSet()) {
             Object v = context.externalValue();
-            if (v instanceof Number) {
-                value = ((Number) v).floatValue();
-            } else {
-                value = Float.parseFloat(v.toString());
-            }
+            value = objectToFloat(v);
         } else if (context.parser().currentToken() == Token.VALUE_NULL) {
             // skip
             return;
@@ -186,6 +182,19 @@ protected void parseCreateField(ParseContext context) throws IOException {
         context.doc().addWithKey(name(), new FeatureField("_feature", name(), value));
     }
 
+    private Float objectToFloat(Object value) {
+        if (value instanceof Number) {
+            return ((Number) value).floatValue();
+        } else {
+            return Float.parseFloat(value.toString());
+        }
+    }
+
+    @Override
+    protected Float parseSourceValue(Object value) {
+        return objectToFloat(value);
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java

@@ -168,6 +168,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         throw new AssertionError("parse is implemented directly");
     }
 
+    @Override
+    protected Object parseSourceValue(Object value) {
+        return value;
+    }
+
     @Override
     protected boolean indexedByDefault() {
         return false;

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

@@ -496,6 +496,13 @@ private static double objectToDouble(Object value) {
         return doubleValue;
     }
 
+    @Override
+    protected Double parseSourceValue(Object value) {
+        double doubleValue = objectToDouble(value);
+        double scalingFactor = fieldType().getScalingFactor();
+        return Math.round(doubleValue * scalingFactor) / scalingFactor;
+    }
+
     private static class ScaledFloatIndexFieldData extends IndexNumericFieldData {
 
         private final IndexNumericFieldData scaledFieldData;

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java

@@ -491,6 +491,11 @@ protected void parseCreateField(ParseContext context) {
             throw new UnsupportedOperationException();
         }
 
+        @Override
+        protected Object parseSourceValue(Object value) {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         protected void mergeOptions(FieldMapper other, List<String> conflicts) {
 
@@ -532,6 +537,11 @@ protected void mergeOptions(FieldMapper other, List<String> conflicts) {
 
         }
 
+        @Override
+        protected Object parseSourceValue(Object value) {
+            throw new UnsupportedOperationException();
+        }
+
         @Override
         protected String contentType() {
             return "shingle";
@@ -684,6 +694,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         }
     }
 
+    @Override
+    protected String parseSourceValue(Object value) {
+        return value.toString();
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java

@@ -158,6 +158,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         context.doc().addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, indexed, docValued, stored));
     }
 
+    @Override
+    protected String parseSourceValue(Object value) {
+        return value.toString();
+    }
+
     /**
      * Count position increments in a token stream.  Package private for testing.
      * @param analyzer analyzer to create token stream

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java

@@ -23,9 +23,12 @@
 import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute;
 import org.apache.lucene.document.FeatureField;
 import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
@@ -186,4 +189,12 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
                 e.getCause().getMessage());
     }
 
+    public void testParseSourceValue() {
+        Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
+        Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
+        RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context);
+
+        assertEquals(3.14f, mapper.parseSourceValue(3.14), 0.0001);
+        assertEquals(42.9f, mapper.parseSourceValue("42.9"), 0.0001);
+    }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -21,9 +21,12 @@
 
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
@@ -398,4 +401,15 @@ public void testMeta() throws Exception {
                 new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE);
         assertEquals(mapping3, mapper.mappingSource().toString());
     }
+
+    public void testParseSourceValue() {
+        Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
+        Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
+        ScaledFloatFieldMapper mapper = new ScaledFloatFieldMapper.Builder("field")
+            .scalingFactor(100)
+            .build(context);
+
+        assertEquals(3.14, mapper.parseSourceValue(3.1415926), 0.00001);
+        assertEquals(3.14, mapper.parseSourceValue("3.1415"), 0.00001);
+    }
 }

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java

@@ -144,6 +144,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         throw new IllegalStateException("Should never be called");
     }
 
+    @Override
+    protected Object parseSourceValue(Object value) {
+        throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
+    }
+
     @Override
     protected String contentType() {
         return CONTENT_TYPE;

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java

@@ -194,6 +194,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         context.doc().add(new SortedDocValuesField(fieldType().name(), binaryValue));
     }
 
+    @Override
+    protected Object parseSourceValue(Object value) {
+        throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
+    }
+
     @Override
     protected void mergeOptions(FieldMapper other, List<String> conflicts) {
         ParentIdFieldMapper parentMergeWith = (ParentIdFieldMapper) other;

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java

@@ -356,6 +356,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
         throw new UnsupportedOperationException("parsing is implemented in parse(), this method should NEVER be called");
     }
 
+    @Override
+    protected Object parseSourceValue(Object value) {
+        return value;
+    }
+
     @Override
     public void parse(ParseContext context) throws IOException {
         context.path().add(simpleName());