Make dates be ReadableDateTimes in scripts

core/src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java

@@ -30,7 +30,7 @@ public interface AtomicFieldData extends Accountable, Releasable {
     /**
      * Returns a "scripting" based values.
      */
-    ScriptDocValues getScriptValues();
+    ScriptDocValues<?> getScriptValues();
 
     /**
      * Return a String representation of the values.

core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java

@@ -27,6 +27,7 @@ public static enum NumericType {
         SHORT(false),
         INT(false),
         LONG(false),
+        DATE(false),
         HALF_FLOAT(true),
         FLOAT(true),
         DOUBLE(true);

core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java

@@ -25,6 +25,8 @@
 import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.MutableDateTime;
@@ -41,7 +43,6 @@
  * and a <code>getValues</code> that return the relevant type that then can be used in scripts.
  */
 public abstract class ScriptDocValues<T> extends AbstractList<T> {
-
     /**
      * Set the current doc ID.
      */
@@ -127,6 +128,7 @@ public int size() {
     }
 
     public static final class Longs extends ScriptDocValues<Long> {
+        protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Longs.class));
 
         private final SortedNumericDocValues values;
         private Dates dates;
@@ -155,15 +157,19 @@ public long getValue() {
             return values.valueAt(0);
         }
 
+        @Deprecated
         public ReadableDateTime getDate() {
+            deprecationLogger.deprecated("getDate on numeric fields is deprecated. Use a date field to get dates.");
             if (dates == null) {
                 dates = new Dates(values);
                 dates.refreshArray();
             }
             return dates.getValue();
         }
 
+        @Deprecated
         public List<ReadableDateTime> getDates() {
+            deprecationLogger.deprecated("getDates on numeric fields is deprecated. Use a date field to get dates.");
             if (dates == null) {
                 dates = new Dates(values);
                 dates.refreshArray();
@@ -183,6 +189,8 @@ public int size() {
     }
 
     public static final class Dates extends ScriptDocValues<ReadableDateTime> {
+        protected static final DeprecationLogger deprecationLogger = new DeprecationLogger(ESLoggerFactory.getLogger(Dates.class));
+
         private static final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC);
 
         private final SortedNumericDocValues values;
@@ -206,6 +214,24 @@ public ReadableDateTime getValue() {
             return get(0);
         }
 
+        /**
+         * Fetch the first value. Added for backwards compatibility with 5.x when date fields were {@link Longs}.
+         */
+        @Deprecated
+        public ReadableDateTime getDate() {
+            deprecationLogger.deprecated("getDate is no longer necisary on date fields as the value is now a date.");
+            return getValue();
+        }
+
+        /**
+         * Fetch all the values. Added for backwards compatibility with 5.x when date fields were {@link Longs}.
+         */
+        @Deprecated
+        public List<ReadableDateTime> getDates() {
+            deprecationLogger.deprecated("getDates is no longer necisary on date fields as the values are now dates.");
+            return this;
+        }
+
         @Override
         public ReadableDateTime get(int index) {
             if (index >= values.count()) {

core/src/main/java/org/elasticsearch/index/fielddata/plain/AtomicLongFieldData.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
@@ -31,12 +32,14 @@
 abstract class AtomicLongFieldData implements AtomicNumericFieldData {
 
     private final long ramBytesUsed;
-    /** True if this numeric data is for a boolean field, and so only has values 0 and 1. */
-    private final boolean isBoolean;
+    /**
+     * Type of this field. Used to expose appropriate types in {@link #getScriptValues()}.
+     */
+    private final NumericType numericType;
 
-    AtomicLongFieldData(long ramBytesUsed, boolean isBoolean) {
+    AtomicLongFieldData(long ramBytesUsed, NumericType numericType) {
         this.ramBytesUsed = ramBytesUsed;
-        this.isBoolean = isBoolean;
+        this.numericType = numericType;
     }
 
     @Override
@@ -45,10 +48,13 @@ public long ramBytesUsed() {
     }
 
     @Override
-    public final ScriptDocValues getScriptValues() {
-        if (isBoolean) {
+    public final ScriptDocValues<?> getScriptValues() {
+        switch (numericType) {
+        case DATE:
+            return new ScriptDocValues.Dates(getLongValues());
+        case BOOLEAN:
             return new ScriptDocValues.Booleans(getLongValues());
-        } else {
+        default:
             return new ScriptDocValues.Longs(getLongValues());
         }
     }

core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java

@@ -96,7 +96,7 @@ public AtomicNumericFieldData load(LeafReaderContext context) {
             case DOUBLE:
                 return new SortedNumericDoubleFieldData(reader, field);
             default:
-                return new SortedNumericLongFieldData(reader, field, numericType == NumericType.BOOLEAN);
+                return new SortedNumericLongFieldData(reader, field, numericType);
         }
     }
 
@@ -117,8 +117,8 @@ static final class SortedNumericLongFieldData extends AtomicLongFieldData {
         final LeafReader reader;
         final String field;
 
-        SortedNumericLongFieldData(LeafReader reader, String field, boolean isBoolean) {
-            super(0L, isBoolean);
+        SortedNumericLongFieldData(LeafReader reader, String field, NumericType numericType) {
+            super(0L, numericType);
             this.reader = reader;
             this.field = field;
         }

core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -376,7 +376,7 @@ public Relation isFieldWithinQuery(IndexReader reader,
         @Override
         public IndexFieldData.Builder fielddataBuilder() {
             failIfNoDocValues();
-            return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG);
+            return new DocValuesIndexFieldData.Builder().numericType(NumericType.DATE);
         }
 
         @Override

core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptDoubleValues.java

@@ -23,10 +23,10 @@
 import org.elasticsearch.index.fielddata.SortingNumericDoubleValues;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
+import org.joda.time.ReadableInstant;
 
 import java.lang.reflect.Array;
 import java.util.Collection;
-import java.util.Iterator;
 
 /**
  * {@link SortingNumericDoubleValues} implementation which is based on a script
@@ -47,36 +47,49 @@ public void setDocument(int docId) {
 
         if (value == null) {
             resize(0);
-        }
-
-        else if (value instanceof Number) {
+        } else if (value instanceof Number) {
             resize(1);
             values[0] = ((Number) value).doubleValue();
-        }
-
-        else if (value.getClass().isArray()) {
+        } else if (value instanceof ReadableInstant) {
+            resize(1);
+            values[0] = ((ReadableInstant) value).getMillis();
+        } else if (value.getClass().isArray()) {
             resize(Array.getLength(value));
             for (int i = 0; i < count(); ++i) {
-                values[i] = ((Number) Array.get(value, i)).doubleValue();
+                values[i] = toDoubleValue(Array.get(value, i));
             }
-        }
-
-        else if (value instanceof Collection) {
+        } else if (value instanceof Collection) {
             resize(((Collection<?>) value).size());
             int i = 0;
-            for (Iterator<?> it = ((Collection<?>) value).iterator(); it.hasNext(); ++i) {
-                values[i] = ((Number) it.next()).doubleValue();
+            for (Object v : (Collection<?>) value) {
+                values[i++] = toDoubleValue(v);
             }
             assert i == count();
-        }
-
-        else {
-            throw new AggregationExecutionException("Unsupported script value [" + value + "]");
+        } else {
+            resize(1);
+            values[0] = toDoubleValue(value);
         }
 
         sort();
     }
 
+    private static double toDoubleValue(Object o) {
+        if (o instanceof Number) {
+            return ((Number) o).doubleValue();
+        } else if (o instanceof ReadableInstant) {
+            // Dates are exposed in scripts as ReadableDateTimes but aggregations want them to be numeric
+            return ((ReadableInstant) o).getMillis();
+        } else if (o instanceof Boolean) {
+            // We do expose boolean fields as boolean in scripts, however aggregations still expect
+            // that scripts return the same internal representation as regular fields, so boolean
+            // values in scripts need to be converted to a number, and the value formatter will
+            // make sure of using true/false in the key_as_string field
+            return ((Boolean) o).booleanValue() ? 1.0 : 0.0;
+        } else {
+            throw new AggregationExecutionException("Unsupported script value [" + o + "], expected a number, date, or boolean");
+        }
+    }
+
     @Override
     public void setScorer(Scorer scorer) {
         script.setScorer(scorer);

core/src/main/java/org/elasticsearch/search/aggregations/support/values/ScriptLongValues.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.index.fielddata.SortingNumericDocValues;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
+import org.joda.time.ReadableInstant;
 
 import java.lang.reflect.Array;
 import java.util.Collection;
@@ -77,14 +78,17 @@ else if (value instanceof Collection) {
     private static long toLongValue(Object o) {
         if (o instanceof Number) {
             return ((Number) o).longValue();
+        } else if (o instanceof ReadableInstant) {
+            // Dates are exposed in scripts as ReadableDateTimes but aggregations want them to be numeric
+            return ((ReadableInstant) o).getMillis();
         } else if (o instanceof Boolean) {
             // We do expose boolean fields as boolean in scripts, however aggregations still expect
             // that scripts return the same internal representation as regular fields, so boolean
             // values in scripts need to be converted to a number, and the value formatter will
             // make sure of using true/false in the key_as_string field
             return ((Boolean) o).booleanValue() ? 1L : 0L;
         } else {
-            throw new AggregationExecutionException("Unsupported script value [" + o + "], expected a number");
+            throw new AggregationExecutionException("Unsupported script value [" + o + "], expected a number, date, or boolean");
         }
     }
 

core/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java

@@ -62,10 +62,12 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
             }
             MappedFieldType fieldType = context.mapperService().fullName(field);
             if (fieldType != null) {
+                /* Because this is called once per document we end up creating a new ScriptDocValues for every document which is important
+                 * because the values inside ScriptDocValues might be reused for different documents (Dates do this). */
                 AtomicFieldData data = context.fieldData().getForField(fieldType).load(hitContext.readerContext());
-                ScriptDocValues values = data.getScriptValues();
+                ScriptDocValues<?> values = data.getScriptValues();
                 values.setNextDocId(hitContext.docId());
-                hitField.values().addAll(values.getValues());
+                hitField.values().addAll(values);
             }
         }
     }

core/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsFetchSubPhase.java

@@ -39,6 +39,8 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
             return;
         }
         for (ScriptFieldsContext.ScriptField scriptField : context.scriptFields().fields()) {
+            /* Because this is called once per document we end up creating new ScriptDocValues for every document which is important because
+             * the values inside ScriptDocValues might be reused for different documents (Dates do this). */
             LeafSearchScript leafScript;
             try {
                 leafScript = scriptField.script().getLeafSearchScript(hitContext.readerContext());

core/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java

@@ -80,6 +80,10 @@ public void testDates() {
             Exception e = expectThrows(UnsupportedOperationException.class, () -> longs.getDates().add(new DateTime()));
             assertEquals("doc values are unmodifiable", e.getMessage());
         }
+
+        assertWarnings(
+                "getDate on numeric fields is deprecated. Use a date field to get dates.",
+                "getDates on numeric fields is deprecated. Use a date field to get dates.");
     }
 
     private Longs wrap(long[][] values) {

core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java

@@ -84,7 +84,7 @@ public static class PlusOneMonthScriptFactory implements NativeScriptFactory {
 
         @Override
         public ExecutableScript newScript(Map<String, Object> params) {
-            return new PlusOneMonthScript((String) params.get("fieldname"));
+            return new PlusOneMonthScript();
         }
 
         @Override
@@ -104,14 +104,9 @@ public String getName() {
     public static class PlusOneMonthScript extends AbstractSearchScript {
 
         public static final String NAME = "date_plus_1_month";
-        private String fieldname;
 
         private Map<String, Object> vars = new HashMap<>();
 
-        public PlusOneMonthScript(String fieldname) {
-            this.fieldname  = fieldname;
-        }
-
         @Override
         public void setNextVar(String name, Object value) {
             vars.put(name, value);

core/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java

@@ -45,6 +45,7 @@
 import org.elasticsearch.test.ESIntegTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.joda.time.ReadableDateTime;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -105,8 +106,8 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
 
             scripts.put("doc['date'].date.millis", vars -> {
                 Map<?, ?> doc = (Map) vars.get("doc");
-                ScriptDocValues.Longs date = (ScriptDocValues.Longs) doc.get("date");
-                return date.getDate().getMillis();
+                ScriptDocValues.Dates dates = (ScriptDocValues.Dates) doc.get("date");
+                return dates.getValue().getMillis();
             });
 
             scripts.put("_fields['num1'].value", vars -> fieldsScript(vars, "num1"));
@@ -777,6 +778,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
 
         client().admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet();
 
+        ReadableDateTime date = new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC);
         client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
                 .field("text_field", "foo")
                 .field("keyword_field", "foo")
@@ -786,7 +788,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
                 .field("long_field", 4L)
                 .field("float_field", 5.0f)
                 .field("double_field", 6.0d)
-                .field("date_field", Joda.forPattern("dateOptionalTime").printer().print(new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC)))
+                .field("date_field", Joda.forPattern("dateOptionalTime").printer().print(date))
                 .field("boolean_field", true)
                 .field("ip_field", "::1")
                 .endObject()).execute().actionGet();
@@ -820,7 +822,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
         assertThat(searchResponse.getHits().getAt(0).fields().get("long_field").value(), equalTo((Object) 4L));
         assertThat(searchResponse.getHits().getAt(0).fields().get("float_field").value(), equalTo((Object) 5.0));
         assertThat(searchResponse.getHits().getAt(0).fields().get("double_field").value(), equalTo((Object) 6.0d));
-        assertThat(searchResponse.getHits().getAt(0).fields().get("date_field").value(), equalTo((Object) 1332374400000L));
+        assertThat(searchResponse.getHits().getAt(0).fields().get("date_field").value(), equalTo(date));
         assertThat(searchResponse.getHits().getAt(0).fields().get("boolean_field").value(), equalTo((Object) true));
         assertThat(searchResponse.getHits().getAt(0).fields().get("text_field").value(), equalTo("foo"));
         assertThat(searchResponse.getHits().getAt(0).fields().get("keyword_field").value(), equalTo("foo"));

docs/reference/migration/migrate_6_0/scripting.asciidoc

@@ -5,3 +5,10 @@
 
 The groovy scripting language was deprecated in elasticsearch 5.0 and is now removed.
 Use painless instead.
+
+==== Date fields now return dates
+
+`doc.some_date_field.value` now returns `ReadableDateTime`s instead of
+milliseconds since epoch as a `long`. The same is true for
+`doc.some_date_field[some_number]`. Use `doc.some_date_field.value.millis` to
+fetch the milliseconds since epoch if you need it.