diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java index f1174a3006298..df2014a179f63 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.time.ZoneId; import java.util.List; +import java.util.Locale; import java.util.Map; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; @@ -141,4 +142,21 @@ protected final void checkAllowExpensiveQueries(QueryShardContext context) { ); } } + + /** + * The format that this field should use. The default implementation is + * {@code null} because most fields don't support formats. + */ + protected String format() { + return null; + } + + /** + * The locale that this field's format should use. The default + * implementation is {@code null} because most fields don't + * support formats. + */ + protected Locale formatLocale() { + return null; + } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java index 37ada5fb0f790..8bf22ee222ea8 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java @@ -6,10 +6,11 @@ package org.elasticsearch.xpack.runtimefields.mapper; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.ParametrizedFieldMapper; @@ -23,6 +24,7 @@ import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.BiFunction; @@ -43,7 +45,7 @@ public FactoryType compile(Script script, ScriptContext> FIELD_TYPE_RESOLVER = Map.of( + static final Map> FIELD_TYPE_RESOLVER = Map.of( DateFieldMapper.CONTENT_TYPE, (builder, context) -> { DateScriptFieldScript.Factory factory = builder.scriptCompiler.compile( builder.script.getValue(), DateScriptFieldScript.CONTEXT ); + String format = builder.format.getValue(); + if (format == null) { + format = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.pattern(); + } + Locale locale = builder.locale.getValue(); + if (locale == null) { + locale = Locale.ROOT; + } + DateFormatter dateTimeFormatter = DateFormatter.forPattern(format).withLocale(locale); return new ScriptDateMappedFieldType( builder.buildFullName(context), builder.script.getValue(), factory, + dateTimeFormatter, builder.meta.getValue() ); }, NumberType.DOUBLE.typeName(), (builder, context) -> { + builder.formatAndLocaleNotSupported(); DoubleScriptFieldScript.Factory factory = builder.scriptCompiler.compile( builder.script.getValue(), DoubleScriptFieldScript.CONTEXT @@ -107,6 +120,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { }, KeywordFieldMapper.CONTENT_TYPE, (builder, context) -> { + builder.formatAndLocaleNotSupported(); StringScriptFieldScript.Factory factory = builder.scriptCompiler.compile( builder.script.getValue(), StringScriptFieldScript.CONTEXT @@ -120,6 +134,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { }, NumberType.LONG.typeName(), (builder, context) -> { + builder.formatAndLocaleNotSupported(); LongScriptFieldScript.Factory factory = builder.scriptCompiler.compile( builder.script.getValue(), LongScriptFieldScript.CONTEXT @@ -159,6 +174,27 @@ private static RuntimeScriptFieldMapper toType(FieldMapper in) { throw new IllegalArgumentException("script must be specified for " + CONTENT_TYPE + " field [" + name + "]"); } }); + private final Parameter format = Parameter.stringParam( + "format", + true, + mapper -> ((AbstractScriptMappedFieldType) mapper.fieldType()).format(), + null + ).setSerializer((b, n, v) -> { + if (v != null && false == v.equals(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.pattern())) { + b.field(n, v); + } + }).acceptsNull(); + private final Parameter locale = new Parameter<>( + "locale", + true, + () -> null, + (n, c, o) -> o == null ? null : LocaleUtils.parse(o.toString()), + mapper -> ((AbstractScriptMappedFieldType) mapper.fieldType()).formatLocale() + ).setSerializer((b, n, v) -> { + if (v != null && false == v.equals(Locale.ROOT)) { + b.field(n, v.toString()); + } + }).acceptsNull(); private final ScriptCompiler scriptCompiler; @@ -169,12 +205,12 @@ protected Builder(String name, ScriptCompiler scriptCompiler) { @Override protected List> getParameters() { - return List.of(meta, runtimeType, script); + return List.of(meta, runtimeType, script, format, locale); } @Override public RuntimeScriptFieldMapper build(BuilderContext context) { - BiFunction fieldTypeResolver = Builder.FIELD_TYPE_RESOLVER.get( + BiFunction fieldTypeResolver = Builder.FIELD_TYPE_RESOLVER.get( runtimeType.getValue() ); if (fieldTypeResolver == null) { @@ -203,6 +239,15 @@ static Script parseScript(String name, Mapper.TypeParser.ParserContext parserCon } return script; } + + private void formatAndLocaleNotSupported() { + if (format.getValue() != null) { + throw new IllegalArgumentException("format can not be specified for runtime_type [" + runtimeType.getValue() + "]"); + } + if (locale.getValue() != null) { + throw new IllegalArgumentException("locale can not be specified for runtime_type [" + runtimeType.getValue() + "]"); + } + } } @FunctionalInterface diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java index 8469904cb8e35..aa3c9c651651f 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java @@ -32,19 +32,24 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Supplier; public class ScriptDateMappedFieldType extends AbstractScriptMappedFieldType { private final DateScriptFieldScript.Factory scriptFactory; - - ScriptDateMappedFieldType(String name, Script script, DateScriptFieldScript.Factory scriptFactory, Map meta) { + private final DateFormatter dateTimeFormatter; + + ScriptDateMappedFieldType( + String name, + Script script, + DateScriptFieldScript.Factory scriptFactory, + DateFormatter dateTimeFormatter, + Map meta + ) { super(name, script, meta); this.scriptFactory = scriptFactory; - } - - private DateFormatter dateTimeFormatter() { - return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; // TODO make configurable + this.dateTimeFormatter = dateTimeFormatter; } @Override @@ -58,12 +63,12 @@ public Object valueForDisplay(Object value) { if (val == null) { return null; } - return dateTimeFormatter().format(Resolution.MILLISECONDS.toInstant(val).atZone(ZoneOffset.UTC)); + return dateTimeFormatter.format(Resolution.MILLISECONDS.toInstant(val).atZone(ZoneOffset.UTC)); } @Override public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { - DateFormatter dateTimeFormatter = dateTimeFormatter(); + DateFormatter dateTimeFormatter = this.dateTimeFormatter; if (format != null) { dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale()); } @@ -99,7 +104,7 @@ public Query rangeQuery( @Nullable DateMathParser parser, QueryShardContext context ) { - parser = parser == null ? DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.toDateMathParser() : parser; + parser = parser == null ? dateTimeFormatter.toDateMathParser() : parser; checkAllowExpensiveQueries(context); return DateFieldType.dateRangeQuery( lowerTerm, @@ -121,7 +126,7 @@ public Query termQuery(Object value, QueryShardContext context) { value, false, null, - dateTimeFormatter().toDateMathParser(), + dateTimeFormatter.toDateMathParser(), now, DateFieldMapper.Resolution.MILLISECONDS ); @@ -143,7 +148,7 @@ public Query termsQuery(List values, QueryShardContext context) { value, false, null, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.toDateMathParser(), + dateTimeFormatter.toDateMathParser(), now, DateFieldMapper.Resolution.MILLISECONDS ) @@ -153,4 +158,14 @@ public Query termsQuery(List values, QueryShardContext context) { return new LongScriptFieldTermsQuery(script, leafFactory(context.lookup())::newInstance, name(), terms); }); } + + @Override + protected String format() { + return dateTimeFormatter.pattern(); + } + + @Override + protected Locale formatLocale() { + return dateTimeFormatter.locale(); + } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java index 7ba5ebf709857..3782d7b6384d1 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java @@ -8,6 +8,8 @@ import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -34,6 +36,7 @@ import java.util.Set; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; public class RuntimeScriptFieldMapperTests extends ESSingleNodeTestCase { @@ -145,6 +148,51 @@ public void testDate() throws IOException { assertEquals(Strings.toString(mapping("date")), Strings.toString(mapperService.documentMapper())); } + public void testDateWithFormat() throws IOException { + CheckedSupplier mapping = () -> mapping("date", b -> b.field("format", "yyyy-MM-dd")); + MapperService mapperService = createIndex("test", Settings.EMPTY, mapping.get()).mapperService(); + FieldMapper mapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertThat(mapper, instanceOf(RuntimeScriptFieldMapper.class)); + assertEquals(Strings.toString(mapping.get()), Strings.toString(mapperService.documentMapper())); + } + + public void testDateWithLocale() throws IOException { + CheckedSupplier mapping = () -> mapping("date", b -> b.field("locale", "en_GB")); + MapperService mapperService = createIndex("test", Settings.EMPTY, mapping.get()).mapperService(); + FieldMapper mapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertThat(mapper, instanceOf(RuntimeScriptFieldMapper.class)); + assertEquals(Strings.toString(mapping.get()), Strings.toString(mapperService.documentMapper())); + } + + public void testDateWithLocaleAndFormat() throws IOException { + CheckedSupplier mapping = () -> mapping( + "date", + b -> b.field("format", "yyyy-MM-dd").field("locale", "en_GB") + ); + MapperService mapperService = createIndex("test", Settings.EMPTY, mapping.get()).mapperService(); + FieldMapper mapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertThat(mapper, instanceOf(RuntimeScriptFieldMapper.class)); + assertEquals(Strings.toString(mapping.get()), Strings.toString(mapperService.documentMapper())); + } + + public void testNonDateWithFormat() throws IOException { + String runtimeType = randomValueOtherThan("date", () -> randomFrom(runtimeTypes)); + Exception e = expectThrows( + MapperParsingException.class, + () -> createIndex("test", Settings.EMPTY, mapping(runtimeType, b -> b.field("format", "yyyy-MM-dd"))) + ); + assertThat(e.getMessage(), equalTo("Failed to parse mapping: format can not be specified for runtime_type [" + runtimeType + "]")); + } + + public void testNonDateWithLocale() throws IOException { + String runtimeType = randomValueOtherThan("date", () -> randomFrom(runtimeTypes)); + Exception e = expectThrows( + MapperParsingException.class, + () -> createIndex("test", Settings.EMPTY, mapping(runtimeType, b -> b.field("locale", "en_GB"))) + ); + assertThat(e.getMessage(), equalTo("Failed to parse mapping: locale can not be specified for runtime_type [" + runtimeType + "]")); + } + public void testFieldCaps() throws Exception { for (String runtimeType : runtimeTypes) { String scriptIndex = "test_" + runtimeType + "_script"; @@ -178,6 +226,10 @@ public void testFieldCaps() throws Exception { } private XContentBuilder mapping(String type) throws IOException { + return mapping(type, builder -> {}); + } + + private XContentBuilder mapping(String type, CheckedConsumer extra) throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject(); { mapping.startObject("_doc"); @@ -192,6 +244,7 @@ private XContentBuilder mapping(String type) throws IOException { mapping.field("source", "dummy_source").field("lang", "test"); } mapping.endObject(); + extra.accept(mapping); } mapping.endObject(); } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldTypeTests.java index bdc815ac50a42..cebe31f7a5c25 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldTypeTests.java @@ -23,10 +23,14 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.plugins.ScriptPlugin; @@ -55,6 +59,7 @@ import java.util.function.BiConsumer; import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class ScriptDateMappedFieldTypeTests extends AbstractNonTextScriptMappedFieldTypeTestCase { @@ -72,6 +77,23 @@ public void testFormat() throws IOException { simpleMappedFieldType().docValueFormat(null, ZoneId.of("America/New_York")).format(1595432181354L), equalTo("2020-07-22T11:36:21.354-04:00") ); + assertThat(coolFormattedFieldType().docValueFormat(null, null).format(1595432181354L), equalTo("2020-07-22(-■_■)15:36:21.354Z")); + } + + public void testFormatDuel() throws IOException { + DateFormatter formatter = DateFormatter.forPattern(randomFrom(FormatNames.values()).getSnakeCaseName()) + .withLocale(randomLocale(random())); + ScriptDateMappedFieldType scripted = build(new Script(ScriptType.INLINE, "test", "read_timestamp", Map.of()), formatter); + DateFieldMapper.DateFieldType indexed = new DateFieldMapper.DateFieldType("test", formatter); + for (int i = 0; i < 100; i++) { + long date = randomLongBetween(0, 3000000000000L); // Maxes out in the year 2065 + assertThat(indexed.docValueFormat(null, null).format(date), equalTo(scripted.docValueFormat(null, null).format(date))); + String format = randomFrom(FormatNames.values()).getSnakeCaseName(); + assertThat(indexed.docValueFormat(format, null).format(date), equalTo(scripted.docValueFormat(format, null).format(date))); + ZoneId zone = randomZone(); + assertThat(indexed.docValueFormat(null, zone).format(date), equalTo(scripted.docValueFormat(null, zone).format(date))); + assertThat(indexed.docValueFormat(format, zone).format(date), equalTo(scripted.docValueFormat(format, zone).format(date))); + } } @Override @@ -224,6 +246,35 @@ public void testRangeQuery() throws IOException { ), equalTo(0) ); + checkBadDate( + () -> searcher.count( + ft.rangeQuery( + "2020-07-22(-■_■)00:00:00.000Z", + "2020-07-23(-■_■)00:00:00.000Z", + false, + false, + null, + null, + null, + mockContext() + ) + ) + ); + assertThat( + searcher.count( + coolFormattedFieldType().rangeQuery( + "2020-07-22(-■_■)00:00:00.000Z", + "2020-07-23(-■_■)00:00:00.000Z", + false, + false, + null, + null, + null, + mockContext() + ) + ), + equalTo(3) + ); } } } @@ -250,6 +301,8 @@ public void testTermQuery() throws IOException { searcher.count(build("add_days", Map.of("days", 1)).termQuery("2020-07-23T15:36:21.354Z", mockContext())), equalTo(1) ); + checkBadDate(() -> searcher.count(simpleMappedFieldType().termQuery("2020-07-22(-■_■)15:36:21.354Z", mockContext()))); + assertThat(searcher.count(coolFormattedFieldType().termQuery("2020-07-22(-■_■)15:36:21.354Z", mockContext())), equalTo(1)); } } } @@ -274,6 +327,23 @@ public void testTermsQuery() throws IOException { assertThat(searcher.count(ft.termsQuery(List.of(1595432181354L, 2595432181354L), mockContext())), equalTo(1)); assertThat(searcher.count(ft.termsQuery(List.of(2595432181354L, 1595432181354L), mockContext())), equalTo(1)); assertThat(searcher.count(ft.termsQuery(List.of(1595432181355L, 1595432181354L), mockContext())), equalTo(2)); + checkBadDate( + () -> searcher.count( + simpleMappedFieldType().termsQuery( + List.of("2020-07-22T15:36:21.354Z", "2020-07-22(-■_■)15:36:21.354Z"), + mockContext() + ) + ) + ); + assertThat( + searcher.count( + coolFormattedFieldType().termsQuery( + List.of("2020-07-22(-■_■)15:36:21.354Z", "2020-07-22(-■_■)15:36:21.355Z"), + mockContext() + ) + ), + equalTo(2) + ); } } } @@ -288,6 +358,10 @@ protected ScriptDateMappedFieldType simpleMappedFieldType() throws IOException { return build("read_timestamp"); } + private ScriptDateMappedFieldType coolFormattedFieldType() throws IOException { + return build(simpleMappedFieldType().script, DateFormatter.forPattern("yyyy-MM-dd(-■_■)HH:mm:ss.SSSz")); + } + @Override protected String runtimeType() { return "date"; @@ -298,10 +372,10 @@ private static ScriptDateMappedFieldType build(String code) throws IOException { } private static ScriptDateMappedFieldType build(String code, Map params) throws IOException { - return build(new Script(ScriptType.INLINE, "test", code, params)); + return build(new Script(ScriptType.INLINE, "test", code, params), DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER); } - private static ScriptDateMappedFieldType build(Script script) throws IOException { + private static ScriptDateMappedFieldType build(Script script, DateFormatter dateTimeFormatter) throws IOException { ScriptPlugin scriptPlugin = new ScriptPlugin() { @Override public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { @@ -361,7 +435,7 @@ public void execute() { ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(scriptPlugin, new RuntimeFields())); try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { DateScriptFieldScript.Factory factory = scriptService.compile(script, DateScriptFieldScript.CONTEXT); - return new ScriptDateMappedFieldType("test", script, factory, emptyMap()); + return new ScriptDateMappedFieldType("test", script, factory, dateTimeFormatter, emptyMap()); } } @@ -373,4 +447,9 @@ private void checkExpensiveQuery(BiConsumer