Move runtime fields to server (#69223)

This commit makes a start on moving runtime fields to server.

The runtime field types are now built-in. The dynamic fields builder extension (needed to make dynamic:runtime work) is removed: further simplifications are needed but are left for a follow-up. It is still possible to plug in custom runtime field types through the same extension point that the xpack plugin used to use.

The runtime fields script contexts are now also moved to server, and the corresponding whitelists are part of painless-lang with the exception of grok and dissect which will require additional work.

The runtime fields xpack plugin still exists to hold the grok and dissect extensions needed, which depend on grok and dissect lib and we will address as a follow-up. Also, the qa tests and yaml tests are yet to be moved.

Despite the need for additional work, runtime fields are fully functional and users should not be impacted by this change.

Author: javanna

modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java

@@ -37,6 +37,13 @@
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.runtimefields.mapper.BooleanFieldScript;
+import org.elasticsearch.runtimefields.mapper.DateFieldScript;
+import org.elasticsearch.runtimefields.mapper.DoubleFieldScript;
+import org.elasticsearch.runtimefields.mapper.GeoPointFieldScript;
+import org.elasticsearch.runtimefields.mapper.IpFieldScript;
+import org.elasticsearch.runtimefields.mapper.LongFieldScript;
+import org.elasticsearch.runtimefields.mapper.StringFieldScript;
 import org.elasticsearch.script.IngestScript;
 import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.ScriptContext;
@@ -88,6 +95,15 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin, Extens
         ingest.add(ingestWhitelist);
         map.put(IngestScript.CONTEXT, ingest);
 
+        // Functions available to runtime fields
+        map.put(BooleanFieldScript.CONTEXT, getRuntimeFieldWhitelist("boolean"));
+        map.put(DateFieldScript.CONTEXT, getRuntimeFieldWhitelist("date"));
+        map.put(DoubleFieldScript.CONTEXT, getRuntimeFieldWhitelist("double"));
+        map.put(LongFieldScript.CONTEXT, getRuntimeFieldWhitelist("long"));
+        map.put(StringFieldScript.CONTEXT, getRuntimeFieldWhitelist("string"));
+        map.put(GeoPointFieldScript.CONTEXT, getRuntimeFieldWhitelist("geopoint"));
+        map.put(IpFieldScript.CONTEXT, getRuntimeFieldWhitelist("ip"));
+
         // Execute context gets everything
         List<Whitelist> test = new ArrayList<>(Whitelist.BASE_WHITELISTS);
         test.add(movFnWhitelist);
@@ -99,6 +115,14 @@ public final class PainlessPlugin extends Plugin implements ScriptPlugin, Extens
         whitelists = map;
     }
 
+    private static List<Whitelist> getRuntimeFieldWhitelist(String fieldType) {
+        List<Whitelist> scriptField = new ArrayList<>(Whitelist.BASE_WHITELISTS);
+        Whitelist whitelist = WhitelistLoader.loadFromResourceFiles(Whitelist.class,
+            "org.elasticsearch.runtimefields." + fieldType + ".txt");
+        scriptField.add(whitelist);
+        return scriptField;
+    }
+
     private final SetOnce<PainlessScriptEngine> painlessScriptEngine = new SetOnce<>();
 
     @Override

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.boolean.txt

@@ -0,0 +1,21 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for boolean-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.BooleanFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.BooleanFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.BooleanFieldScript, boolean) bound_to org.elasticsearch.runtimefields.mapper.BooleanFieldScript$Emit
+}
+

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.date.txt

@@ -0,0 +1,25 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for date-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.DateFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.DateFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.DateFieldScript, long) bound_to org.elasticsearch.runtimefields.mapper.DateFieldScript$Emit
+    # Parse a value from the source to millis since epoch
+    long parse(org.elasticsearch.runtimefields.mapper.DateFieldScript, def) bound_to org.elasticsearch.runtimefields.mapper.DateFieldScript$Parse
+}
+
+
+

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.double.txt

@@ -0,0 +1,21 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for double-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.DoubleFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.DoubleFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.DoubleFieldScript, double) bound_to org.elasticsearch.runtimefields.mapper.DoubleFieldScript$Emit
+}
+

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.geopoint.txt

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for ip-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.GeoPointFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.GeoPointFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.GeoPointFieldScript, double, double) bound_to org.elasticsearch.runtimefields.mapper.GeoPointFieldScript$Emit
+}

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.ip.txt

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for ip-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.IpFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.IpFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.IpFieldScript, String) bound_to org.elasticsearch.runtimefields.mapper.IpFieldScript$Emit
+}

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.long.txt

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for long-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.LongFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.LongFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.LongFieldScript, long) bound_to org.elasticsearch.runtimefields.mapper.LongFieldScript$Emit
+}

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.runtimefields.string.txt

@@ -0,0 +1,20 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the Server Side Public License, v 1; you may not use this file except
+# in compliance with, at your election, the Elastic License 2.0 or the Server
+# Side Public License, v 1.
+#
+
+# The whitelist for string-valued runtime fields
+
+# These two whitelists are required for painless to find the classes
+class org.elasticsearch.runtimefields.mapper.StringFieldScript @no_import {
+}
+class org.elasticsearch.runtimefields.mapper.StringFieldScript$Factory @no_import {
+}
+
+static_import {
+    # The `emit` callback to collect values for the field
+    void emit(org.elasticsearch.runtimefields.mapper.StringFieldScript, String) bound_to org.elasticsearch.runtimefields.mapper.StringFieldScript$Emit
+}

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -8,14 +8,6 @@
 
 package org.elasticsearch.index.mapper;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-import java.util.function.Function;
-
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
@@ -31,6 +23,14 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.SearchExecutionContext;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Function;
+
 /** A parser for documents, given mappings from a DocumentMapper */
 final class DocumentParser {
 
@@ -39,11 +39,10 @@ final class DocumentParser {
     private final DynamicRuntimeFieldsBuilder dynamicRuntimeFieldsBuilder;
 
     DocumentParser(NamedXContentRegistry xContentRegistry,
-                   Function<DateFormatter, Mapper.TypeParser.ParserContext> dateParserContext,
-                   DynamicRuntimeFieldsBuilder dynamicRuntimeFieldsBuilder) {
+                   Function<DateFormatter, Mapper.TypeParser.ParserContext> dateParserContext) {
         this.xContentRegistry = xContentRegistry;
         this.dateParserContext = dateParserContext;
-        this.dynamicRuntimeFieldsBuilder = dynamicRuntimeFieldsBuilder;
+        this.dynamicRuntimeFieldsBuilder = org.elasticsearch.runtimefields.mapper.DynamicRuntimeFieldsBuilder.INSTANCE;
     }
 
     ParsedDocument parseDocument(SourceToParse source,

server/src/main/java/org/elasticsearch/index/mapper/DynamicRuntimeFieldsBuilder.java

@@ -9,7 +9,6 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.plugins.MapperPlugin;
 
 import static org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
 
@@ -18,7 +17,6 @@
  * Plugins that provide runtime field implementations can also plug in their implementation of this interface
  * to define how leaf fields of each supported type can be dynamically created in dynamic runtime mode.
  *
- * @see MapperPlugin#getDynamicRuntimeFieldsBuilder()
  * @see Dynamic
  */
 public interface DynamicRuntimeFieldsBuilder {