From 6f96d5d0f030addf6ca2c7bce1f5145b1f249527 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 4 Sep 2020 15:39:38 +0200 Subject: [PATCH 1/3] Add support for runtime fields This commit includes the work that has been done on the runtime fields feature branch until now. The high level tasks are listed in #59332. The tasks that have not yet been completed can be worked on after merging the feature branch. We are adding a new x-pack plugin called runtime-fields that plugs in a custom mapper which allows to define runtime fields based on a script. The changes included in this commit that were made outside of the x-pack/plugin/runtime-fields directory are minimal and revolve around 1) making the ScriptService available while parsing index mappings so that the scripts associated to runtime fields can be compiled 2) sharing code to manipulate ranges etc. as it can be reused in runtime fields. Co-authored-by: Nik Everett --- .../metadata/MetadataIndexUpgradeService.java | 6 +- .../org/elasticsearch/index/IndexModule.java | 2 +- .../org/elasticsearch/index/IndexService.java | 6 +- .../elasticsearch/index/IndexSortConfig.java | 9 +- .../index/fielddata/ScriptDocValues.java | 17 +- .../fielddata/plain/LeafDoubleFieldData.java | 4 +- .../fielddata/plain/LeafLongFieldData.java | 4 +- .../index/mapper/DateFieldMapper.java | 122 ++-- .../index/mapper/DocumentMapperParser.java | 10 +- .../index/mapper/IpFieldMapper.java | 25 +- .../elasticsearch/index/mapper/Mapper.java | 15 +- .../index/mapper/MapperService.java | 6 +- .../index/mapper/NumberFieldMapper.java | 190 +++--- .../functionscore/DecayFunctionBuilder.java | 1 + .../java/org/elasticsearch/node/Node.java | 2 +- .../org/elasticsearch/script/ScriptCache.java | 2 +- .../MetadataIndexUpgradeServiceTests.java | 3 +- .../gateway/GatewayMetaStateTests.java | 2 +- .../elasticsearch/index/codec/CodecTests.java | 2 +- .../mapper/ExternalFieldMapperTests.java | 6 +- .../index/mapper/ParametrizedMapperTests.java | 2 +- .../index/mapper/TypeParsersTests.java | 3 +- .../indices/cluster/ClusterStateChanges.java | 10 +- .../snapshots/SnapshotResiliencyTests.java | 3 +- .../elasticsearch/index/MapperTestUtils.java | 2 +- .../index/engine/TranslogHandler.java | 2 +- .../index/mapper/MapperServiceTestCase.java | 13 +- .../index/mapper/MapperTestCase.java | 1 - .../aggregations/AggregatorTestCase.java | 2 +- .../test/AbstractBuilderTestCase.java | 2 +- .../yaml/section/ClientYamlTestSection.java | 7 +- .../yaml/section/ClientYamlTestSuite.java | 2 +- .../test/rest/yaml/section/SetupSection.java | 2 +- x-pack/plugin/runtime-fields/build.gradle | 26 + x-pack/plugin/runtime-fields/qa/build.gradle | 1 + .../runtime-fields/qa/rest/build.gradle | 54 ++ .../rest/CoreTestsWithRuntimeFieldsIT.java | 304 ++++++++++ .../qa/with-security/build.gradle | 26 + .../runtime-fields/qa/with-security/roles.yml | 13 + .../xpack/security/PermissionsIT.java | 228 +++++++ .../AbstractLongScriptFieldScript.java | 58 ++ .../AbstractScriptFieldScript.java | 98 +++ .../BooleanScriptFieldScript.java | 95 +++ .../runtimefields/DateScriptFieldScript.java | 78 +++ .../DoubleScriptFieldScript.java | 92 +++ .../runtimefields/IpScriptFieldScript.java | 114 ++++ .../runtimefields/LongScriptFieldScript.java | 53 ++ .../xpack/runtimefields/RuntimeFields.java | 38 ++ .../RuntimeFieldsPainlessExtension.java | 34 ++ .../StringScriptFieldScript.java | 75 +++ .../fielddata/ScriptBinaryFieldData.java | 73 +++ .../fielddata/ScriptBooleanDocValues.java | 39 ++ .../fielddata/ScriptBooleanFieldData.java | 99 ++++ .../fielddata/ScriptDateFieldData.java | 96 +++ .../fielddata/ScriptDoubleDocValues.java | 43 ++ .../fielddata/ScriptDoubleFieldData.java | 99 ++++ .../fielddata/ScriptIpDocValues.java | 44 ++ .../fielddata/ScriptIpFieldData.java | 88 +++ .../fielddata/ScriptLongDocValues.java | 43 ++ .../fielddata/ScriptLongFieldData.java | 96 +++ .../fielddata/ScriptStringDocValues.java | 37 ++ .../fielddata/ScriptStringFieldData.java | 63 ++ .../mapper/AbstractScriptMappedFieldType.java | 174 ++++++ .../mapper/RuntimeScriptFieldMapper.java | 286 +++++++++ .../mapper/ScriptBooleanMappedFieldType.java | 206 +++++++ .../mapper/ScriptDateMappedFieldType.java | 195 ++++++ .../mapper/ScriptDoubleMappedFieldType.java | 114 ++++ .../mapper/ScriptIpMappedFieldType.java | 181 ++++++ .../mapper/ScriptKeywordMappedFieldType.java | 158 +++++ .../mapper/ScriptLongMappedFieldType.java | 128 ++++ .../AbstractBooleanScriptFieldQuery.java | 80 +++ .../query/AbstractDoubleScriptFieldQuery.java | 77 +++ .../query/AbstractIpScriptFieldQuery.java | 78 +++ .../query/AbstractLongScriptFieldQuery.java | 82 +++ .../query/AbstractScriptFieldQuery.java | 53 ++ ...stractStringScriptFieldAutomatonQuery.java | 48 ++ .../query/AbstractStringScriptFieldQuery.java | 68 +++ .../query/BooleanScriptFieldExistsQuery.java | 31 + .../query/BooleanScriptFieldTermQuery.java | 55 ++ .../query/DoubleScriptFieldExistsQuery.java | 31 + .../query/DoubleScriptFieldRangeQuery.java | 72 +++ .../query/DoubleScriptFieldTermQuery.java | 57 ++ .../query/DoubleScriptFieldTermsQuery.java | 73 +++ .../query/IpScriptFieldExistsQuery.java | 32 + .../query/IpScriptFieldRangeQuery.java | 79 +++ .../query/IpScriptFieldTermQuery.java | 60 ++ .../query/IpScriptFieldTermsQuery.java | 90 +++ .../LongScriptFieldDistanceFeatureQuery.java | 218 +++++++ .../query/LongScriptFieldExistsQuery.java | 39 ++ .../query/LongScriptFieldRangeQuery.java | 75 +++ .../query/LongScriptFieldTermQuery.java | 65 ++ .../query/LongScriptFieldTermsQuery.java | 67 +++ .../query/StringScriptFieldExistsQuery.java | 33 ++ .../query/StringScriptFieldFuzzyQuery.java | 68 +++ .../query/StringScriptFieldPrefixQuery.java | 69 +++ .../query/StringScriptFieldRangeQuery.java | 115 ++++ .../query/StringScriptFieldRegexpQuery.java | 68 +++ .../query/StringScriptFieldTermQuery.java | 65 ++ .../query/StringScriptFieldTermsQuery.java | 75 +++ .../query/StringScriptFieldWildcardQuery.java | 60 ++ ...asticsearch.painless.spi.PainlessExtension | 1 + .../xpack/runtimefields/boolean_whitelist.txt | 21 + .../xpack/runtimefields/date_whitelist.txt | 25 + .../xpack/runtimefields/double_whitelist.txt | 19 + .../xpack/runtimefields/ip_whitelist.txt | 18 + .../xpack/runtimefields/long_whitelist.txt | 18 + .../xpack/runtimefields/string_whitelist.txt | 18 + .../BooleanScriptFieldScriptTests.java | 32 + .../DateScriptFieldScriptTests.java | 33 ++ .../DoubleScriptFieldScriptTests.java | 32 + .../IpScriptFieldScriptTests.java | 28 + .../LongScriptFieldScriptTests.java | 28 + .../ScriptFieldScriptTestCase.java | 30 + .../StringScriptFieldScriptTests.java | 32 + .../xpack/runtimefields/TestScriptEngine.java | 51 ++ ...tNonTextScriptMappedFieldTypeTestCase.java | 66 +++ ...AbstractScriptMappedFieldTypeTestCase.java | 133 +++++ .../mapper/RuntimeScriptFieldMapperTests.java | 399 +++++++++++++ .../ScriptBooleanMappedFieldTypeTests.java | 503 ++++++++++++++++ .../ScriptDateMappedFieldTypeTests.java | 558 ++++++++++++++++++ .../ScriptDoubleMappedFieldTypeTests.java | 340 +++++++++++ .../mapper/ScriptIpMappedFieldTypeTests.java | 386 ++++++++++++ .../ScriptKeywordMappedFieldTypeTests.java | 424 +++++++++++++ .../ScriptLongMappedFieldTypeTests.java | 380 ++++++++++++ ...stractBooleanScriptFieldQueryTestCase.java | 49 ++ ...bstractDoubleScriptFieldQueryTestCase.java | 22 + .../AbstractIpScriptFieldQueryTestCase.java | 29 + .../AbstractLongScriptFieldQueryTestCase.java | 23 + .../AbstractScriptFieldQueryTestCase.java | 72 +++ ...bstractStringScriptFieldQueryTestCase.java | 50 ++ .../BooleanScriptFieldExistsQueryTests.java | 42 ++ .../BooleanScriptFieldTermQueryTests.java | 66 +++ .../DoubleScriptFieldExistsQueryTests.java | 41 ++ .../DoubleScriptFieldRangeQueryTests.java | 74 +++ .../DoubleScriptFieldTermQueryTests.java | 59 ++ .../DoubleScriptFieldTermsQueryTests.java | 107 ++++ .../query/IpScriptFieldExistsQueryTests.java | 43 ++ .../query/IpScriptFieldRangeQueryTests.java | 125 ++++ .../query/IpScriptFieldTermQueryTests.java | 64 ++ .../query/IpScriptFieldTermsQueryTests.java | 141 +++++ ...gScriptFieldDistanceFeatureQueryTests.java | 146 +++++ .../LongScriptFieldExistsQueryTests.java | 41 ++ .../query/LongScriptFieldRangeQueryTests.java | 81 +++ .../query/LongScriptFieldTermQueryTests.java | 59 ++ .../query/LongScriptFieldTermsQueryTests.java | 72 +++ .../StringScriptFieldExistsQueryTests.java | 62 ++ .../StringScriptFieldFuzzyQueryTests.java | 116 ++++ .../StringScriptFieldPrefixQueryTests.java | 73 +++ .../StringScriptFieldRangeQueryTests.java | 118 ++++ .../StringScriptFieldRegexpQueryTests.java | 104 ++++ .../StringScriptFieldTermQueryTests.java | 84 +++ .../StringScriptFieldTermsQueryTests.java | 95 +++ .../StringScriptFieldWildcardQueryTests.java | 71 +++ .../test/runtime_fields/10_keyword.yml | 278 +++++++++ .../test/runtime_fields/20_long.yml | 242 ++++++++ .../test/runtime_fields/30_double.yml | 177 ++++++ .../test/runtime_fields/40_date.yml | 162 +++++ .../test/runtime_fields/50_ip.yml | 144 +++++ .../test/runtime_fields/60_boolean.yml | 124 ++++ .../runtime_fields/80_multiple_indices.yml | 240 ++++++++ .../test/runtime_fields/90_loops.yml | 202 +++++++ 161 files changed, 13311 insertions(+), 179 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/build.gradle create mode 100644 x-pack/plugin/runtime-fields/qa/build.gradle create mode 100644 x-pack/plugin/runtime-fields/qa/rest/build.gradle create mode 100644 x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java create mode 100644 x-pack/plugin/runtime-fields/qa/with-security/build.gradle create mode 100644 x-pack/plugin/runtime-fields/qa/with-security/roles.yml create mode 100644 x-pack/plugin/runtime-fields/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanDocValues.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanFieldData.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDateFieldData.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleDocValues.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleFieldData.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpDocValues.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpFieldData.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongFieldData.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringDocValues.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringFieldData.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptBooleanMappedFieldType.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldType.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptIpMappedFieldType.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldType.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldType.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractBooleanScriptFieldQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractDoubleScriptFieldQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractIpScriptFieldQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractScriptFieldQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractStringScriptFieldAutomatonQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractStringScriptFieldQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/BooleanScriptFieldExistsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/BooleanScriptFieldTermQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldExistsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldRangeQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldExistsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldRangeQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldDistanceFeatureQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldExistsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldRangeQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldExistsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldFuzzyQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldPrefixQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldRangeQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldRegexpQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldTermQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldTermsQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldWildcardQuery.java create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/boolean_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/date_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/ip_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/TestScriptEngine.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractNonTextScriptMappedFieldTypeTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptBooleanMappedFieldTypeTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldTypeTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldTypeTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptIpMappedFieldTypeTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractBooleanScriptFieldQueryTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractDoubleScriptFieldQueryTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractIpScriptFieldQueryTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQueryTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractScriptFieldQueryTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractStringScriptFieldQueryTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/BooleanScriptFieldExistsQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/BooleanScriptFieldTermQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldExistsQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldRangeQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldTermsQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldExistsQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldRangeQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermsQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldDistanceFeatureQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldExistsQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldRangeQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermsQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldExistsQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldFuzzyQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldPrefixQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldRangeQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldRegexpQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldTermQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldTermsQueryTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/StringScriptFieldWildcardQueryTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/20_long.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/60_boolean.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/90_loops.yml diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeService.java index a136c2cbc0c6c..cd2e00d869525 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeService.java @@ -61,14 +61,16 @@ public class MetadataIndexUpgradeService { private final MapperRegistry mapperRegistry; private final IndexScopedSettings indexScopedSettings; private final SystemIndices systemIndices; + private final ScriptService scriptService; public MetadataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry, - IndexScopedSettings indexScopedSettings, SystemIndices systemIndices) { + IndexScopedSettings indexScopedSettings, SystemIndices systemIndices, ScriptService scriptService) { this.settings = settings; this.xContentRegistry = xContentRegistry; this.mapperRegistry = mapperRegistry; this.indexScopedSettings = indexScopedSettings; this.systemIndices = systemIndices; + this.scriptService = scriptService; } /** @@ -188,7 +190,7 @@ public Set> entrySet() { try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(analyzerMap, analyzerMap, analyzerMap)) { MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService, - mapperRegistry, () -> null, () -> false); + mapperRegistry, () -> null, () -> false, scriptService); mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY); } } catch (Exception ex) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index fea123aa820d5..030ec8745d6a1 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -505,7 +505,7 @@ public MapperService newIndexMapperService(NamedXContentRegistry xContentRegistr ScriptService scriptService) throws IOException { return new MapperService(indexSettings, analysisRegistry.build(indexSettings), xContentRegistry, new SimilarityService(indexSettings, scriptService, similarities), mapperRegistry, - () -> { throw new UnsupportedOperationException("no index query shard context available"); }, () -> false); + () -> { throw new UnsupportedOperationException("no index query shard context available"); }, () -> false, scriptService); } /** diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 15ddf74eae38e..65889078cb3b4 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -193,16 +193,14 @@ public IndexService( assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, // we parse all percolator queries as they would be parsed on shard 0 - () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled); + () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled, scriptService); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. // The sort order is validated right after the merge of the mapping later in the process. this.indexSortSupplier = () -> indexSettings.getIndexSortConfig().buildIndexSort( mapperService::fieldType, - fieldType -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), () -> { - throw new UnsupportedOperationException("search lookup not available for index sorting"); - }) + (fieldType, searchLookup) -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), searchLookup) ); } else { this.indexSortSupplier = () -> null; diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index a0f65f0ee69a0..392e9c2b5ec37 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -29,12 +29,15 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.SortOrder; import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.function.BiFunction; import java.util.function.Function; +import java.util.function.Supplier; /** * Holds all the information that is used to build the sort order of an index. @@ -181,7 +184,7 @@ public boolean hasPrimarySortOnField(String field) { * or returns null if this index has no sort. */ public Sort buildIndexSort(Function fieldTypeLookup, - Function> fieldDataLookup) { + BiFunction, IndexFieldData> fieldDataLookup) { if (hasIndexSort() == false) { return null; } @@ -200,7 +203,9 @@ public Sort buildIndexSort(Function fieldTypeLookup, } IndexFieldData fieldData; try { - fieldData = fieldDataLookup.apply(ft); + fieldData = fieldDataLookup.apply(ft, () -> { + throw new UnsupportedOperationException("index sorting not supported on runtime field [" + ft.name() + "]"); + }); } catch (Exception e) { throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]", e); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 080ef97d5f86d..1dfb10098b8b5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -481,25 +481,30 @@ protected void resize(int newSize) { public int size() { return count; } - } - public static final class Strings extends BinaryScriptDocValues { - + public static class Strings extends BinaryScriptDocValues { public Strings(SortedBinaryDocValues in) { super(in); } @Override - public String get(int index) { + public final String get(int index) { if (count == 0) { throw new IllegalStateException("A document doesn't have a value for a field! " + "Use doc[].size()==0 to check if a document is missing a field!"); } - return values[index].get().utf8ToString(); + return bytesToString(values[index].get()); + } + + /** + * Convert the stored bytes to a String. + */ + protected String bytesToString(BytesRef bytes) { + return bytes.utf8ToString(); } - public String getValue() { + public final String getValue() { return get(0); } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java index 46cd3a5ccaef2..3ab7847f641d7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java @@ -34,11 +34,11 @@ /** * Specialization of {@link LeafNumericFieldData} for floating-point numerics. */ -abstract class LeafDoubleFieldData implements LeafNumericFieldData { +public abstract class LeafDoubleFieldData implements LeafNumericFieldData { private final long ramBytesUsed; - LeafDoubleFieldData(long ramBytesUsed) { + protected LeafDoubleFieldData(long ramBytesUsed) { this.ramBytesUsed = ramBytesUsed; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java index cf46d51340227..38dede8537b94 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java @@ -29,7 +29,7 @@ /** * Specialization of {@link LeafNumericFieldData} for integers. */ -abstract class LeafLongFieldData implements LeafNumericFieldData { +public abstract class LeafLongFieldData implements LeafNumericFieldData { private final long ramBytesUsed; /** @@ -37,7 +37,7 @@ abstract class LeafLongFieldData implements LeafNumericFieldData { */ private final NumericType numericType; - LeafLongFieldData(long ramBytesUsed, NumericType numericType) { + protected LeafLongFieldData(long ramBytesUsed, NumericType numericType) { this.ramBytesUsed = ramBytesUsed; this.numericType = numericType; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 8104759b0ba0a..d2046c32f6113 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -31,12 +31,12 @@ import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.common.time.DateMathParser; @@ -63,6 +63,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -343,60 +344,83 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser; + return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { + Query query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + + if (context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + } + return query; + }); + } + + public static Query dateRangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + @Nullable ZoneId timeZone, + DateMathParser parser, + QueryShardContext context, + Resolution resolution, + BiFunction builder + ) { + return handleNow(context, nowSupplier -> { + long l, u; + if (lowerTerm == null) { + l = Long.MIN_VALUE; + } else { + l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier, resolution); + if (includeLower == false) { + ++l; + } + } + if (upperTerm == null) { + u = Long.MAX_VALUE; + } else { + u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier, resolution); + if (includeUpper == false) { + --u; + } + } + return builder.apply(l, u); + }); + } + + /** + * Handle {@code now} in queries. + * @param context context from which to read the current time + * @param builder build the query + * @return the result of the builder, wrapped in {@link DateRangeIncludingNowQuery} if {@code now} was used. + */ + public static Query handleNow(QueryShardContext context, Function builder) { boolean[] nowUsed = new boolean[1]; LongSupplier nowSupplier = () -> { nowUsed[0] = true; return context.nowInMillis(); }; - long l, u; - if (lowerTerm == null) { - l = Long.MIN_VALUE; - } else { - l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier); - if (includeLower == false) { - ++l; - } - } - if (upperTerm == null) { - u = Long.MAX_VALUE; - } else { - u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier); - if (includeUpper == false) { - --u; - } - } - - Query query = LongPoint.newRangeQuery(name(), l, u); - if (hasDocValues()) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); - query = new IndexOrDocValuesQuery(query, dvQuery); - - if (context.indexSortedOnField(name())) { - query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); - } - } - - if (nowUsed[0]) { - query = new DateRangeIncludingNowQuery(query); - } - return query; + Query query = builder.apply(nowSupplier); + return nowUsed[0] ? new DateRangeIncludingNowQuery(query) : query; } - public long parseToLong(Object value, boolean roundUp, - @Nullable ZoneId zone, @Nullable DateMathParser forcedDateParser, LongSupplier now) { - DateMathParser dateParser = dateMathParser(); - if (forcedDateParser != null) { - dateParser = forcedDateParser; - } + public long parseToLong(Object value, boolean roundUp, @Nullable ZoneId zone, DateMathParser dateParser, LongSupplier now) { + dateParser = dateParser == null ? dateMathParser() : dateParser; + return parseToLong(value, roundUp, zone, dateParser, now, resolution); + } - String strValue; - if (value instanceof BytesRef) { - strValue = ((BytesRef) value).utf8ToString(); - } else { - strValue = value.toString(); - } - Instant instant = dateParser.parse(strValue, now, roundUp, zone); - return resolution.convert(instant); + public static long parseToLong( + Object value, + boolean roundUp, + @Nullable ZoneId zone, + DateMathParser dateParser, + LongSupplier now, + Resolution resolution + ) { + return resolution.convert(dateParser.parse(BytesRefs.toString(value), now, roundUp, zone)); } @Override @@ -416,7 +440,7 @@ public Relation isFieldWithinQuery(IndexReader reader, long fromInclusive = Long.MIN_VALUE; if (from != null) { - fromInclusive = parseToLong(from, !includeLower, timeZone, dateParser, context::nowInMillis); + fromInclusive = parseToLong(from, !includeLower, timeZone, dateParser, context::nowInMillis, resolution); if (includeLower == false) { if (fromInclusive == Long.MAX_VALUE) { return Relation.DISJOINT; @@ -427,7 +451,7 @@ public Relation isFieldWithinQuery(IndexReader reader, long toInclusive = Long.MAX_VALUE; if (to != null) { - toInclusive = parseToLong(to, includeUpper, timeZone, dateParser, context::nowInMillis); + toInclusive = parseToLong(to, includeUpper, timeZone, dateParser, context::nowInMillis, resolution); if (includeUpper == false) { if (toInclusive == Long.MIN_VALUE) { return Relation.DISJOINT; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 9aa50d2282854..3592f308a0743 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.script.ScriptService; import java.util.HashMap; import java.util.Iterator; @@ -54,13 +55,16 @@ public class DocumentMapperParser { private final Map typeParsers; private final Map rootTypeParsers; + private final ScriptService scriptService; public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperService, NamedXContentRegistry xContentRegistry, - SimilarityService similarityService, MapperRegistry mapperRegistry, Supplier queryShardContextSupplier) { + SimilarityService similarityService, MapperRegistry mapperRegistry, + Supplier queryShardContextSupplier, ScriptService scriptService) { this.mapperService = mapperService; this.xContentRegistry = xContentRegistry; this.similarityService = similarityService; this.queryShardContextSupplier = queryShardContextSupplier; + this.scriptService = scriptService; this.typeParsers = mapperRegistry.getMapperParsers(); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(indexVersionCreated); @@ -68,12 +72,12 @@ public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperSer public Mapper.TypeParser.ParserContext parserContext() { return new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperService, - typeParsers::get, indexVersionCreated, queryShardContextSupplier, null); + typeParsers::get, indexVersionCreated, queryShardContextSupplier, null, scriptService); } public Mapper.TypeParser.ParserContext parserContext(DateFormatter dateFormatter) { return new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperService, - typeParsers::get, indexVersionCreated, queryShardContextSupplier, dateFormatter); + typeParsers::get, indexVersionCreated, queryShardContextSupplier, dateFormatter, scriptService); } public DocumentMapper parse(@Nullable String type, CompressedXContent source) throws MapperParsingException { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 1814f778fe0e1..78b35d2bb42ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -49,6 +49,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.BiFunction; import java.util.function.Supplier; /** A {@link FieldMapper} for ip addresses. */ @@ -128,7 +129,7 @@ public String typeName() { return CONTENT_TYPE; } - private InetAddress parse(Object value) { + private static InetAddress parse(Object value) { if (value instanceof InetAddress) { return (InetAddress) value; } else { @@ -194,6 +195,26 @@ public Query termsQuery(List values, QueryShardContext context) { @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { failIfNotIndexed(); + return rangeQuery( + lowerTerm, + upperTerm, + includeLower, + includeUpper, + (lower, upper) -> InetAddressPoint.newRangeQuery(name(), lower, upper) + ); + } + + /** + * Processes query bounds into {@code long}s and delegates the + * provided {@code builder} to build a range query. + */ + public static Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + BiFunction builder + ) { InetAddress lower; if (lowerTerm == null) { lower = InetAddressPoint.MIN_VALUE; @@ -220,7 +241,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } } - return InetAddressPoint.newRangeQuery(name(), lower, upper); + return builder.apply(lower, upper); } public static final class IpScriptDocValues extends ScriptDocValues { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 666a02a71f6f3..007b267a30a88 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; +import org.elasticsearch.script.ScriptService; import java.util.Map; import java.util.Objects; @@ -91,16 +92,19 @@ class ParserContext { private final DateFormatter dateFormatter; + private final ScriptService scriptService; + public ParserContext(Function similarityLookupService, MapperService mapperService, Function typeParsers, Version indexVersionCreated, Supplier queryShardContextSupplier, - DateFormatter dateFormatter) { + DateFormatter dateFormatter, ScriptService scriptService) { this.similarityLookupService = similarityLookupService; this.mapperService = mapperService; this.typeParsers = typeParsers; this.indexVersionCreated = indexVersionCreated; this.queryShardContextSupplier = queryShardContextSupplier; this.dateFormatter = dateFormatter; + this.scriptService = scriptService; } public IndexAnalyzers getIndexAnalyzers() { @@ -146,6 +150,13 @@ public DateFormatter getDateFormatter() { protected Function similarityLookupService() { return similarityLookupService; } + /** + * The {@linkplain ScriptService} to compile scripts needed by the {@linkplain Mapper}. + */ + public ScriptService scriptService() { + return scriptService; + } + public ParserContext createMultiFieldContext(ParserContext in) { return new MultiFieldParserContext(in); } @@ -153,7 +164,7 @@ public ParserContext createMultiFieldContext(ParserContext in) { static class MultiFieldParserContext extends ParserContext { MultiFieldParserContext(ParserContext in) { super(in.similarityLookupService(), in.mapperService(), in.typeParsers(), - in.indexVersionCreated(), in.queryShardContextSupplier(), in.getDateFormatter()); + in.indexVersionCreated(), in.queryShardContextSupplier(), in.getDateFormatter(), in.scriptService()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 7cd197affd609..524a1776a3fb6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -55,6 +55,7 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.InvalidTypeNameException; import org.elasticsearch.indices.mapper.MapperRegistry; +import org.elasticsearch.script.ScriptService; import java.io.Closeable; import java.io.IOException; @@ -151,12 +152,13 @@ public enum MergeReason { public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, NamedXContentRegistry xContentRegistry, SimilarityService similarityService, MapperRegistry mapperRegistry, - Supplier queryShardContextSupplier, BooleanSupplier idFieldDataEnabled) { + Supplier queryShardContextSupplier, BooleanSupplier idFieldDataEnabled, + ScriptService scriptService) { super(indexSettings); this.indexVersionCreated = indexSettings.getIndexVersionCreated(); this.indexAnalyzers = indexAnalyzers; this.documentParser = new DocumentMapperParser(indexSettings, this, xContentRegistry, similarityService, mapperRegistry, - queryShardContextSupplier); + queryShardContextSupplier, scriptService); this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), MappedFieldType::indexAnalyzer); this.searchAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.getTextSearchInfo().getSearchAnalyzer()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 58780caa5c6dd..4b1e599432ffe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.exc.InputCoercionException; + import org.apache.lucene.document.DoublePoint; import org.apache.lucene.document.Field; import org.apache.lucene.document.FloatPoint; @@ -63,6 +64,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -367,28 +369,16 @@ public Query termsQuery(String field, List values) { public Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, boolean hasDocValues, QueryShardContext context) { - double l = Double.NEGATIVE_INFINITY; - double u = Double.POSITIVE_INFINITY; - if (lowerTerm != null) { - l = parse(lowerTerm, false); - if (includeLower == false) { - l = DoublePoint.nextUp(l); - } - } - if (upperTerm != null) { - u = parse(upperTerm, false); - if (includeUpper == false) { - u = DoublePoint.nextDown(u); + return doubleRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> { + Query query = DoublePoint.newRangeQuery(field, l, u); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, + NumericUtils.doubleToSortableLong(l), + NumericUtils.doubleToSortableLong(u)); + query = new IndexOrDocValuesQuery(query, dvQuery); } - } - Query query = DoublePoint.newRangeQuery(field, l, u); - if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, - NumericUtils.doubleToSortableLong(l), - NumericUtils.doubleToSortableLong(u)); - query = new IndexOrDocValuesQuery(query, dvQuery); - } - return query; + return query; + }); } @Override @@ -654,23 +644,7 @@ public List createFields(String name, Number value, LONG("long", NumericType.LONG) { @Override public Long parse(Object value, boolean coerce) { - if (value instanceof Long) { - return (Long)value; - } - - double doubleValue = objectToDouble(value); - // this check does not guarantee that value is inside MIN_VALUE/MAX_VALUE because values up to 9223372036854776832 will - // be equal to Long.MAX_VALUE after conversion to double. More checks ahead. - if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) { - throw new IllegalArgumentException("Value [" + value + "] is out of range for a long"); - } - if (!coerce && doubleValue % 1 != 0) { - throw new IllegalArgumentException("Value [" + value + "] has a decimal part"); - } - - // longs need special handling so we don't lose precision while parsing - String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); - return Numbers.toLong(stringValue, coerce); + return objectToLong(value, coerce); } @Override @@ -717,44 +691,17 @@ public Query termsQuery(String field, List values) { public Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, boolean hasDocValues, QueryShardContext context) { - long l = Long.MIN_VALUE; - long u = Long.MAX_VALUE; - if (lowerTerm != null) { - l = parse(lowerTerm, true); - // if the lower bound is decimal: - // - if the bound is positive then we increment it: - // if lowerTerm=1.5 then the (inclusive) bound becomes 2 - // - if the bound is negative then we leave it as is: - // if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue - boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm); - if ((lowerTermHasDecimalPart == false && includeLower == false) || - (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) { - if (l == Long.MAX_VALUE) { - return new MatchNoDocsQuery(); + return longRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (l, u) -> { + Query query = LongPoint.newRangeQuery(field, l, u); + if (hasDocValues) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + if (context.indexSortedOnField(field)) { + query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query); } - ++l; } - } - if (upperTerm != null) { - u = parse(upperTerm, true); - boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm); - if ((upperTermHasDecimalPart == false && includeUpper == false) || - (upperTermHasDecimalPart && signum(upperTerm) < 0)) { - if (u == Long.MIN_VALUE) { - return new MatchNoDocsQuery(); - } - --u; - } - } - Query query = LongPoint.newRangeQuery(field, l, u); - if (hasDocValues) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, l, u); - query = new IndexOrDocValuesQuery(query, dvQuery); - if (context.indexSortedOnField(field)) { - query = new IndexSortSortedNumericDocValuesRangeQuery(field, l, u, query); - } - } - return query; + return query; + }); } @Override @@ -812,7 +759,7 @@ Number valueForSearch(Number value) { /** * Returns true if the object is a number and has a decimal part */ - boolean hasDecimalPart(Object number) { + public static boolean hasDecimalPart(Object number) { if (number instanceof Number) { double doubleValue = ((Number) number).doubleValue(); return doubleValue % 1 != 0; @@ -829,7 +776,7 @@ boolean hasDecimalPart(Object number) { /** * Returns -1, 0, or 1 if the value is lower than, equal to, or greater than 0 */ - double signum(Object value) { + static double signum(Object value) { if (value instanceof Number) { double doubleValue = ((Number) value).doubleValue(); return Math.signum(doubleValue); @@ -843,7 +790,7 @@ boolean hasDecimalPart(Object number) { /** * Converts an Object to a double by checking it against known types first */ - private static double objectToDouble(Object value) { + public static double objectToDouble(Object value) { double doubleValue; if (value instanceof Number) { @@ -856,6 +803,95 @@ private static double objectToDouble(Object value) { return doubleValue; } + + /** + * Converts and Object to a {@code long} by checking it against known + * types and checking its range. + */ + public static long objectToLong(Object value, boolean coerce) { + if (value instanceof Long) { + return (Long)value; + } + + double doubleValue = objectToDouble(value); + // this check does not guarantee that value is inside MIN_VALUE/MAX_VALUE because values up to 9223372036854776832 will + // be equal to Long.MAX_VALUE after conversion to double. More checks ahead. + if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) { + throw new IllegalArgumentException("Value [" + value + "] is out of range for a long"); + } + if (!coerce && doubleValue % 1 != 0) { + throw new IllegalArgumentException("Value [" + value + "] has a decimal part"); + } + + // longs need special handling so we don't lose precision while parsing + String stringValue = (value instanceof BytesRef) ? ((BytesRef) value).utf8ToString() : value.toString(); + return Numbers.toLong(stringValue, coerce); + } + + public static Query doubleRangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + BiFunction builder + ) { + double l = Double.NEGATIVE_INFINITY; + double u = Double.POSITIVE_INFINITY; + if (lowerTerm != null) { + l = objectToDouble(lowerTerm); + if (includeLower == false) { + l = DoublePoint.nextUp(l); + } + } + if (upperTerm != null) { + u = objectToDouble(upperTerm); + if (includeUpper == false) { + u = DoublePoint.nextDown(u); + } + } + return builder.apply(l, u); + } + + /** + * Processes query bounds into {@code long}s and delegates the + * provided {@code builder} to build a range query. + */ + public static Query longRangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + BiFunction builder + ) { + long l = Long.MIN_VALUE; + long u = Long.MAX_VALUE; + if (lowerTerm != null) { + l = objectToLong(lowerTerm, true); + // if the lower bound is decimal: + // - if the bound is positive then we increment it: + // if lowerTerm=1.5 then the (inclusive) bound becomes 2 + // - if the bound is negative then we leave it as is: + // if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue + boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm); + if ((lowerTermHasDecimalPart == false && includeLower == false) || (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) { + if (l == Long.MAX_VALUE) { + return new MatchNoDocsQuery(); + } + ++l; + } + } + if (upperTerm != null) { + u = objectToLong(upperTerm, true); + boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm); + if ((upperTermHasDecimalPart == false && includeUpper == false) || (upperTermHasDecimalPart && signum(upperTerm) < 0)) { + if (u == Long.MIN_VALUE) { + return new MatchNoDocsQuery(); + } + --u; + } + } + return builder.apply(l, u); + } } public static final class NumberFieldType extends SimpleMappedFieldType { diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index 0b4177490b17b..47e08b1286ea3 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -210,6 +210,7 @@ private AbstractDistanceScoreFunction parseVariable(String fieldName, XContentPa // dates and time and geo need special handling parser.nextToken(); + // TODO these ain't gonna work with runtime fields if (fieldType instanceof DateFieldMapper.DateFieldType) { return parseDateVariable(parser, context, fieldType, mode); } else if (fieldType instanceof GeoPointFieldType) { diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index df9da16e17e83..7907aa04c6d4d 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -550,7 +550,7 @@ protected Node(final Environment initialEnvironment, .collect(Collectors.toList()); final MetadataUpgrader metadataUpgrader = new MetadataUpgrader(indexTemplateMetadataUpgraders); final MetadataIndexUpgradeService metadataIndexUpgradeService = new MetadataIndexUpgradeService(settings, xContentRegistry, - indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), systemIndices); + indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), systemIndices, scriptService); clusterService.addListener(new SystemIndexMetadataUpgradeService(systemIndices, clusterService)); new TemplateUpgradeService(client, clusterService, threadPool, indexTemplateMetadataUpgraders); final Transport transport = networkModule.getTransportSupplier().get(); diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCache.java b/server/src/main/java/org/elasticsearch/script/ScriptCache.java index 046d394ffa2c5..d3b3f11c5180a 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptCache.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptCache.java @@ -42,7 +42,7 @@ public class ScriptCache { private static final Logger logger = LogManager.getLogger(ScriptService.class); - static final CompilationRate UNLIMITED_COMPILATION_RATE = new CompilationRate(0, TimeValue.ZERO); + public static final CompilationRate UNLIMITED_COMPILATION_RATE = new CompilationRate(0, TimeValue.ZERO); private final Cache cache; private final ScriptMetrics scriptMetrics; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java index 8b71de3db12fd..48aac8a713d33 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java @@ -156,7 +156,8 @@ private MetadataIndexUpgradeService getMetadataIndexUpgradeService() { new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new SystemIndices(Collections.singletonMap("system-plugin", - Collections.singletonList(new SystemIndexDescriptor(".system", "a system index")))) + Collections.singletonList(new SystemIndexDescriptor(".system", "a system index")))), + null ); } diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index f260cb767c925..6ceee1e13c691 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -159,7 +159,7 @@ private static class MockMetadataIndexUpgradeService extends MetadataIndexUpgrad private final boolean upgrade; MockMetadataIndexUpgradeService(boolean upgrade) { - super(Settings.EMPTY, null, null, null, null); + super(Settings.EMPTY, null, null, null, null, null); this.upgrade = upgrade; } diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index e1eb795e5be1c..aa86888cdefe9 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -94,7 +94,7 @@ private CodecService createCodecService() throws IOException { IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers; MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry, - () -> null, () -> false); + () -> null, () -> false, null); return new CodecService(service, LogManager.getLogger("test")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java index f1dd3c9596d90..fc37785cb2586 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java @@ -75,7 +75,7 @@ public void testExternalValues() throws Exception { return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null); }; DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext); + indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext, null); DocumentMapper documentMapper = parser.parse("type", new CompressedXContent( Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject(ExternalMetadataMapper.CONTENT_TYPE) @@ -122,7 +122,7 @@ public void testExternalValuesWithMultifield() throws Exception { return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null); }; DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext); + indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext, null); DocumentMapper documentMapper = parser.parse("type", new CompressedXContent( Strings @@ -190,7 +190,7 @@ public void testExternalValuesWithMultifieldTwoLevels() throws Exception { return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null); }; DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), - indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext); + indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext, null); DocumentMapper documentMapper = parser.parse("type", new CompressedXContent( Strings diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 032422ec34908..2fbf69538eb32 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -217,7 +217,7 @@ private static TestMapper fromMapping(String mapping, Version version) { return BinaryFieldMapper.PARSER; } return null; - }, version, () -> null, null); + }, version, () -> null, null, null); return (TestMapper) new TypeParser() .parse("field", XContentHelper.convertToMap(JsonXContent.jsonXContent, mapping, true), pc) .build(new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0))); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index 79839b3f44b0d..82933960293e9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -191,8 +191,7 @@ public void testMultiFieldWithinMultiField() throws IOException { MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); Mapper.TypeParser.ParserContext olderContext = new Mapper.TypeParser.ParserContext( - null, mapperService, type -> typeParser, Version.CURRENT, null, null); - + null, mapperService, type -> typeParser, Version.CURRENT, null, null, null); TypeParsers.parseField(builder, "some-field", fieldNode, olderContext); assertWarnings("At least one multi-field, [sub-field], " + "was encountered that itself contains a multi-field. Defining multi-fields within a multi-field is deprecated " + diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index e1db4ec339c5f..f24d2fbfcbc4f 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -191,8 +191,14 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> DiscoveryNode.createLocal(SETTINGS, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), clusterSettings, Collections.emptySet()); - MetadataIndexUpgradeService metadataIndexUpgradeService = - new MetadataIndexUpgradeService(SETTINGS, xContentRegistry, null, null, null) { + MetadataIndexUpgradeService metadataIndexUpgradeService = new MetadataIndexUpgradeService( + SETTINGS, + xContentRegistry, + null, + null, + null, + null + ) { // metadata upgrader should do nothing @Override public IndexMetadata upgradeIndexMetadata(IndexMetadata indexMetadata, Version minimumIndexCompatibilityVersion) { diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 0a4c5685bda13..aa71d6a6a2377 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -1595,7 +1595,8 @@ clusterService, indicesService, threadPool, shardStateAction, mappingUpdatedActi settings, namedXContentRegistry, mapperRegistry, indexScopedSettings, - new SystemIndices(emptyMap())), + new SystemIndices(emptyMap()), + null), clusterSettings, shardLimitValidator ); diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index f8057d7bfa8a2..e9e9b3b043d40 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -73,7 +73,7 @@ public static MapperService newMapperService(NamedXContentRegistry xContentRegis xContentRegistry, similarityService, mapperRegistry, - () -> null, () -> false); + () -> null, () -> false, null); } public static void assertConflicts(String mapping1, diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index 7a15d7dcf2d21..c9ef3b010ab49 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -68,7 +68,7 @@ public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings ind SimilarityService similarityService = new SimilarityService(indexSettings, null, emptyMap()); MapperRegistry mapperRegistry = new IndicesModule(emptyList()).getMapperRegistry(); mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, - () -> null, () -> false); + () -> null, () -> false, null); } private DocumentMapperForType docMapper(String type) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index d9d12c391d2d8..812ed29944519 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -45,6 +45,8 @@ import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; @@ -105,11 +107,15 @@ protected final MapperService createMapperService(Settings settings, XContentBui .numberOfReplicas(0) .numberOfShards(1) .build(); - IndexSettings indexSettings = new IndexSettings(meta, settings); + IndexSettings indexSettings = new IndexSettings(meta, Settings.EMPTY); MapperRegistry mapperRegistry = new IndicesModule( getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList()) ).getMapperRegistry(); - ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap()); + ScriptModule scriptModule = new ScriptModule( + Settings.EMPTY, + getPlugins().stream().filter(p -> p instanceof ScriptPlugin).map(p -> (ScriptPlugin) p).collect(toList()) + ); + ScriptService scriptService = new ScriptService(settings, scriptModule.engines, scriptModule.contexts); SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, emptyMap()); MapperService mapperService = new MapperService( indexSettings, @@ -118,7 +124,8 @@ protected final MapperService createMapperService(Settings settings, XContentBui similarityService, mapperRegistry, () -> { throw new UnsupportedOperationException(); }, - () -> true + () -> true, + scriptService ); merge(mapperService, mapping); return mapperService; diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index d96a5137b69a6..9190185daa461 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -39,7 +39,6 @@ * Base class for testing {@link Mapper}s. */ public abstract class MapperTestCase extends MapperServiceTestCase { - protected abstract void minimalMapping(XContentBuilder b) throws IOException; public final void testEmptyName() { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 680542b088398..41fc7f39cdc5e 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -834,7 +834,7 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd private class MockParserContext extends Mapper.TypeParser.ParserContext { MockParserContext() { - super(null, null, null, Version.CURRENT, null, null); + super(null, null, null, Version.CURRENT, null, null, null); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index b038b75c16905..889b7a4962b46 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -360,7 +360,7 @@ private static class ServiceHolder implements Closeable { similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap()); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); mapperService = new MapperService(idxSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, - () -> createShardContext(null), () -> false); + () -> createShardContext(null), () -> false, null); IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() { }); indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache, diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java index 48e7fc031139b..0a1b03279f7bb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java @@ -60,7 +60,12 @@ public static ClientYamlTestSection parse(XContentParser parser) throws IOExcept private final SkipSection skipSection; private final List executableSections; - ClientYamlTestSection(XContentLocation location, String name, SkipSection skipSection, List executableSections) { + public ClientYamlTestSection( + XContentLocation location, + String name, + SkipSection skipSection, + List executableSections + ) { this.location = location; this.name = name; this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null"); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java index fbccce730523a..57070ef8d2c8f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java @@ -111,7 +111,7 @@ public static ClientYamlTestSuite parse(String api, String suiteName, XContentPa private final TeardownSection teardownSection; private final List testSections; - ClientYamlTestSuite(String api, String name, SetupSection setupSection, TeardownSection teardownSection, + public ClientYamlTestSuite(String api, String name, SetupSection setupSection, TeardownSection teardownSection, List testSections) { this.api = api; this.name = name; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java index 38a034d47e769..dd718ea684b05 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java @@ -70,7 +70,7 @@ public static SetupSection parse(XContentParser parser) throws IOException { private final SkipSection skipSection; private final List executableSections; - SetupSection(SkipSection skipSection, List executableSections) { + public SetupSection(SkipSection skipSection, List executableSections) { this.skipSection = Objects.requireNonNull(skipSection, "skip section cannot be null"); this.executableSections = Collections.unmodifiableList(executableSections); } diff --git a/x-pack/plugin/runtime-fields/build.gradle b/x-pack/plugin/runtime-fields/build.gradle new file mode 100644 index 0000000000000..1f44b5c019e55 --- /dev/null +++ b/x-pack/plugin/runtime-fields/build.gradle @@ -0,0 +1,26 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'x-pack-runtime-fields' + description 'A module which adds support for runtime fields' + classname 'org.elasticsearch.xpack.runtimefields.RuntimeFields' + extendedPlugins = ['x-pack-core', 'lang-painless'] +} +archivesBaseName = 'x-pack-runtime-fields' + +compileJava.options.compilerArgs << "-Xlint:-rawtypes" +compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" + +dependencies { + compileOnly project(":server") + compileOnly project(':modules:lang-painless:spi') + compileOnly project(path: xpackModule('core'), configuration: 'default') +} + +dependencyLicenses { + ignoreSha 'x-pack-core' +} + +integTest.enabled = false diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle new file mode 100644 index 0000000000000..7dc01b73ed9ec --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -0,0 +1 @@ +// Empty project so we can pick up its subproject diff --git a/x-pack/plugin/runtime-fields/qa/rest/build.gradle b/x-pack/plugin/runtime-fields/qa/rest/build.gradle new file mode 100644 index 0000000000000..bb63670f1696a --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/rest/build.gradle @@ -0,0 +1,54 @@ +apply plugin: 'elasticsearch.yaml-rest-test' + +restResources { + restApi { + includeXpack 'async_search', 'graph', '*_point_in_time' + } + restTests { + includeCore '*' + includeXpack 'async_search', 'graph' + } +} + +testClusters.yamlRestTest { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' +} + +yamlRestTest { + systemProperty 'tests.rest.suite', + [ + 'async_search', + 'field_caps', + 'graph', + 'msearch', + 'search', + 'search.aggregation', + 'search.highlight', + 'search.inner_hits', + 'search_shards', + 'suggest', + ].join(',') + systemProperty 'tests.rest.blacklist', + [ + /////// TO FIX /////// + 'search/330_fetch_fields/*', // The whole API is not yet supported + 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? + 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. + 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? + 'search.aggregation/10_histogram/*', // runtime_script doesn't support sub-fields. Maybe it should? + 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', + /////// TO FIX /////// + + /////// NOT SUPPORTED /////// + 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it + // Runtime fields don't have global ords + 'search.aggregation/20_terms/string profiler via global ordinals', + 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', + //dynamic template causes a type _doc to be created, these tests use another type but only one type is allowed + 'search.aggregation/51_filter_with_types/*', + 'search/171_terms_query_with_types/*', + 'msearch/12_basic_with_types/*' + /////// NOT SUPPORTED /////// + ].join(',') +} diff --git a/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java new file mode 100644 index 0000000000000..5fa6495b9c220 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java @@ -0,0 +1,304 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.rest; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; +import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; +import org.elasticsearch.test.rest.yaml.section.DoSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; +import org.elasticsearch.test.rest.yaml.section.SetupSection; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.hamcrest.Matchers.equalTo; + +public class CoreTestsWithRuntimeFieldsIT extends ESClientYamlSuiteTestCase { + public CoreTestsWithRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + /** + * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, + * replacing the body of index creation commands so that fields are {@code runtime_script}s + * that load from {@code source} instead of their original type. Test configurations that + * do are not modified to contain runtime fields are not returned as they are tested + * elsewhere. + */ + @ParametersFactory + public static Iterable parameters() throws Exception { + Map suites = new HashMap<>(); + List result = new ArrayList<>(); + for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { + assert orig.length == 1; + ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; + ClientYamlTestSuite suite = suites.computeIfAbsent(candidate.getTestPath(), k -> modifiedSuite(candidate)); + if (suite == null) { + // The setup section contains an unsupported option + continue; + } + if (false == modifySection(candidate.getTestSection().getExecutableSections())) { + // The test section contains an unsupported option + continue; + } + ClientYamlTestSection modified = new ClientYamlTestSection( + candidate.getTestSection().getLocation(), + candidate.getTestSection().getName(), + candidate.getTestSection().getSkipSection(), + candidate.getTestSection().getExecutableSections() + ); + result.add(new Object[] { new ClientYamlTestCandidate(suite, modified) }); + } + return result; + } + + /** + * Modify the setup section to setup a dynamic template that replaces + * field configurations with scripts that load from source + * and replaces field configurations in {@code incides.create} + * with scripts that load from source. + */ + private static ClientYamlTestSuite modifiedSuite(ClientYamlTestCandidate candidate) { + if (false == modifySection(candidate.getSetupSection().getExecutableSections())) { + return null; + } + List setup = new ArrayList<>(candidate.getSetupSection().getExecutableSections().size() + 1); + setup.add(ADD_TEMPLATE); + setup.addAll(candidate.getSetupSection().getExecutableSections()); + return new ClientYamlTestSuite( + candidate.getApi(), + candidate.getName(), + new SetupSection(candidate.getSetupSection().getSkipSection(), setup), + candidate.getTeardownSection(), + Collections.emptyList() + ); + } + + /** + * Replace field configuration in {@code indices.create} with scripts + * that load from the source. + */ + private static boolean modifySection(List executables) { + for (ExecutableSection section : executables) { + if (false == (section instanceof DoSection)) { + continue; + } + DoSection doSection = (DoSection) section; + if (false == doSection.getApiCallSection().getApi().equals("indices.create")) { + continue; + } + for (Map body : doSection.getApiCallSection().getBodies()) { + Object settings = body.get("settings"); + if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { + /* + * You can't sort the index on a runtime_keyword and it is + * hard to figure out if the sort was a runtime_keyword so + * let's just skip this test. + */ + continue; + } + Object mappings = body.get("mappings"); + if (false == (mappings instanceof Map)) { + continue; + } + Object properties = ((Map) mappings).get("properties"); + if (false == (properties instanceof Map)) { + continue; + } + for (Map.Entry property : ((Map) properties).entrySet()) { + if (false == property.getValue() instanceof Map) { + continue; + } + @SuppressWarnings("unchecked") + Map propertyMap = (Map) property.getValue(); + String name = property.getKey().toString(); + String type = Objects.toString(propertyMap.get("type")); + if ("nested".equals(type)) { + // Our loading scripts can't be made to manage nested fields so we have to skip those tests. + return false; + } + if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { + // If doc_values is false we can't emulate with scripts. `null` and `true` are fine. + continue; + } + if ("false".equals(Objects.toString(propertyMap.get("index")))) { + // If index is false we can't emulate with scripts + continue; + } + if ("true".equals(Objects.toString(propertyMap.get("store")))) { + // If store is true we can't emulate with scripts + continue; + } + if (propertyMap.containsKey("ignore_above")) { + // Scripts don't support ignore_above so we skip those fields + continue; + } + if (propertyMap.containsKey("ignore_malformed")) { + // Our source reading script doesn't emulate ignore_malformed + continue; + } + String toLoad = painlessToLoadFromSource(name, type); + if (toLoad == null) { + continue; + } + propertyMap.put("type", "runtime_script"); + propertyMap.put("runtime_type", type); + propertyMap.put("script", toLoad); + propertyMap.remove("store"); + propertyMap.remove("index"); + propertyMap.remove("doc_values"); + } + } + } + return true; + } + + private static String painlessToLoadFromSource(String name, String type) { + String emit = PAINLESS_TO_EMIT.get(type); + if (emit == null) { + return null; + } + StringBuilder b = new StringBuilder(); + b.append("def v = params._source['").append(name).append("'];\n"); + b.append("if (v instanceof Iterable) {\n"); + b.append(" for (def vv : ((Iterable) v)) {\n"); + b.append(" if (vv != null) {\n"); + b.append(" def value = vv;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append(" }\n"); + b.append("} else {\n"); + b.append(" if (v != null) {\n"); + b.append(" def value = v;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append("}\n"); + return b.toString(); + } + + private static final Map PAINLESS_TO_EMIT = org.elasticsearch.common.collect.Map.of( + BooleanFieldMapper.CONTENT_TYPE, + "emitValue(parse(value));", + DateFieldMapper.CONTENT_TYPE, + "emitValue(parse(value.toString()));", + NumberType.DOUBLE.typeName(), + "emitValue(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));", + KeywordFieldMapper.CONTENT_TYPE, + "emitValue(value.toString());", + IpFieldMapper.CONTENT_TYPE, + "emitValue(value.toString());", + NumberType.LONG.typeName(), + "emitValue(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" + ); + + private static final ExecutableSection ADD_TEMPLATE = new ExecutableSection() { + @Override + public XContentLocation getLocation() { + return new XContentLocation(-1, -1); + } + + @Override + public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { + Map params = org.elasticsearch.common.collect.Map.of("name", "convert_to_source_only", "create", "true"); + List> dynamicTemplates = new ArrayList<>(); + for (String type : PAINLESS_TO_EMIT.keySet()) { + if (type.equals("ip")) { + // There isn't a dynamic template to pick up ips. They'll just look like strings. + continue; + } + Map mapping = org.elasticsearch.common.collect.Map.of( + "type", + "runtime_script", + "runtime_type", + type, + "script", + painlessToLoadFromSource("{name}", type) + ); + if (type.contentEquals("keyword")) { + /* + * For "string"-type dynamic mappings emulate our default + * behavior with a top level text field and a `.keyword` + * multi-field. But instead of the default, use a runtime + * field for the multi-field. + */ + mapping = org.elasticsearch.common.collect.Map.of( + "type", + "text", + "fields", + org.elasticsearch.common.collect.Map.of("keyword", mapping) + ); + dynamicTemplates.add( + org.elasticsearch.common.collect.Map.of( + type, + org.elasticsearch.common.collect.Map.of("match_mapping_type", "string", "mapping", mapping) + ) + ); + } else { + dynamicTemplates.add( + org.elasticsearch.common.collect.Map.of( + type, + org.elasticsearch.common.collect.Map.of("match_mapping_type", type, "mapping", mapping) + ) + ); + } + } + Map map = org.elasticsearch.common.collect.Map.of( + "index_patterns", + "*", + "priority", + Integer.MAX_VALUE - 1, + "template", + org.elasticsearch.common.collect.Map.of( + "settings", + Collections.emptyMap(), + "mappings", + org.elasticsearch.common.collect.Map.of("dynamic_templates", dynamicTemplates) + ) + ); + + XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent()).map(map); + String string = Strings.toString(builder); + System.out.println(string); + + List> bodies = Collections.singletonList( + map + ); + ClientYamlTestResponse response = executionContext.callApi( + "indices.put_index_template", + params, + bodies, + Collections.emptyMap() + ); + + + assertThat(response.getStatusCode(), equalTo(200)); + // There are probably some warning about overlapping templates. Ignore them. + } + }; +} diff --git a/x-pack/plugin/runtime-fields/qa/with-security/build.gradle b/x-pack/plugin/runtime-fields/qa/with-security/build.gradle new file mode 100644 index 0000000000000..8442e1aa7b0c2 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/with-security/build.gradle @@ -0,0 +1,26 @@ +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testImplementation project(path: xpackProject('plugin').path, configuration: 'testArtifacts') +} + +def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'), + password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')] + +integTest { + systemProperty 'tests.rest.cluster.username', clusterCredentials.username + systemProperty 'tests.rest.cluster.password', clusterCredentials.password +} + +testClusters.integTest { + testDistribution = 'DEFAULT' + setting 'xpack.security.enabled', 'true' + setting 'xpack.watcher.enabled', 'false' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.license.self_generated.type', 'trial' + extraConfigFile 'roles.yml', file('roles.yml') + user clusterCredentials + user username: "test", password: "x-pack-test-password", role: "test" +} diff --git a/x-pack/plugin/runtime-fields/qa/with-security/roles.yml b/x-pack/plugin/runtime-fields/qa/with-security/roles.yml new file mode 100644 index 0000000000000..b38ad1b8d0ecf --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/with-security/roles.yml @@ -0,0 +1,13 @@ +test: + indices: + - names: [ 'dls' ] + privileges: + - read + query: "{\"match\": {\"year\": 2016}}" + - names: [ 'fls' ] + privileges: + - read + field_security: + grant: [ '*' ] + except: [ 'year', 'hidden' ] + diff --git a/x-pack/plugin/runtime-fields/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java b/x-pack/plugin/runtime-fields/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java new file mode 100644 index 0000000000000..fe6732519a782 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.security; + +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.AfterClass; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; + +public class PermissionsIT extends ESRestTestCase { + + private static HighLevelClient highLevelClient; + private static HighLevelClient adminHighLevelClient; + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("test", new SecureString("x-pack-test-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Override + protected Settings restAdminSettings() { + String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Before + public void initHighLevelClient() { + if (highLevelClient == null) { + highLevelClient = new HighLevelClient(client()); + adminHighLevelClient = new HighLevelClient(adminClient()); + } + } + + @AfterClass + public static void closeHighLevelClients() throws IOException { + highLevelClient.close(); + adminHighLevelClient.close(); + highLevelClient = null; + adminHighLevelClient = null; + } + + public void testDLS() throws IOException { + Request createIndex = new Request("PUT", "/dls"); + createIndex.setJsonEntity( + "{\n" + + " \"mappings\" : {\n" + + " \"properties\" : {\n" + + " \"date\" : {\"type\" : \"keyword\"},\n" + + " \"year\" : {\n" + + " \"type\" : \"runtime_script\", \n" + + " \"runtime_type\" : \"keyword\",\n" + + " \"script\" : \"emitValue(doc['date'].value.substring(0,4))\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + ); + assertOK(adminClient().performRequest(createIndex)); + + Request indexDoc1 = new Request("PUT", "/dls/_doc/1"); + indexDoc1.setJsonEntity("{\n" + " \"date\" : \"2009-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc1)); + + Request indexDoc2 = new Request("PUT", "/dls/_doc/2"); + indexDoc2.setJsonEntity("{\n" + " \"date\" : \"2016-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc2)); + + Request indexDoc3 = new Request("PUT", "/dls/_doc/3"); + indexDoc3.addParameter("refresh", "true"); + indexDoc3.setJsonEntity("{\n" + " \"date\" : \"2018-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc3)); + + SearchRequest searchRequest = new SearchRequest("dls"); + { + SearchResponse searchResponse = adminHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + assertEquals(3, searchResponse.getHits().getTotalHits().value); + } + { + SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT); + assertEquals(1, searchResponse.getHits().getTotalHits().value); + } + } + + public void testFLSProtectsData() throws IOException { + Request createIndex = new Request("PUT", "/fls"); + createIndex.setJsonEntity( + "{\n" + + " \"mappings\" : {\n" + + " \"properties\" : {\n" + + " \"hidden\" : {\"type\" : \"keyword\"},\n" + + " \"hidden_values_count\" : {\n" + + " \"type\" : \"runtime_script\", \n" + + " \"runtime_type\" : \"long\",\n" + + " \"script\" : \"emitValue(doc['hidden'].size())\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + ); + assertOK(adminClient().performRequest(createIndex)); + + Request indexDoc1 = new Request("PUT", "/fls/_doc/1"); + indexDoc1.setJsonEntity("{\n" + " \"hidden\" : \"should not be read\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc1)); + + Request indexDoc2 = new Request("PUT", "/fls/_doc/2"); + indexDoc2.setJsonEntity("{\n" + " \"hidden\" : \"should not be read\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc2)); + + Request indexDoc3 = new Request("PUT", "/fls/_doc/3"); + indexDoc3.addParameter("refresh", "true"); + indexDoc3.setJsonEntity("{\n" + " \"hidden\" : \"should not be read\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc3)); + + SearchRequest searchRequest = new SearchRequest("fls").source(new SearchSourceBuilder().docValueField("hidden_values_count")); + { + SearchResponse searchResponse = adminHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + assertEquals(3, searchResponse.getHits().getTotalHits().value); + for (SearchHit hit : searchResponse.getHits().getHits()) { + assertEquals(1, hit.getFields().size()); + assertEquals(1, (int) hit.getFields().get("hidden_values_count").getValue()); + } + } + { + SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT); + assertEquals(3, searchResponse.getHits().getTotalHits().value); + for (SearchHit hit : searchResponse.getHits().getHits()) { + assertEquals(0, (int) hit.getFields().get("hidden_values_count").getValue()); + } + } + } + + public void testFLSOnRuntimeField() throws IOException { + Request createIndex = new Request("PUT", "/fls"); + createIndex.setJsonEntity( + "{\n" + + " \"mappings\" : {\n" + + " \"properties\" : {\n" + + " \"date\" : {\"type\" : \"keyword\"},\n" + + " \"year\" : {\n" + + " \"type\" : \"runtime_script\", \n" + + " \"runtime_type\" : \"keyword\",\n" + + " \"script\" : \"emitValue(doc['date'].value.substring(0,4))\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n" + ); + assertOK(adminClient().performRequest(createIndex)); + + Request indexDoc1 = new Request("PUT", "/fls/_doc/1"); + indexDoc1.setJsonEntity("{\n" + " \"date\" : \"2009-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc1)); + + Request indexDoc2 = new Request("PUT", "/fls/_doc/2"); + indexDoc2.setJsonEntity("{\n" + " \"date\" : \"2016-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc2)); + + Request indexDoc3 = new Request("PUT", "/fls/_doc/3"); + indexDoc3.addParameter("refresh", "true"); + indexDoc3.setJsonEntity("{\n" + " \"date\" : \"2018-11-15T14:12:12\"\n" + "}\n"); + assertOK(adminClient().performRequest(indexDoc3)); + + // There is no FLS directly on runtime fields + SearchRequest searchRequest = new SearchRequest("fls").source(new SearchSourceBuilder().docValueField("year")); + SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT); + assertEquals(3, searchResponse.getHits().getTotalHits().value); + for (SearchHit hit : searchResponse.getHits().getHits()) { + Map fields = hit.getFields(); + assertEquals(1, fields.size()); + switch (hit.getId()) { + case "1": + assertEquals("2009", fields.get("year").getValue().toString()); + break; + case "2": + assertEquals("2016", fields.get("year").getValue().toString()); + break; + case "3": + assertEquals("2018", fields.get("year").getValue().toString()); + break; + default: + throw new UnsupportedOperationException(); + } + } + + { + FieldCapabilitiesRequest fieldCapsRequest = new FieldCapabilitiesRequest().indices("fls").fields("year"); + FieldCapabilitiesResponse fieldCapabilitiesResponse = adminHighLevelClient.fieldCaps(fieldCapsRequest, RequestOptions.DEFAULT); + assertNotNull(fieldCapabilitiesResponse.get().get("year")); + } + { + // Though field_caps filters runtime fields out like ordinary fields + FieldCapabilitiesRequest fieldCapsRequest = new FieldCapabilitiesRequest().indices("fls").fields("year"); + FieldCapabilitiesResponse fieldCapabilitiesResponse = highLevelClient.fieldCaps(fieldCapsRequest, RequestOptions.DEFAULT); + assertEquals(0, fieldCapabilitiesResponse.get().size()); + } + } + + private static class HighLevelClient extends RestHighLevelClient { + private HighLevelClient(RestClient restClient) { + super(restClient, (client) -> {}, Collections.emptyList()); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java new file mode 100644 index 0000000000000..d7b261a4fae78 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.util.Map; + +/** + * Common base class for script field scripts that return long values. + */ +public abstract class AbstractLongScriptFieldScript extends AbstractScriptFieldScript { + private long[] values = new long[1]; + private int count; + + public AbstractLongScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + */ + public final void runForDoc(int docId) { + count = 0; + setDocument(docId); + execute(); + } + + /** + * Values from the last time {@link #runForDoc(int)} was called. This array + * is mutable and will change with the next call of {@link #runForDoc(int)}. + * It is also oversized and will contain garbage at all indices at and + * above {@link #count()}. + */ + public final long[] values() { + return values; + } + + /** + * The number of results produced the last time {@link #runForDoc(int)} was called. + */ + public final int count() { + return count; + } + + protected final void emitValue(long v) { + if (values.length < count + 1) { + values = ArrayUtil.grow(values, count + 1); + } + values[count++] = v; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java new file mode 100644 index 0000000000000..5fe3d296b3ab7 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.script.DynamicMap; +import org.elasticsearch.script.ScriptCache; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.LeafSearchLookup; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; + +/** + * Abstract base for scripts to execute to build scripted fields. Inspired by + * {@link AggregationScript} but hopefully with less historical baggage. + */ +public abstract class AbstractScriptFieldScript { + public static ScriptContext newContext(String name, Class factoryClass) { + return new ScriptContext( + name + "_script_field", + factoryClass, + /* + * In an ideal world we wouldn't need the script cache at all + * because we have a hard reference to the script. The trouble + * is that we compile the scripts a few times when performing + * a mapping update. This is unfortunate, but we rely on the + * cache to speed this up. + */ + 100, + timeValueMillis(0), + /* + * Disable compilation rate limits for scripted fields so we + * don't prevent mapping updates because we've performed too + * many recently. That'd just be lame. + */ + ScriptCache.UNLIMITED_COMPILATION_RATE.asTuple() + ); + } + + private static final Map> PARAMS_FUNCTIONS = org.elasticsearch.common.collect.Map.of( + "_source", + value -> ((SourceLookup) value).loadSourceIfNeeded() + ); + + private final Map params; + private final LeafSearchLookup leafSearchLookup; + + public AbstractScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + this.leafSearchLookup = searchLookup.getLeafSearchLookup(ctx); + params = new HashMap<>(params); + params.put("_source", leafSearchLookup.source()); + params.put("_fields", leafSearchLookup.fields()); + this.params = new DynamicMap(params, PARAMS_FUNCTIONS); + } + + /** + * Set the document to run the script against. + */ + public final void setDocument(int docId) { + this.leafSearchLookup.setDocument(docId); + } + + /** + * Expose the {@code params} of the script to the script itself. + */ + public final Map getParams() { + return params; + } + + /** + * Expose the {@code _source} to the script. + */ + @SuppressWarnings("unchecked") + protected final Map getSource() { + return leafSearchLookup.source(); + } + + /** + * Expose field data to the script as {@code doc}. + */ + public final Map> getDoc() { + return leafSearchLookup.doc(); + } + + public abstract void execute(); +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScript.java new file mode 100644 index 0000000000000..3e4ef9ce11e06 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScript.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.Booleans; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public abstract class BooleanScriptFieldScript extends AbstractScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("boolean_script_field", Factory.class); + + static List whitelist() { + return Collections.singletonList( + WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "boolean_whitelist.txt") + ); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + BooleanScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private int trues; + private int falses; + + public BooleanScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + */ + public final void runForDoc(int docId) { + trues = 0; + falses = 0; + setDocument(docId); + execute(); + } + + /** + * How many {@code true} values were returned for this document. + */ + public final int trues() { + return trues; + } + + /** + * How many {@code false} values were returned for this document. + */ + public final int falses() { + return falses; + } + + protected final void emitValue(boolean v) { + if (v) { + trues++; + } else { + falses++; + } + } + + public static boolean parse(Object str) { + return Booleans.parseBoolean(str.toString()); + } + + public static class EmitValue { + private final BooleanScriptFieldScript script; + + public EmitValue(BooleanScriptFieldScript script) { + this.script = script; + } + + public void value(boolean v) { + script.emitValue(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java new file mode 100644 index 0000000000000..9b9ff47cd6871 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public abstract class DateScriptFieldScript extends AbstractLongScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("date", Factory.class); + + static List whitelist() { + return Collections.singletonList(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "date_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup, DateFormatter formatter); + } + + public interface LeafFactory { + DateScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private final DateFormatter formatter; + + public DateScriptFieldScript(Map params, SearchLookup searchLookup, DateFormatter formatter, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + this.formatter = formatter; + } + + public static long toEpochMilli(TemporalAccessor v) { + // TemporalAccessor is a nanos API so we have to convert. + long millis = Math.multiplyExact(v.getLong(ChronoField.INSTANT_SECONDS), 1000); + millis = Math.addExact(millis, v.get(ChronoField.NANO_OF_SECOND) / 1_000_000); + return millis; + } + + public static class EmitValue { + private final DateScriptFieldScript script; + + public EmitValue(DateScriptFieldScript script) { + this.script = script; + } + + public void emitValue(long v) { + script.emitValue(v); + } + } + + public static class Parse { + private final DateScriptFieldScript script; + + public Parse(DateScriptFieldScript script) { + this.script = script; + } + + public long parse(Object str) { + return script.formatter.parseMillis(str.toString()); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java new file mode 100644 index 0000000000000..a4fc32542067e --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public abstract class DoubleScriptFieldScript extends AbstractScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("double_script_field", Factory.class); + + static List whitelist() { + return Collections.singletonList( + WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "double_whitelist.txt") + ); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + DoubleScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private double[] values = new double[1]; + private int count; + + public DoubleScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + */ + public final void runForDoc(int docId) { + count = 0; + setDocument(docId); + execute(); + } + + /** + * Values from the last time {@link #runForDoc(int)} was called. This array + * is mutable and will change with the next call of {@link #runForDoc(int)}. + * It is also oversized and will contain garbage at all indices at and + * above {@link #count()}. + */ + public final double[] values() { + return values; + } + + /** + * The number of results produced the last time {@link #runForDoc(int)} was called. + */ + public final int count() { + return count; + } + + protected final void emitValue(double v) { + if (values.length < count + 1) { + values = ArrayUtil.grow(values, count + 1); + } + values[count++] = v; + } + + public static class EmitValue { + private final DoubleScriptFieldScript script; + + public EmitValue(DoubleScriptFieldScript script) { + this.script = script; + } + + public void emitValue(double v) { + script.emitValue(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java new file mode 100644 index 0000000000000..e99ebb4513da9 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Script producing IP addresses. Unlike the other {@linkplain AbstractScriptFieldScript}s + * which deal with their native java objects this converts its values to the same format + * that Lucene uses to store its fields, {@link InetAddressPoint}. There are a few compelling + * reasons to do this: + *
    + *
  • {@link Inet4Address}es and {@link Inet6Address} are not comparable with one another. + * That is correct in some contexts, but not for our queries. Our queries must consider the + * IPv4 address equal to the address that it maps to in IPv6 rfc4291). + *
  • {@link InetAddress}es are not ordered, but we need to implement range queries with + * same same ordering as {@link IpFieldMapper}. That also uses {@link InetAddressPoint} + * so it saves us a lot of trouble to use the same representation. + *
+ */ +public abstract class IpScriptFieldScript extends AbstractScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("ip_script_field", Factory.class); + + static List whitelist() { + return Collections.singletonList(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "ip_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + IpScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private BytesRef[] values = new BytesRef[1]; + private int count; + + public IpScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + */ + public final void runForDoc(int docId) { + count = 0; + setDocument(docId); + execute(); + } + + /** + * Values from the last time {@link #runForDoc(int)} was called. This array + * is mutable and will change with the next call of {@link #runForDoc(int)}. + * It is also oversized and will contain garbage at all indices at and + * above {@link #count()}. + *

+ * All values are IPv6 addresses so they are 16 bytes. IPv4 addresses are + * encoded by rfc4291. + */ + public final BytesRef[] values() { + return values; + } + + /** + * The number of results produced the last time {@link #runForDoc(int)} was called. + */ + public final int count() { + return count; + } + + protected final void emitValue(String v) { + if (values.length < count + 1) { + values = ArrayUtil.grow(values, count + 1); + } + values[count++] = new BytesRef(InetAddressPoint.encode(InetAddresses.forString(v))); + } + + public static class EmitValue { + private final IpScriptFieldScript script; + + public EmitValue(IpScriptFieldScript script) { + this.script = script; + } + + public void emitValue(String v) { + script.emitValue(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java new file mode 100644 index 0000000000000..e50ccee619f05 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public abstract class LongScriptFieldScript extends AbstractLongScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("long_script_field", Factory.class); + + static List whitelist() { + return Collections.singletonList(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "long_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + public LongScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + public static class EmitValue { + private final LongScriptFieldScript script; + + public EmitValue(LongScriptFieldScript script) { + this.script = script; + } + + public void emitValue(long v) { + script.emitValue(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java new file mode 100644 index 0000000000000..2e8ce5bee4efb --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.xpack.runtimefields.mapper.RuntimeScriptFieldMapper; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin { + + @Override + public Map getMappers() { + return Collections.singletonMap(RuntimeScriptFieldMapper.CONTENT_TYPE, RuntimeScriptFieldMapper.PARSER); + } + + @Override + public List> getContexts() { + return org.elasticsearch.common.collect.List.of( + BooleanScriptFieldScript.CONTEXT, + DateScriptFieldScript.CONTEXT, + DoubleScriptFieldScript.CONTEXT, + IpScriptFieldScript.CONTEXT, + LongScriptFieldScript.CONTEXT, + StringScriptFieldScript.CONTEXT + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java new file mode 100644 index 0000000000000..5d82c5935ff04 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.script.ScriptContext; + +import java.util.List; +import java.util.Map; + +public class RuntimeFieldsPainlessExtension implements PainlessExtension { + @Override + public Map, List> getContextWhitelists() { + return org.elasticsearch.common.collect.Map.of( + BooleanScriptFieldScript.CONTEXT, + BooleanScriptFieldScript.whitelist(), + DateScriptFieldScript.CONTEXT, + DateScriptFieldScript.whitelist(), + DoubleScriptFieldScript.CONTEXT, + DoubleScriptFieldScript.whitelist(), + IpScriptFieldScript.CONTEXT, + IpScriptFieldScript.whitelist(), + LongScriptFieldScript.CONTEXT, + LongScriptFieldScript.whitelist(), + StringScriptFieldScript.CONTEXT, + StringScriptFieldScript.whitelist() + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java new file mode 100644 index 0000000000000..7faf8fbffc897 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public abstract class StringScriptFieldScript extends AbstractScriptFieldScript { + public static final ScriptContext CONTEXT = newContext("string_script_field", Factory.class); + + static List whitelist() { + return Collections.singletonList( + WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "string_whitelist.txt") + ); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + StringScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private final List results = new ArrayList<>(); + + public StringScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + *

+ * @return a mutable {@link List} that contains the results of the script + * and will be modified the next time you call {@linkplain #resultsForDoc}. + */ + public final List resultsForDoc(int docId) { + results.clear(); + setDocument(docId); + execute(); + return results; + } + + protected final void emitValue(String v) { + results.add(v); + } + + public static class EmitValue { + private final StringScriptFieldScript script; + + public EmitValue(StringScriptFieldScript script) { + this.script = script; + } + + public void emitValue(String v) { + script.emitValue(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java new file mode 100644 index 0000000000000..7db0d6bd79b4d --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafFieldData; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +public abstract class ScriptBinaryFieldData implements IndexFieldData { + private final String fieldName; + + protected ScriptBinaryFieldData(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ScriptBinaryLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + final XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); + return new SortField(getFieldName(), source, reverse); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("only supported on numeric fields"); + } + + public abstract class ScriptBinaryLeafFieldData implements LeafFieldData { + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() { + + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanDocValues.java new file mode 100644 index 0000000000000..f91fe3d300eaf --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanDocValues.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; +import org.elasticsearch.xpack.runtimefields.BooleanScriptFieldScript; + +import java.io.IOException; + +public final class ScriptBooleanDocValues extends AbstractSortedNumericDocValues { + private final BooleanScriptFieldScript script; + private int cursor; + + ScriptBooleanDocValues(BooleanScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + script.runForDoc(docId); + cursor = 0; + return script.trues() > 0 || script.falses() > 0; + } + + @Override + public long nextValue() throws IOException { + // Emit all false values before all true values + return cursor++ < script.falses() ? 0 : 1; + } + + @Override + public int docValueCount() { + return script.trues() + script.falses(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanFieldData.java new file mode 100644 index 0000000000000..41a8c814bddc2 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBooleanFieldData.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.plain.LeafLongFieldData; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.BooleanScriptFieldScript; + +import java.io.IOException; + +public final class ScriptBooleanFieldData extends IndexNumericFieldData { + + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final BooleanScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, BooleanScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptBooleanFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptBooleanFieldData(name, leafFactory); + } + } + + private final String fieldName; + private final BooleanScriptFieldScript.LeafFactory leafFactory; + + private ScriptBooleanFieldData(String fieldName, BooleanScriptFieldScript.LeafFactory leafFactory) { + this.fieldName = fieldName; + this.leafFactory = leafFactory; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BOOLEAN; + } + + @Override + public ScriptBooleanLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public ScriptBooleanLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + return new ScriptBooleanLeafFieldData(new ScriptBooleanDocValues(leafFactory.newInstance(context))); + } + + @Override + public NumericType getNumericType() { + return NumericType.BOOLEAN; + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + public static class ScriptBooleanLeafFieldData extends LeafLongFieldData { + private final ScriptBooleanDocValues scriptBooleanDocValues; + + ScriptBooleanLeafFieldData(ScriptBooleanDocValues scriptBooleanDocValues) { + super(0, NumericType.BOOLEAN); + this.scriptBooleanDocValues = scriptBooleanDocValues; + } + + @Override + public SortedNumericDocValues getLongValues() { + return scriptBooleanDocValues; + } + + @Override + public void close() {} + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDateFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDateFieldData.java new file mode 100644 index 0000000000000..4a4e9b58e47ef --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDateFieldData.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.plain.LeafLongFieldData; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; + +import java.io.IOException; + +public final class ScriptDateFieldData extends IndexNumericFieldData { + + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final DateScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, DateScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptDateFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptDateFieldData(name, leafFactory); + } + } + + private final String fieldName; + private final DateScriptFieldScript.LeafFactory leafFactory; + + private ScriptDateFieldData(String fieldName, DateScriptFieldScript.LeafFactory leafFactory) { + this.fieldName = fieldName; + this.leafFactory = leafFactory; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.DATE; + } + + @Override + public ScriptDateLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public ScriptDateLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + return new ScriptDateLeafFieldData(new ScriptLongDocValues(leafFactory.newInstance(context))); + } + + @Override + public NumericType getNumericType() { + return NumericType.DATE; + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + public static class ScriptDateLeafFieldData extends LeafLongFieldData { + private final ScriptLongDocValues scriptLongDocValues; + + ScriptDateLeafFieldData(ScriptLongDocValues scriptLongDocValues) { + super(0, NumericType.DATE); + this.scriptLongDocValues = scriptLongDocValues; + } + + @Override + public SortedNumericDocValues getLongValues() { + return scriptLongDocValues; + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleDocValues.java new file mode 100644 index 0000000000000..135229e260eba --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleDocValues.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +import java.io.IOException; +import java.util.Arrays; + +public final class ScriptDoubleDocValues extends SortedNumericDoubleValues { + private final DoubleScriptFieldScript script; + private int cursor; + + ScriptDoubleDocValues(DoubleScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + script.runForDoc(docId); + if (script.count() == 0) { + return false; + } + Arrays.sort(script.values(), 0, script.count()); + cursor = 0; + return true; + } + + @Override + public double nextValue() throws IOException { + return script.values()[cursor++]; + } + + @Override + public int docValueCount() { + return script.count(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleFieldData.java new file mode 100644 index 0000000000000..a15a79016e434 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDoubleFieldData.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.fielddata.plain.LeafDoubleFieldData; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; + +import java.io.IOException; + +public final class ScriptDoubleFieldData extends IndexNumericFieldData { + + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final DoubleScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, DoubleScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptDoubleFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptDoubleFieldData(name, leafFactory); + } + } + + private final String fieldName; + DoubleScriptFieldScript.LeafFactory leafFactory; + + private ScriptDoubleFieldData(String fieldName, DoubleScriptFieldScript.LeafFactory leafFactory) { + this.fieldName = fieldName; + this.leafFactory = leafFactory; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + + @Override + public ScriptDoubleLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public ScriptDoubleLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + return new ScriptDoubleLeafFieldData(new ScriptDoubleDocValues(leafFactory.newInstance(context))); + } + + @Override + public NumericType getNumericType() { + return NumericType.DOUBLE; + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + public static class ScriptDoubleLeafFieldData extends LeafDoubleFieldData { + private final ScriptDoubleDocValues scriptDoubleDocValues; + + ScriptDoubleLeafFieldData(ScriptDoubleDocValues scriptDoubleDocValues) { + super(0); + this.scriptDoubleDocValues = scriptDoubleDocValues; + } + + @Override + public SortedNumericDoubleValues getDoubleValues() { + return scriptDoubleDocValues; + } + + @Override + public void close() {} + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpDocValues.java new file mode 100644 index 0000000000000..0cecde2302d41 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpDocValues.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; + +import java.io.IOException; +import java.util.Arrays; + +public final class ScriptIpDocValues extends SortedBinaryDocValues { + private final IpScriptFieldScript script; + private int cursor; + + ScriptIpDocValues(IpScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + script.runForDoc(docId); + if (script.count() == 0) { + return false; + } + Arrays.sort(script.values(), 0, script.count()); + cursor = 0; + return true; + } + + @Override + public BytesRef nextValue() throws IOException { + return script.values()[cursor++]; + } + + @Override + public int docValueCount() { + return script.count(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpFieldData.java new file mode 100644 index 0000000000000..e9e7e86b3ab32 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpFieldData.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; + +import java.net.InetAddress; + +public class ScriptIpFieldData extends ScriptBinaryFieldData { + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final IpScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, IpScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptIpFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptIpFieldData(name, leafFactory); + } + } + + private final IpScriptFieldScript.LeafFactory leafFactory; + + private ScriptIpFieldData(String fieldName, IpScriptFieldScript.LeafFactory leafFactory) { + super(fieldName); + this.leafFactory = leafFactory; + } + + @Override + public ScriptBinaryLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + IpScriptFieldScript script = leafFactory.newInstance(context); + return new ScriptBinaryLeafFieldData() { + @Override + public ScriptDocValues getScriptValues() { + return new IpScriptDocValues(getBytesValues()); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return new ScriptIpDocValues(script); + } + }; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.IP; + } + + /** + * Doc values implementation for ips. We can't share + * {@link IpFieldMapper.IpFieldType.IpScriptDocValues} because it is based + * on global ordinals and we don't have those. + */ + public static class IpScriptDocValues extends ScriptDocValues.Strings { + public IpScriptDocValues(SortedBinaryDocValues in) { + super(in); + } + + @Override + protected String bytesToString(BytesRef bytes) { + InetAddress addr = InetAddressPoint.decode(BytesReference.toBytes(new BytesArray(bytes))); + return InetAddresses.toAddrString(addr); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java new file mode 100644 index 0000000000000..3f7da9cec5a5c --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; +import org.elasticsearch.xpack.runtimefields.AbstractLongScriptFieldScript; + +import java.io.IOException; +import java.util.Arrays; + +public final class ScriptLongDocValues extends AbstractSortedNumericDocValues { + private final AbstractLongScriptFieldScript script; + private int cursor; + + ScriptLongDocValues(AbstractLongScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + script.runForDoc(docId); + if (script.count() == 0) { + return false; + } + Arrays.sort(script.values(), 0, script.count()); + cursor = 0; + return true; + } + + @Override + public long nextValue() throws IOException { + return script.values()[cursor++]; + } + + @Override + public int docValueCount() { + return script.count(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongFieldData.java new file mode 100644 index 0000000000000..7ad3e377d9254 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongFieldData.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.plain.LeafLongFieldData; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; + +import java.io.IOException; + +public final class ScriptLongFieldData extends IndexNumericFieldData { + + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final LongScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, LongScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptLongFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptLongFieldData(name, leafFactory); + } + } + + private final String fieldName; + private final LongScriptFieldScript.LeafFactory leafFactory; + + private ScriptLongFieldData(String fieldName, LongScriptFieldScript.LeafFactory leafFactory) { + this.fieldName = fieldName; + this.leafFactory = leafFactory; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.NUMERIC; + } + + @Override + public ScriptLongLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public ScriptLongLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + return new ScriptLongLeafFieldData(new ScriptLongDocValues(leafFactory.newInstance(context))); + } + + @Override + public NumericType getNumericType() { + return NumericType.LONG; + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + public static class ScriptLongLeafFieldData extends LeafLongFieldData { + private final ScriptLongDocValues scriptLongDocValues; + + ScriptLongLeafFieldData(ScriptLongDocValues scriptLongDocValues) { + super(0, NumericType.LONG); + this.scriptLongDocValues = scriptLongDocValues; + } + + @Override + public SortedNumericDocValues getLongValues() { + return scriptLongDocValues; + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringDocValues.java new file mode 100644 index 0000000000000..ada5dcff13e29 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringDocValues.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.elasticsearch.index.fielddata.SortingBinaryDocValues; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +import java.util.List; + +public final class ScriptStringDocValues extends SortingBinaryDocValues { + private final StringScriptFieldScript script; + + ScriptStringDocValues(StringScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + List results = script.resultsForDoc(docId); + count = results.size(); + if (count == 0) { + return false; + } + + grow(); + int i = 0; + for (String value : results) { + values[i++].copyChars(value); + } + sort(); + return true; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringFieldData.java new file mode 100644 index 0000000000000..6c851a6be852a --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringFieldData.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +public class ScriptStringFieldData extends ScriptBinaryFieldData { + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final StringScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, StringScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptStringFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptStringFieldData(name, leafFactory); + } + } + + private final StringScriptFieldScript.LeafFactory leafFactory; + + private ScriptStringFieldData(String fieldName, StringScriptFieldScript.LeafFactory leafFactory) { + super(fieldName); + this.leafFactory = leafFactory; + } + + @Override + public ScriptBinaryLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + StringScriptFieldScript script = leafFactory.newInstance(context); + return new ScriptBinaryLeafFieldData() { + @Override + public ScriptDocValues getScriptValues() { + return new ScriptDocValues.Strings(getBytesValues()); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return new ScriptStringDocValues(script); + } + }; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java new file mode 100644 index 0000000000000..f35464a2bcf2c --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldType.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; +import org.apache.lucene.search.spans.SpanQuery; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.script.Script; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; + +/** + * Abstract base {@linkplain MappedFieldType} for scripted fields. + */ +abstract class AbstractScriptMappedFieldType extends MappedFieldType { + protected final Script script; + + AbstractScriptMappedFieldType(String name, Script script, Map meta) { + super(name, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + this.script = script; + } + + protected abstract String runtimeType(); + + @Override + public final String typeName() { + return RuntimeScriptFieldMapper.CONTENT_TYPE; + } + + @Override + public final String familyTypeName() { + return runtimeType(); + } + + @Override + public final boolean isSearchable() { + return true; + } + + @Override + public final boolean isAggregatable() { + return true; + } + + public abstract Query termsQuery(List values, QueryShardContext context); + + @Override + public final Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ShapeRelation relation, + ZoneId timeZone, + DateMathParser parser, + QueryShardContext context + ) { + if (relation == ShapeRelation.DISJOINT) { + String message = "Field [%s] of type [%s] with runtime type [%s] does not support DISJOINT ranges"; + throw new IllegalArgumentException(String.format(Locale.ROOT, message, name(), typeName(), runtimeType())); + } + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context); + } + + protected abstract Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ZoneId timeZone, + DateMathParser parser, + QueryShardContext context + ); + + public Query fuzzyQuery( + Object value, + Fuzziness fuzziness, + int prefixLength, + int maxExpansions, + boolean transpositions, + QueryShardContext context + ) { + throw new IllegalArgumentException(unsupported("fuzzy", "keyword and text")); + } + + public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { + throw new IllegalArgumentException(unsupported("prefix", "keyword, text and wildcard")); + } + + public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { + throw new IllegalArgumentException(unsupported("wildcard", "keyword, text and wildcard")); + } + + public Query regexpQuery( + String value, + int flags, + int maxDeterminizedStates, + MultiTermQuery.RewriteMethod method, + QueryShardContext context + ) { + throw new IllegalArgumentException(unsupported("regexp", "keyword and text")); + } + + public abstract Query existsQuery(QueryShardContext context); + + public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { + throw new IllegalArgumentException(unsupported("phrase", "text")); + } + + public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { + throw new IllegalArgumentException(unsupported("phrase", "text")); + } + + public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions) throws IOException { + throw new IllegalArgumentException(unsupported("phrase prefix", "text")); + } + + public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRewriteMethod method, QueryShardContext context) { + throw new IllegalArgumentException(unsupported("span prefix", "text")); + } + + private String unsupported(String query, String supported) { + String thisField = "[" + name() + "] which is of type [script] with runtime_type [" + runtimeType() + "]"; + return "Can only use " + query + " queries on " + supported + " fields - not on " + thisField; + } + + protected final void checkAllowExpensiveQueries(QueryShardContext context) { + if (context.allowExpensiveQueries() == false) { + throw new ElasticsearchException( + "queries cannot be executed against [" + + RuntimeScriptFieldMapper.CONTENT_TYPE + + "] fields while [" + + ALLOW_EXPENSIVE_QUERIES.getKey() + + "] is set to [false]." + ); + } + } + + /** + * The format that this field should use. The default implementation is + * {@code null} because most fields don't support formats. + */ + protected String format() { + return null; + } + + /** + * The locale that this field's format should use. The default + * implementation is {@code null} because most fields don't + * support formats. + */ + protected Locale formatLocale() { + return null; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java new file mode 100644 index 0000000000000..e3e0410eb0638 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java @@ -0,0 +1,286 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.util.LocaleUtils; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.index.mapper.ParametrizedFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.xpack.runtimefields.BooleanScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.BiFunction; + +public final class RuntimeScriptFieldMapper extends ParametrizedFieldMapper { + + public static final String CONTENT_TYPE = "runtime_script"; + + public static final TypeParser PARSER = new TypeParser((name, parserContext) -> new Builder(name, new ScriptCompiler() { + @Override + public FactoryType compile(Script script, ScriptContext context) { + return parserContext.scriptService().compile(script, context); + } + })); + + private final String runtimeType; + private final Script script; + private final ScriptCompiler scriptCompiler; + + protected RuntimeScriptFieldMapper( + String simpleName, + AbstractScriptMappedFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + String runtimeType, + Script script, + ScriptCompiler scriptCompiler + ) { + super(simpleName, mappedFieldType, multiFields, copyTo); + this.runtimeType = runtimeType; + this.script = script; + this.scriptCompiler = scriptCompiler; + } + + @Override + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new RuntimeScriptFieldMapper.Builder(simpleName(), scriptCompiler).init(this); + } + + @Override + protected void parseCreateField(ParseContext context) { + // there is no lucene field + } + + @Override + public ValueFetcher valueFetcher(MapperService mapperService, String format) { + throw new UnsupportedOperationException(); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + public static class Builder extends ParametrizedFieldMapper.Builder { + + static final Map> FIELD_TYPE_RESOLVER = + org.elasticsearch.common.collect.Map.of(BooleanFieldMapper.CONTENT_TYPE, (builder, context) -> { + builder.formatAndLocaleNotSupported(); + BooleanScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + BooleanScriptFieldScript.CONTEXT + ); + return new ScriptBooleanMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, DateFieldMapper.CONTENT_TYPE, (builder, context) -> { + DateScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + DateScriptFieldScript.CONTEXT + ); + String format = builder.format.getValue(); + if (format == null) { + format = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.pattern(); + } + Locale locale = builder.locale.getValue(); + if (locale == null) { + locale = Locale.ROOT; + } + DateFormatter dateTimeFormatter = DateFormatter.forPattern(format).withLocale(locale); + return new ScriptDateMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + dateTimeFormatter, + builder.meta.getValue() + ); + }, NumberType.DOUBLE.typeName(), (builder, context) -> { + builder.formatAndLocaleNotSupported(); + DoubleScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + DoubleScriptFieldScript.CONTEXT + ); + return new ScriptDoubleMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, IpFieldMapper.CONTENT_TYPE, (builder, context) -> { + builder.formatAndLocaleNotSupported(); + IpScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + IpScriptFieldScript.CONTEXT + ); + return new ScriptIpMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, KeywordFieldMapper.CONTENT_TYPE, (builder, context) -> { + builder.formatAndLocaleNotSupported(); + StringScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + StringScriptFieldScript.CONTEXT + ); + return new ScriptKeywordMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, NumberType.LONG.typeName(), (builder, context) -> { + builder.formatAndLocaleNotSupported(); + LongScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + LongScriptFieldScript.CONTEXT + ); + return new ScriptLongMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }); + + private static RuntimeScriptFieldMapper toType(FieldMapper in) { + return (RuntimeScriptFieldMapper) in; + } + + private final Parameter> meta = Parameter.metaParam(); + private final Parameter runtimeType = Parameter.stringParam( + "runtime_type", + true, + mapper -> toType(mapper).runtimeType, + null + ).setValidator(runtimeType -> { + if (runtimeType == null) { + throw new IllegalArgumentException("runtime_type must be specified for " + CONTENT_TYPE + " field [" + name + "]"); + } + }); + private final Parameter