From 7efca6675098cfe4f57697399e7f200b47c17e79 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 10 Nov 2020 12:38:59 -0500 Subject: [PATCH 1/5] Add `runtime_mappings` to search request (backport of #64374) This adds a way to specify the `runtime_mappings` on a search request which are always "runtime" fields. It looks like: ``` curl -XDELETE -uelastic:password -HContent-Type:application/json localhost:9200/test curl -XPOST -uelastic:password -HContent-Type:application/json 'localhost:9200/test/_bulk?pretty&refresh' -d' {"index": {}} {"animal": "cat", "sound": "meow"} {"index": {}} {"animal": "dog", "sound": "woof"} {"index": {}} {"animal": "snake", "sound": "hisssssssssssssssss"} ' curl -XPOST -uelastic:password -HContent-Type:application/json localhost:9200/test/_search?pretty -d' { "runtime_mappings": { "animal.upper": { "type": "keyword", "script": "for (String s : doc[\"animal.keyword\"]) {emit(s.toUpperCase())}" } }, "query": { "match": { "animal.upper": "DOG" } } }' ``` NOTE: If we have to send a search request with runtime mappings to a node that doesn't support runtime mappings at all then we'll fail the search request entirely. The alternative would be to not send those runtime mappings and let the node fail the search request with an "unknown field" error. I believe this is would be hard to surprising because you defined the field in the search request. NOTE: It isn't obvious but you can also use `runtime_mappings` to override fields inside objects by naming the runtime fields with `.` in them. Like this: ``` curl -XDELETE -uelastic:password -HContent-Type:application/json localhost:9200/test curl -uelastic:password -XPOST -HContent-Type:application/json localhost:9200/test/_bulk?refresh -d' {"index":{}} {"name": {"first": "Andrew", "last": "Wiggin"}} {"index":{}} {"name": {"first": "Julian", "last": "Delphiki", "suffix": "II"}} ' curl -uelastic:password -XPOST -HContent-Type:application/json localhost:9200/test/_search?pretty -d'{ "runtime_mappings": { "name.first": { "type": "keyword", "script": "if (\"Wiggin\".equals(doc[\"name.last.keyword\"].value)) {emit(\"Ender\");} else if (\"Delphiki\".equals(doc[\"name.last.keyword\"].value)) {emit(\"Bean\");}" } }, "query": { "match": { "name.first": "Bean" } } }' ``` Relates to #59332 --- .../common/DisableGraphQueryTests.java | 3 +- .../action/PainlessExecuteAction.java | 5 +- .../painless/NeedsScoreTests.java | 4 +- .../PercolatorFieldMapperTests.java | 3 +- .../PercolatorQuerySearchTests.java | 3 +- ...ulkByScrollParallelizationHelperTests.java | 1 + .../test/index/80_date_nanos.yml | 33 ++ .../test/search/240_date_nanos.yml | 22 - .../TransportSimulateIndexTemplateAction.java | 3 +- .../action/search/ExpandSearchPhase.java | 3 +- .../metadata/MetadataCreateIndexService.java | 8 +- .../metadata/MetadataIndexAliasesService.java | 3 +- .../MetadataIndexTemplateService.java | 3 +- .../org/elasticsearch/index/IndexService.java | 12 +- .../index/query/QueryShardContext.java | 128 +++++- .../search/DefaultSearchContext.java | 9 +- .../elasticsearch/search/SearchService.java | 2 +- .../search/builder/SearchSourceBuilder.java | 44 +- .../search/internal/ShardSearchRequest.java | 7 +- .../action/search/ExpandSearchPhaseTests.java | 10 +- .../action/search/SearchRequestTests.java | 4 + .../fielddata/AbstractFieldDataTestCase.java | 3 +- .../index/query/QueryShardContextTests.java | 165 ++++++- .../index/search/MultiMatchQueryTests.java | 47 +- .../index/search/NestedHelperTests.java | 4 +- .../search/nested/NestedSortingTests.java | 3 +- .../search/AbstractSearchTestCase.java | 17 +- .../search/DefaultSearchContextTests.java | 8 +- .../builder/SearchSourceBuilderTests.java | 34 +- .../search/RandomSearchRequestGenerator.java | 9 +- .../elasticsearch/test/TestSearchContext.java | 4 +- .../action/EnrichShardMultiSearchAction.java | 14 +- x-pack/plugin/runtime-fields/qa/build.gradle | 75 ++- .../qa/core-with-mapped/build.gradle | 1 + .../mapped/CoreWithMappedRuntimeFieldsIT.java | 76 +++ .../qa/core-with-search/build.gradle | 1 + .../CoreTestsWithSearchRuntimeFieldsIT.java | 225 +++++++++ .../runtime-fields/qa/rest/build.gradle | 54 --- .../rest/CoreTestsWithRuntimeFieldsIT.java | 294 ------------ .../test/CoreTestTranslater.java | 432 ++++++++++++++++++ .../xpack/security/Security.java | 6 +- .../test/runtime_fields/10_keyword.yml | 107 +++++ 42 files changed, 1425 insertions(+), 464 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml create mode 100644 x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle create mode 100644 x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java create mode 100644 x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle create mode 100644 x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java delete mode 100644 x-pack/plugin/runtime-fields/qa/rest/build.gradle delete mode 100644 x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java create mode 100644 x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java index d1792e94f7331..c7ac35d3febce 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java @@ -46,6 +46,7 @@ import java.util.Collection; import java.util.Collections; +import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; /** @@ -84,7 +85,7 @@ public void setup() { indexService = createIndex("test", settings, "t", "text_shingle", "type=text,analyzer=text_shingle", "text_shingle_unigram", "type=text,analyzer=text_shingle_unigram"); - shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null); + shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); // parsed queries for "text_shingle_unigram:(foo bar baz)" with query parsers // that ignores position length attribute diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index d538ceb8545de..1ad7fd02389c6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -29,9 +29,9 @@ import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; -import org.elasticsearch.Version; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; @@ -89,6 +89,7 @@ import java.util.Objects; import static java.util.Arrays.asList; +import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableList; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -577,7 +578,7 @@ private static Response prepareRamIndex(Request request, searcher.setQueryCache(null); final long absoluteStartMillis = System.currentTimeMillis(); QueryShardContext context = - indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null); + indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null, emptyMap()); return handler.apply(context, indexReader.leaves().get(0)); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index e9a6ca60509e3..ca885cdfdff6a 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -32,6 +32,8 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; + /** * Test that needsScores() is reported correctly depending on whether _score is used */ @@ -45,7 +47,7 @@ public void testNeedsScores() { contexts.put(NumberSortScript.CONTEXT, Whitelist.BASE_WHITELISTS); PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts); - QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null); + QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null, emptyMap()); NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap()); NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index f88d2b4dea4c7..35e3e85d3561c 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -103,6 +103,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -535,7 +536,7 @@ public void testQueryWithRewrite() throws Exception { QueryShardContext shardContext = indexService.newQueryShardContext( randomInt(20), null, () -> { throw new UnsupportedOperationException(); - }, null); + }, null, emptyMap()); PlainActionFuture future = new PlainActionFuture<>(); Rewriteable.rewriteAndFetch(queryBuilder, shardContext, future); assertQueryBuilder(qbSource, future.get()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java index 7193a696b4706..4851c5fdf06c6 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -51,6 +51,7 @@ import java.util.Map; import java.util.function.Function; +import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -258,7 +259,7 @@ public void testRangeQueriesWithNow() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { long[] currentTime = new long[] {System.currentTimeMillis()}; QueryShardContext queryShardContext = - indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null); + indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null, emptyMap()); BytesReference source = BytesReference.bytes(jsonBuilder().startObject() .field("field1", "value") diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java index a64415d08b1b1..b2512e6a62cfd 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelperTests.java @@ -38,6 +38,7 @@ public void testSliceIntoSubRequests() throws IOException { () -> null, () -> null, () -> emptyList(), + () -> null, () -> null)); if (searchRequest.source() != null) { // Clear the slice builder if there is one set. We can't call sliceIntoSubRequests if it is. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml new file mode 100644 index 0000000000000..ada085ebb148e --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml @@ -0,0 +1,33 @@ +--- +"date_nanos requires dates after 1970 and before 2262": + + - do: + indices.create: + index: date_ns + body: + settings: + number_of_shards: 3 + number_of_replicas: 0 + mappings: + properties: + date: + type: date_nanos + field: + type: long + + - do: + bulk: + refresh: true + body: + - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }' + - '{"date" : "1969-10-28T12:12:12.123456789Z" }' + - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }' + - '{"date" : "2263-10-29T12:12:12.123456789Z" }' + + - match: { errors: true } + - match: { items.0.index.status: 400 } + - match: { items.0.index.error.type: mapper_parsing_exception } + - match: { items.0.index.error.caused_by.reason: "date[1969-10-28T12:12:12.123456789Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution" } + - match: { items.1.index.status: 400 } + - match: { items.1.index.error.type: mapper_parsing_exception } + - match: { items.1.index.error.caused_by.reason: "date[2263-10-29T12:12:12.123456789Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml index 7657dc2bebb36..6a4aece31b142 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/240_date_nanos.yml @@ -73,28 +73,6 @@ setup: - match: { hits.hits.1._id: "second" } - match: { hits.hits.1.sort: [1540815132987654321] } - ---- -"date_nanos requires dates after 1970 and before 2262": - - - do: - bulk: - refresh: true - body: - - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_1" } }' - - '{"date" : "1969-10-28T12:12:12.123456789Z" }' - - '{ "index" : { "_index" : "date_ns", "_id" : "date_ns_2" } }' - - '{"date" : "2263-10-29T12:12:12.123456789Z" }' - - - match: { errors: true } - - match: { items.0.index.status: 400 } - - match: { items.0.index.error.type: mapper_parsing_exception } - - match: { items.0.index.error.caused_by.reason: "date[1969-10-28T12:12:12.123456789Z] is before the epoch in 1970 and cannot be stored in nanosecond resolution" } - - match: { items.1.index.status: 400 } - - match: { items.1.index.error.type: mapper_parsing_exception } - - match: { items.1.index.error.caused_by.reason: "date[2263-10-29T12:12:12.123456789Z] is after 2262-04-11T23:47:16.854775807 and cannot be stored in nanosecond resolution" } - - --- "doc value fields are working as expected across date and date_nanos fields": diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 52f25d09672aa..d835a71b27ede 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -55,6 +55,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV1Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV2Templates; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; @@ -181,7 +182,7 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri resolvedAliases, tempClusterState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - tempIndexService.newQueryShardContext(0, null, () -> 0L, null))); + tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()))); Map aliasesByName = aliases.stream().collect( Collectors.toMap(AliasMetadata::getAlias, Function.identity())); diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 57082ed450941..9ee9bf947a46d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -89,7 +89,8 @@ public void run() { CollapseBuilder innerCollapseBuilder = innerHitBuilder.getInnerCollapseBuilder(); SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder, innerCollapseBuilder) .query(groupQuery) - .postFilter(searchRequest.source().postFilter()); + .postFilter(searchRequest.source().postFilter()) + .runtimeMappings(searchRequest.source().runtimeMappings()); SearchRequest groupRequest = new SearchRequest(searchRequest); groupRequest.source(sourceBuilder); multiRequest.add(groupRequest); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index fe1529c899952..e79c2b28c8e90 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster.metadata; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -100,6 +101,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; import static java.util.stream.Collectors.toList; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING; @@ -494,7 +496,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c MetadataIndexTemplateService.resolveAliases(templates), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), templates.stream().map(IndexTemplateMetadata::getName).collect(toList()), metadataTransformer); } @@ -527,7 +529,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null)), + xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), Collections.singletonList(templateName), metadataTransformer); } @@ -578,7 +580,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt currentState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - indexService.newQueryShardContext(0, null, () -> 0L, null)), + indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), org.elasticsearch.common.collect.List.of(), metadataTransformer); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index 55c9635470a13..3ced1267655f1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -47,6 +47,7 @@ import java.util.function.Function; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; /** @@ -149,7 +150,7 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable System.currentTimeMillis(), null), xContentRegistry); + () -> System.currentTimeMillis(), null, emptyMap()), xContentRegistry); } }; if (action.apply(newAliasValidator, metadata, index)) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index b6856d507bf77..e762a0decfb7a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -80,6 +80,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping; import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED; @@ -1092,7 +1093,7 @@ private static void validateCompositeTemplate(final ClusterState state, new AliasValidator(), // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null)); + xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())); // triggers inclusion of _timestamp field and its validation: String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index ebe9449f3a34f..93ec4686633c9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -193,7 +193,7 @@ public IndexService( assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, // we parse all percolator queries as they would be parsed on shard 0 - () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled, scriptService); + () -> newQueryShardContext(0, null, System::currentTimeMillis, null, emptyMap()), idFieldDataEnabled, scriptService); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. @@ -588,13 +588,19 @@ public IndexSettings getIndexSettings() { * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make * {@link IndexReader}-specific optimizations, such as rewriting containing range queries. */ - public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) { + public QueryShardContext newQueryShardContext( + int shardId, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Map runtimeMappings + ) { final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver); return new QueryShardContext( shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, - indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry); + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, runtimeMappings); } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 5748cc1808995..ee380cc7868a2 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; @@ -36,6 +37,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; @@ -72,6 +74,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -81,6 +84,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; +import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; /** @@ -120,7 +124,11 @@ public String[] getTypes() { private boolean mapUnmappedFieldAsString; private NestedScope nestedScope; private final ValuesSourceRegistry valuesSourceRegistry; + private final Map runtimeMappings; + /** + * Build a {@linkplain QueryShardContext} without any information from the search request. + */ public QueryShardContext(int shardId, IndexSettings indexSettings, BigArrays bigArrays, @@ -139,16 +147,63 @@ public QueryShardContext(int shardId, BooleanSupplier allowExpensiveQueries, ValuesSourceRegistry valuesSourceRegistry) { this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService, - scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher, - new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), - indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry); + scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, + indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, emptyMap()); + } + + /** + * Build a {@linkplain QueryShardContext} with information from the search request. + */ + public QueryShardContext( + int shardId, + IndexSettings indexSettings, + BigArrays bigArrays, + BitsetFilterCache bitsetFilterCache, + TriFunction, IndexFieldData> indexFieldDataLookup, + MapperService mapperService, + SimilarityService similarityService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + NamedWriteableRegistry namedWriteableRegistry, + Client client, + IndexSearcher searcher, + LongSupplier nowInMillis, + String clusterAlias, + Predicate indexNameMatcher, + BooleanSupplier allowExpensiveQueries, + ValuesSourceRegistry valuesSourceRegistry, + Map runtimeMappings + ) { + this( + shardId, + indexSettings, + bigArrays, + bitsetFilterCache, + indexFieldDataLookup, + mapperService, + similarityService, + scriptService, + xContentRegistry, + namedWriteableRegistry, + client, + searcher, + nowInMillis, + indexNameMatcher, + new Index( + RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()), + indexSettings.getIndex().getUUID() + ), + allowExpensiveQueries, + valuesSourceRegistry, + parseRuntimeMappings(runtimeMappings, mapperService, indexSettings) + ); } public QueryShardContext(QueryShardContext source) { this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(), source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.indexNameMatcher, - source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry); + source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry, source.runtimeMappings); } private QueryShardContext(int shardId, @@ -167,7 +222,8 @@ private QueryShardContext(int shardId, Predicate indexNameMatcher, Index fullyQualifiedIndex, BooleanSupplier allowExpensiveQueries, - ValuesSourceRegistry valuesSourceRegistry) { + ValuesSourceRegistry valuesSourceRegistry, + Map runtimeMappings) { super(xContentRegistry, namedWriteableRegistry, client, nowInMillis); this.shardId = shardId; this.similarityService = similarityService; @@ -184,6 +240,7 @@ private QueryShardContext(int shardId, this.fullyQualifiedIndex = fullyQualifiedIndex; this.allowExpensiveQueries = allowExpensiveQueries; this.valuesSourceRegistry = valuesSourceRegistry; + this.runtimeMappings = runtimeMappings; } private void reset() { @@ -260,7 +317,20 @@ public boolean hasMappings() { * type then the fields will be returned with a type prefix. */ public Set simpleMatchToIndexNames(String pattern) { - return mapperService.simpleMatchToFullName(pattern); + if (runtimeMappings.isEmpty()) { + return mapperService.simpleMatchToFullName(pattern); + } + if (Regex.isSimpleMatchPattern(pattern) == false) { + // no wildcards + return Collections.singleton(pattern); + } + Set matches = new HashSet<>(mapperService.simpleMatchToFullName(pattern)); + for (String name : runtimeMappings.keySet()) { + if (Regex.simpleMatch(pattern, name)) { + matches.add(name); + } + } + return matches; } /** @@ -273,14 +343,19 @@ public Set simpleMatchToIndexNames(String pattern) { * @see QueryShardContext#setMapUnmappedFieldAsString(boolean) */ public MappedFieldType getFieldType(String name) { - return failIfFieldMappingNotFound(name, mapperService.fieldType(name)); + return failIfFieldMappingNotFound(name, fieldType(name)); } /** * Returns true if the field identified by the provided name is mapped, false otherwise */ public boolean isFieldMapped(String name) { - return mapperService.fieldType(name) != null; + return fieldType(name) != null; + } + + private MappedFieldType fieldType(String name) { + MappedFieldType fieldType = runtimeMappings.get(name); + return fieldType == null ? mapperService.fieldType(name) : fieldType; } public ObjectMapper getObjectMapper(String name) { @@ -321,13 +396,22 @@ public MappedFieldType buildAnonymousFieldType(String type) { "[unmapped_type:string] should be replaced with [unmapped_type:keyword]"); type = "keyword"; } - final Mapper.TypeParser.ParserContext parserContext = mapperService.parserContext(); + return buildFieldType(type, "__anonymous_" + type, Collections.emptyMap(), mapperService.parserContext(), indexSettings); + } + + private static MappedFieldType buildFieldType( + String type, + String field, + Map node, + Mapper.TypeParser.ParserContext parserContext, + IndexSettings indexSettings + ) { Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new IllegalArgumentException("No mapper found for type [" + type + "]"); } - final Mapper.Builder builder = typeParser.parse("__anonymous_" + type, Collections.emptyMap(), parserContext); - Mapper mapper = builder.build(new ContentPath(1)); + Mapper.Builder builder = typeParser.parse(field, node, parserContext); + Mapper mapper = builder.build(new ContentPath(0)); if (mapper instanceof FieldMapper) { return ((FieldMapper)mapper).fieldType(); } @@ -589,4 +673,26 @@ public BigArrays bigArrays() { return bigArrays; } + private static Map parseRuntimeMappings( + Map mappings, + MapperService mapperService, + IndexSettings indexSettings + ) { + Map runtimeMappings = new HashMap<>(); + for (Map.Entry entry : mappings.entrySet()) { + String field = entry.getKey(); + if (entry.getValue() instanceof Map == false) { + throw new ElasticsearchParseException("runtime mappings must be a map type"); + } + @SuppressWarnings("unchecked") + Map node = new HashMap<>((Map) entry.getValue()); + // Replace the type until we have native support for the runtime section + Object oldRuntimeType = node.put("runtime_type", node.remove("type")); + if (oldRuntimeType != null) { + throw new ElasticsearchParseException("use [type] in [runtime_mappings] instead of [runtime_type]"); + } + runtimeMappings.put(field, buildFieldType("runtime", field, node, mapperService.parserContext(), indexSettings)); + } + return runtimeMappings; + } } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 1f689f00ed13f..b2db754d61ad1 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -180,8 +180,13 @@ final class DefaultSearchContext extends SearchContext { this.relativeTimeSupplier = relativeTimeSupplier; this.timeout = timeout; this.minNodeVersion = minNodeVersion; - queryShardContext = indexService.newQueryShardContext(request.shardId().id(), this.searcher, - request::nowInMillis, shardTarget.getClusterAlias()); + queryShardContext = indexService.newQueryShardContext( + request.shardId().id(), + this.searcher, + request::nowInMillis, + shardTarget.getClusterAlias(), + request.getRuntimeMappings() + ); queryShardContext.setTypes(request.types()); queryBoost = request.indexBoost(); this.lowLevelCancellation = lowLevelCancellation; diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 7bc6a17db45bf..85b1b3b764f97 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1176,7 +1176,7 @@ private CanMatchResponse canMatch(ShardSearchRequest request, boolean checkRefre try (Releasable ignored2 = canMatchSearcher) { QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher, - request::nowInMillis, request.getClusterAlias()); + request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); Rewriteable.rewrite(request.getRewriteable(), context, false); final boolean aliasFilterCanMatch = request.getAliasFilter() .getQueryBuilder() instanceof MatchNoneQueryBuilder == false; diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 755d26d0fd3d1..0835b55defa53 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -64,9 +64,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_ACCURATE; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; @@ -114,6 +116,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R public static final ParseField COLLAPSE = new ParseField("collapse"); public static final ParseField SLICE = new ParseField("slice"); public static final ParseField POINT_IN_TIME = new ParseField("pit"); + public static final ParseField RUNTIME_MAPPINGS_FIELD = new ParseField("runtime_mappings"); public static SearchSourceBuilder fromXContent(XContentParser parser) throws IOException { return fromXContent(parser, true); @@ -194,6 +197,8 @@ public static HighlightBuilder highlight() { private PointInTimeBuilder pointInTimeBuilder = null; + private Map runtimeMappings = emptyMap(); + /** * Constructs a new search source builder. */ @@ -275,6 +280,9 @@ public SearchSourceBuilder(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_7_10_0)) { pointInTimeBuilder = in.readOptionalWriteable(PointInTimeBuilder::new); } + if (in.getVersion().onOrAfter(Version.V_7_11_0)) { + runtimeMappings = in.readMap(); + } } @Override @@ -350,6 +358,15 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_10_0)) { out.writeOptionalWriteable(pointInTimeBuilder); } + if (out.getVersion().onOrAfter(Version.V_7_11_0)) { + out.writeMap(runtimeMappings); + } else { + if (false == runtimeMappings.isEmpty()) { + throw new IllegalArgumentException( + "Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + out.getVersion() + "]" + ); + } + } } /** @@ -1009,6 +1026,21 @@ public SearchSourceBuilder pointInTimeBuilder(PointInTimeBuilder builder) { return this; } + /** + * Mappings specified on this search request that override built in mappings. + */ + public Map runtimeMappings() { + return runtimeMappings; + } + + /** + * Specify the mappings specified on this search request that override built in mappings. + */ + public SearchSourceBuilder runtimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings == null ? emptyMap() : runtimeMappings; + return this; + } + /** * Rewrites this search source builder into its primitive form. e.g. by * rewriting the QueryBuilder. If the builder did not change the identity @@ -1093,6 +1125,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; rewrittenBuilder.collapse = collapse; rewrittenBuilder.pointInTimeBuilder = pointInTimeBuilder; + rewrittenBuilder.runtimeMappings = runtimeMappings; return rewrittenBuilder; } @@ -1216,6 +1249,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th collapse = CollapseBuilder.fromXContent(parser); } else if (POINT_IN_TIME.match(currentFieldName, parser.getDeprecationHandler())) { pointInTimeBuilder = PointInTimeBuilder.fromXContent(parser); + } else if (RUNTIME_MAPPINGS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + runtimeMappings = parser.map(); } else { throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].", parser.getTokenLocation()); @@ -1423,6 +1458,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (pointInTimeBuilder != null) { pointInTimeBuilder.toXContent(builder, params); } + if (false == runtimeMappings.isEmpty()) { + builder.field(RUNTIME_MAPPINGS_FIELD.getPreferredName(), runtimeMappings); + } + return builder; } @@ -1635,7 +1674,7 @@ public int hashCode() { return Objects.hash(aggregations, explain, fetchSourceContext, fetchFields, docValueFields, storedFieldsContext, from, highlightBuilder, indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, - seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo, pointInTimeBuilder); + seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo, pointInTimeBuilder, runtimeMappings); } @Override @@ -1676,7 +1715,8 @@ public boolean equals(Object obj) { && Objects.equals(extBuilders, other.extBuilders) && Objects.equals(collapse, other.collapse) && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo) - && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder); + && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder) + && Objects.equals(runtimeMappings, other.runtimeMappings); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index 680dd6177ab8f..d7e2a4e01aee7 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -47,8 +47,8 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.AliasFilterParsingException; import org.elasticsearch.indices.InvalidAliasNameException; -import org.elasticsearch.search.SearchSortValuesAndFormats; import org.elasticsearch.search.Scroll; +import org.elasticsearch.search.SearchSortValuesAndFormats; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.sort.FieldSortBuilder; @@ -60,6 +60,7 @@ import java.util.Map; import java.util.function.Function; +import static java.util.Collections.emptyMap; import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; /** @@ -540,4 +541,8 @@ public static QueryBuilder parseAliasFilter(CheckedFunction getRuntimeMappings() { + return source == null ? emptyMap() : source.runtimeMappings(); + } } diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 38def06729d0e..ca6e52e0f7106 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.AbstractSearchTestCase; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -39,10 +40,14 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.IntStream; +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.equalTo; + public class ExpandSearchPhaseTests extends ESTestCase { public void testCollapseSingleHit() throws IOException { @@ -59,6 +64,7 @@ public void testCollapseSingleHit() throws IOException { AtomicBoolean executedMultiSearch = new AtomicBoolean(false); QueryBuilder originalQuery = randomBoolean() ? null : QueryBuilders.termQuery("foo", "bar"); + Map runtimeMappings = randomBoolean() ? emptyMap() : AbstractSearchTestCase.randomRuntimeMappings(); final MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); String collapseValue = randomBoolean() ? null : "boom"; @@ -67,7 +73,7 @@ public void testCollapseSingleHit() throws IOException { .collapse(new CollapseBuilder("someField") .setInnerHits(IntStream.range(0, numInnerHits).mapToObj(hitNum -> new InnerHitBuilder().setName("innerHit" + hitNum)) .collect(Collectors.toList())))); - mockSearchPhaseContext.getRequest().source().query(originalQuery); + mockSearchPhaseContext.getRequest().source().query(originalQuery).runtimeMappings(runtimeMappings); mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) { @Override void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener listener) { @@ -87,7 +93,7 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL } assertArrayEquals(mockSearchPhaseContext.getRequest().indices(), searchRequest.indices()); assertArrayEquals(mockSearchPhaseContext.getRequest().types(), searchRequest.types()); - + assertThat(searchRequest.source().runtimeMappings(), equalTo(runtimeMappings)); List mSearchResponses = new ArrayList<>(numInnerHits); for (int innerHitNum = 0; innerHitNum < numInnerHits; innerHitNum++) { diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java index d1cb6795fbb19..cbf231b611004 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java @@ -86,6 +86,10 @@ public void testSerialization() throws Exception { public void testRandomVersionSerialization() throws IOException { SearchRequest searchRequest = createSearchRequest(); Version version = VersionUtils.randomVersion(random()); + if (version.before(Version.V_7_11_0) && searchRequest.source() != null) { + // Versions before 7.11.0 don't support runtime mappings + searchRequest.source().runtimeMappings(emptyMap()); + } SearchRequest deserializedRequest = copyWriteable(searchRequest, namedWriteableRegistry, SearchRequest::new, version); if (version.before(Version.V_7_0_0)) { assertTrue(deserializedRequest.isCcsMinimizeRoundtrips()); diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 3725bb7b87a77..b6ac781a015d7 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -59,6 +59,7 @@ import java.util.Collection; import java.util.List; +import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; @@ -139,7 +140,7 @@ public void setup() throws Exception { writer = new IndexWriter( new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()) ); - shardContext = indexService.newQueryShardContext(0, null, () -> 0, null); + shardContext = indexService.newQueryShardContext(0, null, () -> 0, null, emptyMap()); } protected final List refreshReader() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 863ff56472c67..eaed67f424abb 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -50,14 +50,21 @@ import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.Mapper.TypeParser; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; @@ -184,9 +191,24 @@ public void testIndexSortedOnField() { IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); QueryShardContext context = new QueryShardContext( - 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, - null, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> 0L, null, null, () -> true, null); + 0, + indexSettings, + BigArrays.NON_RECYCLING_INSTANCE, + null, + null, + null, + null, + null, + NamedXContentRegistry.EMPTY, + new NamedWriteableRegistry(Collections.emptyList()), + null, + null, + () -> 0L, + null, + null, + () -> true, + null + ); assertTrue(context.indexSortedOnField("sort_field")); assertFalse(context.indexSortedOnField("second_sort_field")); @@ -290,12 +312,86 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { assertEquals(Arrays.asList(expectedFirstDoc.toString(), expectedSecondDoc.toString()), collect("field", queryShardContext)); } + public void testRuntimeFields() throws IOException { + MapperService mapperService = mockMapperService("test", org.elasticsearch.common.collect.List.of(new MapperPlugin() { + @Override + public Map getMappers() { + return org.elasticsearch.common.collect.Map.of("runtime", (name, node, parserContext) -> new Mapper.Builder(name) { + @Override + public Mapper build(ContentPath path) { + return new DummyMapper(name, new DummyMappedFieldType(name)); + } + }); + } + })); + when(mapperService.fieldType("pig")).thenReturn(new DummyMappedFieldType("pig")); + when(mapperService.simpleMatchToFullName("*")).thenReturn(org.elasticsearch.common.collect.Set.of("pig")); + /* + * Making these immutable here test that we don't modify them. + * Modifying them would cause all kinds of problems if two + * shards are parsed on the same node. + */ + Map runtimeMappings = org.elasticsearch.common.collect.Map.ofEntries( + org.elasticsearch.common.collect.Map.entry("cat", org.elasticsearch.common.collect.Map.of("type", "keyword")), + org.elasticsearch.common.collect.Map.entry("dog", org.elasticsearch.common.collect.Map.of("type", "keyword")) + ); + QueryShardContext qsc = new QueryShardContext( + 0, + mapperService.getIndexSettings(), + BigArrays.NON_RECYCLING_INSTANCE, + null, + (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), + mapperService, + null, + null, + NamedXContentRegistry.EMPTY, + new NamedWriteableRegistry(org.elasticsearch.common.collect.List.of()), + null, + null, + () -> 0, + "test", + null, + () -> true, + null, + runtimeMappings + ); + assertTrue(qsc.isFieldMapped("cat")); + assertThat(qsc.getFieldType("cat"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("cat"), equalTo(org.elasticsearch.common.collect.Set.of("cat"))); + assertTrue(qsc.isFieldMapped("dog")); + assertThat(qsc.getFieldType("dog"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("dog"), equalTo(org.elasticsearch.common.collect.Set.of("dog"))); + assertTrue(qsc.isFieldMapped("pig")); + assertThat(qsc.getFieldType("pig"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.simpleMatchToIndexNames("pig"), equalTo(org.elasticsearch.common.collect.Set.of("pig"))); + assertThat(qsc.simpleMatchToIndexNames("*"), equalTo(org.elasticsearch.common.collect.Set.of("cat", "dog", "pig"))); + } + public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) { return createQueryShardContext(indexUuid, clusterAlias, null); } - private static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias, - TriFunction runtimeDocValues) { + private static QueryShardContext createQueryShardContext( + String indexUuid, + String clusterAlias, + TriFunction runtimeDocValues + ) { + MapperService mapperService = mockMapperService(indexUuid, org.elasticsearch.common.collect.List.of()); + if (runtimeDocValues != null) { + when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { + String fieldName = (String)fieldTypeInv.getArguments()[0]; + return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); + }); + } + final long nowInMillis = randomNonNegativeLong(); + return new QueryShardContext( + 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, + (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), + mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), + null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); + } + + private static MapperService mockMapperService(String indexUuid, List mapperPlugins) { IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder("index"); indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT) .put("index.number_of_shards", 1) @@ -303,34 +399,24 @@ private static QueryShardContext createQueryShardContext(String indexUuid, Strin .put(IndexMetadata.SETTING_INDEX_UUID, indexUuid) ); IndexMetadata indexMetadata = indexMetadataBuilder.build(); - IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); IndexAnalyzers indexAnalyzers = new IndexAnalyzers( Collections.singletonMap("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, null)), Collections.emptyMap(), Collections.emptyMap() ); + IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); + MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.index()).thenReturn(indexMetadata.getIndex()); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); - Map typeParserMap = IndicesModule.getMappers(Collections.emptyList()); + Map typeParserMap = IndicesModule.getMappers(mapperPlugins); Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(name -> null, typeParserMap::get, Version.CURRENT, () -> null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); }); when(mapperService.parserContext()).thenReturn(parserContext); - if (runtimeDocValues != null) { - when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { - String fieldName = (String)fieldTypeInv.getArguments()[0]; - return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); - }); - } - final long nowInMillis = randomNonNegativeLong(); - return new QueryShardContext( - 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, - (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), - mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); + return mapperService; } private static MappedFieldType mockFieldType(String fieldName, BiFunction runtimeDocValues) { @@ -428,4 +514,45 @@ public void collect(int doc) throws IOException { } } + private static class DummyMapper extends FieldMapper { + protected DummyMapper(String simpleName, MappedFieldType mappedFieldType) { + super(simpleName, mappedFieldType, org.elasticsearch.common.collect.Map.of(), MultiFields.empty(), CopyTo.empty()); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Builder getMergeBuilder() { + throw new UnsupportedOperationException(); + } + + @Override + protected String contentType() { + throw new UnsupportedOperationException(); + } + } + + private static class DummyMappedFieldType extends MappedFieldType { + DummyMappedFieldType(String name) { + super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, null); + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + + @Override + public String typeName() { + return "runtime"; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new UnsupportedOperationException(); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index f8b0ab4abf4b7..8410636583024 100644 --- a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -58,6 +58,7 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery; import static org.hamcrest.Matchers.equalTo; @@ -99,7 +100,7 @@ public void setup() throws IOException { public void testCrossFieldMultiMatchQuery() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); queryShardContext.setAllowUnmappedFields(true); for (float tieBreaker : new float[] {0.0f, 0.5f}) { Query parsedQuery = multiMatchQuery("banon") @@ -126,8 +127,13 @@ public void testBlendTerms() { float[] boosts = new float[] {2, 3}; Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), null, 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + null, + 1f, + false, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } @@ -140,8 +146,13 @@ public void testBlendTermsWithFieldBoosts() { float[] boosts = new float[] {200, 30}; Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), null, 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + null, + 1f, + false, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } @@ -160,8 +171,13 @@ public Query termQuery(Object value, QueryShardContext context) { BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f) ), 1f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), null, 1f, true, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + null, + 1f, + true, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } @@ -173,7 +189,7 @@ public Query termQuery(Object value, QueryShardContext context) { } }; expectThrows(IllegalArgumentException.class, () -> MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), new BytesRef("baz"), null, 1f, false, Arrays.asList(new FieldAndBoost(ft, 1)))); } @@ -195,14 +211,19 @@ public Query termQuery(Object value, QueryShardContext context) { expectedDisjunct1 ), 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null), - new BytesRef("baz"), null, 1f, false, Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3))); + indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + new BytesRef("baz"), + null, + 1f, + false, + Arrays.asList(new FieldAndBoost(ft1, 2), new FieldAndBoost(ft2, 3)) + ); assertEquals(expected, actual); } public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); @@ -234,7 +255,7 @@ public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { public void testMultiMatchCrossFieldsWithSynonymsPhrase() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); Map fieldNames = new HashMap<>(); @@ -303,7 +324,7 @@ public void testKeywordSplitQueriesOnWhitespace() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( randomInt(20), null, () -> { throw new UnsupportedOperationException(); - }, null); + }, null, emptyMap()); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); Map fieldNames = new HashMap<>(); fieldNames.put("field", 1.0f); diff --git a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java index c4ec440eb2e80..4cd75d8b04bea 100644 --- a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java @@ -43,6 +43,8 @@ import java.io.IOException; import java.util.Collections; +import static java.util.Collections.emptyMap; + public class NestedHelperTests extends ESSingleNodeTestCase { IndexService indexService; @@ -333,7 +335,7 @@ public void testConjunction() { } public void testNested() throws IOException { - QueryShardContext context = indexService.newQueryShardContext(0, new IndexSearcher(new MultiReader()), () -> 0, null); + QueryShardContext context = indexService.newQueryShardContext(0, new IndexSearcher(new MultiReader()), () -> 0, null, emptyMap()); NestedQueryBuilder queryBuilder = new NestedQueryBuilder("nested1", new MatchAllQueryBuilder(), ScoreMode.Avg); ESToParentBlockJoinQuery query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context); diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java index 2480192cdf7d8..1c354050ab0b6 100644 --- a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java @@ -74,6 +74,7 @@ import java.util.Collections; import java.util.List; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.mapper.SeqNoFieldMapper.PRIMARY_TERM_NAME; import static org.hamcrest.Matchers.equalTo; @@ -613,7 +614,7 @@ public void testMultiLevelNestedSorting() throws IOException { DirectoryReader reader = DirectoryReader.open(writer); reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId(indexService.index(), 0)); IndexSearcher searcher = new IndexSearcher(reader); - QueryShardContext queryShardContext = indexService.newQueryShardContext(0, searcher, () -> 0L, null); + QueryShardContext queryShardContext = indexService.newQueryShardContext(0, searcher, () -> 0L, null, emptyMap()); FieldSortBuilder sortBuilder = new FieldSortBuilder("chapters.paragraphs.word_count"); sortBuilder.setNestedSort(new NestedSortBuilder("chapters").setNestedSort(new NestedSortBuilder("chapters.paragraphs"))); diff --git a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java index 2144ce057a53a..56abe168c3137 100644 --- a/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java @@ -92,7 +92,22 @@ protected SearchSourceBuilder createSearchSourceBuilder() { SuggestBuilderTests::randomSuggestBuilder, QueryRescorerBuilderTests::randomRescoreBuilder, randomExtBuilders, - CollapseBuilderTests::randomCollapseBuilder); + CollapseBuilderTests::randomCollapseBuilder, + AbstractSearchTestCase::randomRuntimeMappings); + } + + public static Map randomRuntimeMappings() { + int count = between(1, 100); + Map runtimeFields = new HashMap<>(count); + while (runtimeFields.size() < count) { + int size = between(1, 10); + Map config = new HashMap<>(size); + while (config.size() < size) { + config.put(randomAlphaOfLength(5), randomAlphaOfLength(5)); + } + runtimeFields.put(randomAlphaOfLength(5), config); + } + return runtimeFields; } protected SearchRequest createSearchRequest() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index 182740da8d57b..3992b04f40e73 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -109,7 +109,9 @@ public void testPreProcess() throws Exception { when(indexCache.query()).thenReturn(queryCache); when(indexService.cache()).thenReturn(indexCache); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString())).thenReturn(queryShardContext); + when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString(), anyObject())).thenReturn( + queryShardContext + ); MapperService mapperService = mock(MapperService.class); when(mapperService.hasNested()).thenReturn(randomBoolean()); when(indexService.mapperService()).thenReturn(mapperService); @@ -254,7 +256,9 @@ public void testClearQueryCancellationsOnClose() throws IOException { IndexService indexService = mock(IndexService.class); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString())).thenReturn(queryShardContext); + when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString(), anyObject())).thenReturn( + queryShardContext + ); BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 0879157180149..f88da07130530 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -20,11 +20,10 @@ package org.elasticsearch.search.builder; import com.fasterxml.jackson.core.JsonParseException; + +import org.elasticsearch.Version; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -93,16 +92,18 @@ private static void assertParseSearchSource(SearchSourceBuilder testBuilder, XCo } public void testSerialization() throws IOException { - SearchSourceBuilder testBuilder = createSearchSourceBuilder(); - try (BytesStreamOutput output = new BytesStreamOutput()) { - testBuilder.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { - SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in); - assertEquals(deserializedBuilder, testBuilder); - assertEquals(deserializedBuilder.hashCode(), testBuilder.hashCode()); - assertNotSame(deserializedBuilder, testBuilder); - } - } + SearchSourceBuilder original = createSearchSourceBuilder(); + SearchSourceBuilder copy = copyBuilder(original); + assertEquals(copy, original); + assertEquals(copy.hashCode(), original.hashCode()); + assertNotSame(copy, original); + } + + public void testSerializingWithRuntimeFieldsBeforeSupportedThrows() { + SearchSourceBuilder original = new SearchSourceBuilder().runtimeMappings(randomRuntimeMappings()); + Version v = Version.V_7_11_0.minimumCompatibilityVersion(); + Exception e = expectThrows(IllegalArgumentException.class, () -> copyBuilder(original, v)); + assertThat(e.getMessage(), equalTo("Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + v + "]")); } public void testShallowCopy() { @@ -118,9 +119,12 @@ public void testEqualsAndHashcode() throws IOException { EqualsHashCodeTestUtils.checkEqualsAndHashCode(createSearchSourceBuilder(), this::copyBuilder); } - //we use the streaming infra to create a copy of the builder provided as argument private SearchSourceBuilder copyBuilder(SearchSourceBuilder original) throws IOException { - return ESTestCase.copyWriteable(original, namedWriteableRegistry, SearchSourceBuilder::new); + return copyBuilder(original, Version.CURRENT); + } + + private SearchSourceBuilder copyBuilder(SearchSourceBuilder original, Version version) throws IOException { + return ESTestCase.copyWriteable(original, namedWriteableRegistry, SearchSourceBuilder::new, version); } public void testParseIncludeExclude() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index a30e4332a34e0..bc879cd59ec73 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Supplier; import static java.util.Collections.emptyMap; @@ -82,7 +83,7 @@ private RandomSearchRequestGenerator() {} * Build a random search request. * * @param randomSearchSourceBuilder builds a random {@link SearchSourceBuilder}. You can use - * {@link #randomSearchSourceBuilder(Supplier, Supplier, Supplier, Supplier, Supplier)}. + * {@link #randomSearchSourceBuilder}. */ public static SearchRequest randomSearchRequest(Supplier randomSearchSourceBuilder) { SearchRequest searchRequest = new SearchRequest(); @@ -125,7 +126,8 @@ public static SearchSourceBuilder randomSearchSourceBuilder( Supplier randomSuggestBuilder, Supplier> randomRescoreBuilder, Supplier> randomExtBuilders, - Supplier randomCollapseBuilder) { + Supplier randomCollapseBuilder, + Supplier> randomRuntimeMappings) { SearchSourceBuilder builder = new SearchSourceBuilder(); if (randomBoolean()) { builder.from(randomIntBetween(0, 10000)); @@ -381,6 +383,9 @@ public static SearchSourceBuilder randomSearchSourceBuilder( } builder.pointInTimeBuilder(pit); } + if (randomBoolean()) { + builder.runtimeMappings(randomRuntimeMappings.get()); + } return builder; } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 9193e90bdbaf4..a99a4f0b6b077 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -63,6 +63,8 @@ import java.util.List; import java.util.Map; +import static java.util.Collections.emptyMap; + public class TestSearchContext extends SearchContext { public static final SearchShardTarget SHARD_TARGET = new SearchShardTarget("test", new ShardId("test", "test", 0), null, OriginalIndices.NONE); @@ -97,7 +99,7 @@ public TestSearchContext(BigArrays bigArrays, IndexService indexService) { this.indexService = indexService; this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache(); this.indexShard = indexService.getShardOrNull(0); - queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L, null); + queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); } public TestSearchContext(QueryShardContext queryShardContext) { diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 0ee9994ebd531..9eca0ce4cc24d 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -69,8 +69,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.Map; import java.util.Set; +import static java.util.Collections.emptyMap; + /** * This is an internal action, that executes msearch requests for enrich indices in a more efficient manner. * Currently each search request inside a msearch request is executed as a separate search. If many search requests @@ -231,11 +234,20 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t final IndexShard indexShard = indicesService.getShardOrNull(shardId); try (Engine.Searcher searcher = indexShard.acquireSearcher("enrich_msearch")) { final FieldsVisitor visitor = new FieldsVisitor(true); + /* + * Enrich doesn't support defining runtime fields in its + * configuration. We could add support for that if we'd + * like it but, for now at least, you can't configure any + * runtime fields so it is safe to build the context without + * any. + */ + Map runtimeFields = emptyMap(); final QueryShardContext context = indexService.newQueryShardContext( shardId.id(), searcher, () -> { throw new UnsupportedOperationException(); }, - null + null, + runtimeFields ); final String type = context.getType(); final Text typeText = new Text(type); diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle index 7dc01b73ed9ec..61e9797913849 100644 --- a/x-pack/plugin/runtime-fields/qa/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -1 +1,74 @@ -// Empty project so we can pick up its subproject +// Shared infratructure + +apply plugin: 'elasticsearch.build' + +dependencies { + api project(":test:framework") +} + +test.enabled = false // We don't currently have any tests for this because they are test utilities. + +subprojects { + if (project.name.startsWith('core-with-')) { + apply plugin: 'elasticsearch.yaml-rest-test' + + dependencies { + yamlRestTestImplementation xpackProject("plugin:runtime-fields:qa") + } + + restResources { + restApi { + includeXpack 'async_search', 'graph', '*_point_in_time' + } + restTests { + includeCore '*' + includeXpack 'async_search', 'graph' + } + } + + testClusters.yamlRestTest { + testDistribution = 'DEFAULT' + setting 'xpack.license.self_generated.type', 'trial' + } + + yamlRestTest { + def suites = [ + 'async_search', + 'search', + 'search.aggregation', + 'search.highlight', + 'search.inner_hits', + 'search_shards', + 'suggest', + ] + if (project.name.equals('core-with-mapped')) { + suites += [ + // These two don't support runtime fields on the request. Should they? + 'field_caps', + 'graph', + // The search request tests don't know how to support msearch for now + 'msearch', + ] + } + systemProperty 'tests.rest.suite', suites.join(',') + systemProperty 'tests.rest.blacklist', + [ + /////// TO FIX /////// + 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? + 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. + 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? + 'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should? + 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', + /////// TO FIX /////// + + /////// NOT SUPPORTED /////// + 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it + // Runtime fields don't have global ords + 'search.aggregation/20_terms/string profiler via global ordinals', + 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', + 'search.aggregation/170_cardinality_metric/profiler string', + /////// NOT SUPPORTED /////// + ].join(',') + } + } +} \ No newline at end of file diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle b/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle new file mode 100644 index 0000000000000..ea347a8a55e7a --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/build.gradle @@ -0,0 +1 @@ +// Configured by parent project diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java new file mode 100644 index 0000000000000..0c19187c088d0 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.test.mapped; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; + +import java.util.HashMap; +import java.util.Map; + +/** + * Runs elasticsearch's core rest tests replacing all field mappings with runtime fields + * that load from {@code _source}. Tests that configure the field in a way that are not + * supported by runtime fields are skipped. + */ +public class CoreWithMappedRuntimeFieldsIT extends ESClientYamlSuiteTestCase { + public CoreWithMappedRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return new MappingRuntimeFieldTranslater().parameters(); + } + + private static class MappingRuntimeFieldTranslater extends CoreTestTranslater { + @Override + protected Map dynamicTemplateFor(String type) { + return dynamicTemplateToAddRuntimeFields(type); + } + + @Override + protected Suite suite(ClientYamlTestCandidate candidate) { + return new Suite(candidate) { + @Override + protected boolean modifyMappingProperties(String index, Map properties) { + Map newProperties = new HashMap<>(properties.size()); + Map> runtimeProperties = new HashMap<>(properties.size()); + if (false == runtimeifyMappingProperties(properties, newProperties, runtimeProperties)) { + return false; + } + for (Map.Entry> runtimeProperty : runtimeProperties.entrySet()) { + runtimeProperty.getValue().put("runtime_type", runtimeProperty.getValue().get("type")); + runtimeProperty.getValue().put("type", "runtime"); + newProperties.put(runtimeProperty.getKey(), runtimeProperty.getValue()); + } + properties.clear(); + properties.putAll(newProperties); + return true; + } + + @Override + protected boolean modifySearch(ApiCallSection search) { + // We don't need to modify the search request if the mappings are in the index + return true; + } + + @Override + protected boolean handleIndex(IndexRequest index) { + // We don't need to scrape anything out of the index requests. + return true; + } + }; + } + } +} diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle new file mode 100644 index 0000000000000..ea347a8a55e7a --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/build.gradle @@ -0,0 +1 @@ +// Configured by parent project diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java new file mode 100644 index 0000000000000..02092699ecc02 --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -0,0 +1,225 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.test.search; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; +import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static java.util.Collections.unmodifiableMap; + +/** + * Runs elasticsearch's core rest tests disabling all mappings and replacing them + * with runtime fields defined on the search request that load from {@code _source}. Tests + * that configure the field in a way that are not supported by runtime fields are skipped. + */ +public class CoreTestsWithSearchRuntimeFieldsIT extends ESClientYamlSuiteTestCase { + public CoreTestsWithSearchRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return new SearchRequestRuntimeFieldTranslater().parameters(); + } + + /** + * Translating the tests is fairly difficult here because instead of ES + * tracking the mappings we have to track them. We don't have to do it as + * well as ES, just well enough that we can decorate the search requests + * with types that make most tests "just work". + */ + private static class SearchRequestRuntimeFieldTranslater extends CoreTestTranslater { + @Override + protected Map dynamicTemplateFor(String type) { + return dynamicTemplateToDisableRuntimeCompatibleFields(type); + } + + @Override + protected Suite suite(ClientYamlTestCandidate candidate) { + return new Suite(candidate) { + private Map>> runtimeMappingsAfterSetup; + private Map> mappedFieldsAfterSetup; + private Map>> runtimeMappings; + private Map> mappedFields; + + @Override + public boolean modifySections(List executables) { + if (runtimeMappingsAfterSetup == null) { + // We're modifying the setup section + runtimeMappings = new HashMap<>(); + mappedFields = new HashMap<>(); + if (false == super.modifySections(executables)) { + return false; + } + runtimeMappingsAfterSetup = unmodifiableMap(runtimeMappings); + runtimeMappings = null; + mappedFieldsAfterSetup = unmodifiableMap(mappedFields); + mappedFields = null; + return true; + } + runtimeMappings = new HashMap<>(runtimeMappingsAfterSetup); + mappedFields = new HashMap<>(mappedFieldsAfterSetup); + return super.modifySections(executables); + } + + @Override + protected boolean modifyMappingProperties(String index, Map properties) { + Map untouchedMapping = new HashMap<>(); + Map> runtimeMapping = new HashMap<>(); + if (false == runtimeifyMappingProperties(properties, untouchedMapping, runtimeMapping)) { + return false; + } + properties.clear(); + properties.putAll(untouchedMapping); + mappedFields.put(index, untouchedMapping.keySet()); + runtimeMappings.put(index, runtimeMapping); + return true; + } + + @Override + protected boolean modifySearch(ApiCallSection search) { + if (search.getBodies().isEmpty()) { + search.addBody(new HashMap<>()); + } + for (Map body : search.getBodies()) { + Map runtimeMapping = runtimeMappings(search.getParams().get("index")); + if (runtimeMapping == null) { + return false; + } + body.put("runtime_mappings", runtimeMapping); + } + return true; + } + + private Map runtimeMappings(String index) { + if (index == null) { + return mergeMappings(new String[] { "*" }); + } + String[] patterns = Arrays.stream(index.split(",")).map(m -> m.equals("_all") ? "*" : m).toArray(String[]::new); + if (patterns.length == 0 && Regex.isSimpleMatchPattern(patterns[0])) { + return runtimeMappings.get(patterns[0]); + } + return mergeMappings(patterns); + } + + private Map mergeMappings(String[] patterns) { + Map> merged = new HashMap<>(); + for (Map.Entry>> indexEntry : runtimeMappings.entrySet()) { + if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { + continue; + } + for (Map.Entry> field : indexEntry.getValue().entrySet()) { + Map mergedConfig = merged.get(field.getKey()); + if (mergedConfig == null) { + merged.put(field.getKey(), field.getValue()); + } else if (false == mergedConfig.equals(field.getValue())) { + // The two indices have different runtime mappings for a field so we have to give up on running the test. + return null; + } + } + } + for (Map.Entry> indexEntry : mappedFields.entrySet()) { + if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { + continue; + } + for (String mappedField : indexEntry.getValue()) { + if (merged.containsKey(mappedField)) { + // We have a runtime mappings for a field *and* regular mapping. We can't make this test work so skip it. + return null; + } + } + } + return merged; + } + + @Override + protected boolean handleIndex(IndexRequest index) { + /* + * Ok! Let's reverse engineer dynamic mapping. Sort of. We're + * really just looking to figure out which of the runtime fields + * is "close enough" to what dynamic mapping would do. + */ + if (index.getPipeline() != null) { + // We can't attempt local dynamic mappings with pipelines + return false; + } + Map map = XContentHelper.convertToMap(index.source(), false, index.getContentType()).v2(); + Map> indexRuntimeMappings = runtimeMappings.computeIfAbsent( + index.index(), + i -> new HashMap<>() + ); + Set indexMappedfields = mappedFields.computeIfAbsent(index.index(), i -> Set.of()); + for (Map.Entry e : map.entrySet()) { + String name = e.getKey(); + if (indexRuntimeMappings.containsKey(name) || indexMappedfields.contains(name)) { + continue; + } + Object value = e.getValue(); + if (value == null) { + continue; + } + if (value instanceof Boolean) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "boolean")); + continue; + } + if (value instanceof Long || value instanceof Integer) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "long")); + continue; + } + if (value instanceof Double) { + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "double")); + continue; + } + if (false == value instanceof String) { + continue; + } + try { + Long.parseLong(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "long")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + try { + Double.parseDouble(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "double")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + try { + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(value.toString()); + indexRuntimeMappings.put(name, runtimeFieldLoadingFromSource(name, "date")); + continue; + } catch (IllegalArgumentException iae) { + // Try the next one + } + // Strings are funny, the regular dynamic mapping puts them in "name.keyword" so we follow along. + indexRuntimeMappings.put(name + ".keyword", runtimeFieldLoadingFromSource(name, "keyword")); + } + return true; + } + }; + } + } +} diff --git a/x-pack/plugin/runtime-fields/qa/rest/build.gradle b/x-pack/plugin/runtime-fields/qa/rest/build.gradle deleted file mode 100644 index 03bc35ae2a856..0000000000000 --- a/x-pack/plugin/runtime-fields/qa/rest/build.gradle +++ /dev/null @@ -1,54 +0,0 @@ -apply plugin: 'elasticsearch.yaml-rest-test' - -restResources { - restApi { - includeXpack 'async_search', 'graph', '*_point_in_time' - } - restTests { - includeCore '*' - includeXpack 'async_search', 'graph' - } -} - -testClusters.yamlRestTest { - testDistribution = 'DEFAULT' - setting 'xpack.license.self_generated.type', 'trial' -} - -yamlRestTest { - systemProperty 'tests.rest.suite', - [ - 'async_search', - 'field_caps', - 'graph', - 'msearch', - 'search', - 'search.aggregation', - 'search.highlight', - 'search.inner_hits', - 'search_shards', - 'suggest', - ].join(',') - systemProperty 'tests.rest.blacklist', - [ - /////// TO FIX /////// - 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing? - 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. - 'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do? - 'search.aggregation/10_histogram/*', // runtime doesn't support sub-fields. Maybe it should? - 'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit', - /////// TO FIX /////// - - /////// NOT SUPPORTED /////// - 'search.aggregation/280_rare_terms/*', // Requires an index and we won't have it - // Runtime fields don't have global ords - 'search.aggregation/20_terms/string profiler via global ordinals', - 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', - 'search.aggregation/170_cardinality_metric/profiler string', - //dynamic template causes a type _doc to be created, these tests use another type but only one type is allowed - 'search.aggregation/51_filter_with_types/*', - 'search/171_terms_query_with_types/*', - 'msearch/12_basic_with_types/*' - /////// NOT SUPPORTED /////// - ].join(',') -} diff --git a/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java deleted file mode 100644 index 3f46930bd6c5a..0000000000000 --- a/x-pack/plugin/runtime-fields/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/rest/CoreTestsWithRuntimeFieldsIT.java +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields.rest; - -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.index.mapper.BooleanFieldMapper; -import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.IpFieldMapper; -import org.elasticsearch.index.mapper.KeywordFieldMapper; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; -import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; -import org.elasticsearch.test.rest.yaml.section.DoSection; -import org.elasticsearch.test.rest.yaml.section.ExecutableSection; -import org.elasticsearch.test.rest.yaml.section.SetupSection; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.hamcrest.Matchers.equalTo; - -public class CoreTestsWithRuntimeFieldsIT extends ESClientYamlSuiteTestCase { - public CoreTestsWithRuntimeFieldsIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { - super(testCandidate); - } - - /** - * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, - * replacing the body of index creation commands so that fields are {@code runtime}s - * that load from {@code source} instead of their original type. Test configurations that - * do are not modified to contain runtime fields are not returned as they are tested - * elsewhere. - */ - @ParametersFactory - public static Iterable parameters() throws Exception { - Map suites = new HashMap<>(); - List result = new ArrayList<>(); - for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { - assert orig.length == 1; - ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; - ClientYamlTestSuite suite = suites.computeIfAbsent(candidate.getTestPath(), k -> modifiedSuite(candidate)); - if (suite == null) { - // The setup section contains an unsupported option - continue; - } - if (false == modifySection(candidate.getTestSection().getExecutableSections())) { - // The test section contains an unsupported option - continue; - } - ClientYamlTestSection modified = new ClientYamlTestSection( - candidate.getTestSection().getLocation(), - candidate.getTestSection().getName(), - candidate.getTestSection().getSkipSection(), - candidate.getTestSection().getExecutableSections() - ); - result.add(new Object[] { new ClientYamlTestCandidate(suite, modified) }); - } - return result; - } - - /** - * Modify the setup section to setup a dynamic template that replaces - * field configurations with scripts that load from source - * and replaces field configurations in {@code incides.create} - * with scripts that load from source. - */ - private static ClientYamlTestSuite modifiedSuite(ClientYamlTestCandidate candidate) { - if (false == modifySection(candidate.getSetupSection().getExecutableSections())) { - return null; - } - List setup = new ArrayList<>(candidate.getSetupSection().getExecutableSections().size() + 1); - setup.add(ADD_TEMPLATE); - setup.addAll(candidate.getSetupSection().getExecutableSections()); - return new ClientYamlTestSuite( - candidate.getApi(), - candidate.getName(), - new SetupSection(candidate.getSetupSection().getSkipSection(), setup), - candidate.getTeardownSection(), - Collections.emptyList() - ); - } - - /** - * Replace field configuration in {@code indices.create} with scripts - * that load from the source. - */ - private static boolean modifySection(List executables) { - for (ExecutableSection section : executables) { - if (false == (section instanceof DoSection)) { - continue; - } - DoSection doSection = (DoSection) section; - if (false == doSection.getApiCallSection().getApi().equals("indices.create")) { - continue; - } - for (Map body : doSection.getApiCallSection().getBodies()) { - Object settings = body.get("settings"); - if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { - /* - * You can't sort the index on a runtime_keyword and it is - * hard to figure out if the sort was a runtime_keyword so - * let's just skip this test. - */ - continue; - } - Object mappings = body.get("mappings"); - if (false == (mappings instanceof Map)) { - continue; - } - Object properties = ((Map) mappings).get("properties"); - if (false == (properties instanceof Map)) { - continue; - } - for (Map.Entry property : ((Map) properties).entrySet()) { - if (false == property.getValue() instanceof Map) { - continue; - } - @SuppressWarnings("unchecked") - Map propertyMap = (Map) property.getValue(); - String name = property.getKey().toString(); - String type = Objects.toString(propertyMap.get("type")); - if ("nested".equals(type)) { - // Our loading scripts can't be made to manage nested fields so we have to skip those tests. - return false; - } - if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { - // If doc_values is false we can't emulate with scripts. `null` and `true` are fine. - continue; - } - if ("false".equals(Objects.toString(propertyMap.get("index")))) { - // If index is false we can't emulate with scripts - continue; - } - if ("true".equals(Objects.toString(propertyMap.get("store")))) { - // If store is true we can't emulate with scripts - continue; - } - if (propertyMap.containsKey("ignore_above")) { - // Scripts don't support ignore_above so we skip those fields - continue; - } - if (propertyMap.containsKey("ignore_malformed")) { - // Our source reading script doesn't emulate ignore_malformed - continue; - } - String toLoad = painlessToLoadFromSource(name, type); - if (toLoad == null) { - continue; - } - propertyMap.put("type", "runtime"); - propertyMap.put("runtime_type", type); - propertyMap.put("script", toLoad); - propertyMap.remove("store"); - propertyMap.remove("index"); - propertyMap.remove("doc_values"); - } - } - } - return true; - } - - private static String painlessToLoadFromSource(String name, String type) { - String emit = PAINLESS_TO_EMIT.get(type); - if (emit == null) { - return null; - } - StringBuilder b = new StringBuilder(); - b.append("def v = params._source['").append(name).append("'];\n"); - b.append("if (v instanceof Iterable) {\n"); - b.append(" for (def vv : ((Iterable) v)) {\n"); - b.append(" if (vv != null) {\n"); - b.append(" def value = vv;\n"); - b.append(" ").append(emit).append("\n"); - b.append(" }\n"); - b.append(" }\n"); - b.append("} else {\n"); - b.append(" if (v != null) {\n"); - b.append(" def value = v;\n"); - b.append(" ").append(emit).append("\n"); - b.append(" }\n"); - b.append("}\n"); - return b.toString(); - } - - private static final Map PAINLESS_TO_EMIT = org.elasticsearch.common.collect.Map.of( - BooleanFieldMapper.CONTENT_TYPE, - "emit(Boolean.parseBoolean(value.toString()));", - DateFieldMapper.CONTENT_TYPE, - "emit(parse(value.toString()));", - NumberType.DOUBLE.typeName(), - "emit(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));", - KeywordFieldMapper.CONTENT_TYPE, - "emit(value.toString());", - IpFieldMapper.CONTENT_TYPE, - "emit(value.toString());", - NumberType.LONG.typeName(), - "emit(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" - ); - - private static final ExecutableSection ADD_TEMPLATE = new ExecutableSection() { - @Override - public XContentLocation getLocation() { - return new XContentLocation(-1, -1); - } - - @Override - public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { - Map params = org.elasticsearch.common.collect.Map.of("name", "convert_to_source_only", "create", "true"); - List> dynamicTemplates = new ArrayList<>(); - for (String type : PAINLESS_TO_EMIT.keySet()) { - if (type.equals("ip")) { - // There isn't a dynamic template to pick up ips. They'll just look like strings. - continue; - } - Map mapping = org.elasticsearch.common.collect.Map.of( - "type", - "runtime", - "runtime_type", - type, - "script", - painlessToLoadFromSource("{name}", type) - ); - if (type.contentEquals("keyword")) { - /* - * For "string"-type dynamic mappings emulate our default - * behavior with a top level text field and a `.keyword` - * multi-field. But instead of the default, use a runtime - * field for the multi-field. - */ - mapping = org.elasticsearch.common.collect.Map.of( - "type", - "text", - "fields", - org.elasticsearch.common.collect.Map.of("keyword", mapping) - ); - dynamicTemplates.add( - org.elasticsearch.common.collect.Map.of( - type, - org.elasticsearch.common.collect.Map.of("match_mapping_type", "string", "mapping", mapping) - ) - ); - } else { - dynamicTemplates.add( - org.elasticsearch.common.collect.Map.of( - type, - org.elasticsearch.common.collect.Map.of("match_mapping_type", type, "mapping", mapping) - ) - ); - } - } - - List> bodies = Collections.singletonList( - org.elasticsearch.common.collect.Map.of( - "index_patterns", - "*", - "priority", - Integer.MAX_VALUE - 1, - "template", - org.elasticsearch.common.collect.Map.of( - "settings", - Collections.emptyMap(), - "mappings", - org.elasticsearch.common.collect.Map.of("dynamic_templates", dynamicTemplates) - ) - ) - ); - ClientYamlTestResponse response = executionContext.callApi( - "indices.put_index_template", - params, - bodies, - Collections.emptyMap() - ); - - assertThat(response.getStatusCode(), equalTo(200)); - // There are probably some warning about overlapping templates. Ignore them. - } - }; -} diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java new file mode 100644 index 0000000000000..b034b572c7e6e --- /dev/null +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -0,0 +1,432 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.runtimefields.test; + +import org.elasticsearch.action.bulk.BulkRequestParser; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; +import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; +import org.elasticsearch.test.rest.yaml.section.DoSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; +import org.elasticsearch.test.rest.yaml.section.SetupSection; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; + +/** + * Builds test parameters similarly to {@link ESClientYamlSuiteTestCase#createParameters()}, + * replacing all fields with runtime fields that load from {@code _source} if possible. Tests + * that configure the field in a way that are not supported by runtime fields are skipped. + */ +public abstract class CoreTestTranslater { + public Iterable parameters() throws Exception { + Map suites = new HashMap<>(); + List result = new ArrayList<>(); + for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { + assert orig.length == 1; + ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; + Suite suite = suites.computeIfAbsent(candidate.getSuitePath(), k -> suite(candidate)); + if (suite.modified == null) { + // The setup section contains an unsupported option + continue; + } + if (false == suite.modifySections(candidate.getTestSection().getExecutableSections())) { + // The test section contains an unsupported option + continue; + } + ClientYamlTestSection modified = new ClientYamlTestSection( + candidate.getTestSection().getLocation(), + candidate.getTestSection().getName(), + candidate.getTestSection().getSkipSection(), + candidate.getTestSection().getExecutableSections() + ); + result.add(new Object[] { new ClientYamlTestCandidate(suite.modified, modified) }); + } + return result; + } + + protected abstract Suite suite(ClientYamlTestCandidate candidate); + + private static String painlessToLoadFromSource(String name, String type) { + String emit = PAINLESS_TO_EMIT.get(type); + if (emit == null) { + return null; + } + StringBuilder b = new StringBuilder(); + b.append("def v = params._source['").append(name).append("'];\n"); + b.append("if (v instanceof Iterable) {\n"); + b.append(" for (def vv : ((Iterable) v)) {\n"); + b.append(" if (vv != null) {\n"); + b.append(" def value = vv;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append(" }\n"); + b.append("} else {\n"); + b.append(" if (v != null) {\n"); + b.append(" def value = v;\n"); + b.append(" ").append(emit).append("\n"); + b.append(" }\n"); + b.append("}\n"); + return b.toString(); + } + + private static final Map PAINLESS_TO_EMIT = org.elasticsearch.common.collect.Map.ofEntries( + org.elasticsearch.common.collect.Map.entry(BooleanFieldMapper.CONTENT_TYPE, "emit(Boolean.parseBoolean(value.toString()));"), + org.elasticsearch.common.collect.Map.entry(DateFieldMapper.CONTENT_TYPE, "emit(parse(value.toString()));"), + org.elasticsearch.common.collect.Map.entry( + NumberType.DOUBLE.typeName(), + "emit(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));" + ), + org.elasticsearch.common.collect.Map.entry(KeywordFieldMapper.CONTENT_TYPE, "emit(value.toString());"), + org.elasticsearch.common.collect.Map.entry(IpFieldMapper.CONTENT_TYPE, "emit(value.toString());"), + org.elasticsearch.common.collect.Map.entry( + NumberType.LONG.typeName(), + "emit(value instanceof Number ? ((Number) value).longValue() : Long.parseLong(value.toString()));" + ) + ); + + protected abstract Map dynamicTemplateFor(String type); + + protected static Map dynamicTemplateToDisableRuntimeCompatibleFields(String type) { + return org.elasticsearch.common.collect.Map.of("type", type, "index", false, "doc_values", false); + } + + protected static Map dynamicTemplateToAddRuntimeFields(String type) { + return org.elasticsearch.common.collect.Map.ofEntries( + org.elasticsearch.common.collect.Map.entry("type", "runtime"), + org.elasticsearch.common.collect.Map.entry("runtime_type", type), + org.elasticsearch.common.collect.Map.entry("script", painlessToLoadFromSource("{name}", type)) + ); + } + + protected static Map runtimeFieldLoadingFromSource(String name, String type) { + return org.elasticsearch.common.collect.Map.of("type", type, "script", painlessToLoadFromSource(name, type)); + } + + private ExecutableSection addIndexTemplate() { + return new ExecutableSection() { + @Override + public XContentLocation getLocation() { + return new XContentLocation(-1, -1); + } + + @Override + public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { + Map params = org.elasticsearch.common.collect.Map.of("name", "hack_dynamic_mappings", "create", "true"); + List> dynamicTemplates = new ArrayList<>(); + for (String type : PAINLESS_TO_EMIT.keySet()) { + if (type.equals("ip")) { + // There isn't a dynamic template to pick up ips. They'll just look like strings. + continue; + } + Map mapping = dynamicTemplateFor(type); + if (type.equals("keyword")) { + /* + * For "string"-type dynamic mappings emulate our default + * behavior with a top level text field and a `.keyword` + * multi-field. In our case we disable the keyword field + * and substitute it with an enabled one on the search + * request. + */ + mapping = org.elasticsearch.common.collect.Map.of( + "type", + "text", + "fields", + org.elasticsearch.common.collect.Map.of("keyword", mapping) + ); + dynamicTemplates.add( + org.elasticsearch.common.collect.Map.of( + type, + org.elasticsearch.common.collect.Map.of("match_mapping_type", "string", "mapping", mapping) + ) + ); + } else { + dynamicTemplates.add( + org.elasticsearch.common.collect.Map.of( + type, + org.elasticsearch.common.collect.Map.of("match_mapping_type", type, "mapping", mapping) + ) + ); + } + } + Map indexTemplate = org.elasticsearch.common.collect.Map.of( + "settings", + org.elasticsearch.common.collect.Map.of(), + "mappings", + org.elasticsearch.common.collect.Map.of("dynamic_templates", dynamicTemplates) + ); + List> bodies = org.elasticsearch.common.collect.List.of( + org.elasticsearch.common.collect.Map.ofEntries( + org.elasticsearch.common.collect.Map.entry("index_patterns", "*"), + org.elasticsearch.common.collect.Map.entry("priority", Integer.MAX_VALUE - 1), + org.elasticsearch.common.collect.Map.entry("template", indexTemplate) + ) + ); + ClientYamlTestResponse response = executionContext.callApi( + "indices.put_index_template", + params, + bodies, + org.elasticsearch.common.collect.Map.of() + ); + assertThat(response.getStatusCode(), equalTo(200)); + // There are probably some warning about overlapping templates. Ignore them. + } + }; + } + + /** + * A modified suite. + */ + protected abstract class Suite { + private final ClientYamlTestSuite modified; + + public Suite(ClientYamlTestCandidate candidate) { + if (false == modifySections(candidate.getSetupSection().getExecutableSections())) { + modified = null; + return; + } + /* + * Modify the setup section to rewrite and create index commands and + * to add a dynamic template that sets up any dynamic indices how we + * expect them. + */ + List setup = new ArrayList<>(candidate.getSetupSection().getExecutableSections().size() + 1); + setup.add(addIndexTemplate()); + setup.addAll(candidate.getSetupSection().getExecutableSections()); + modified = new ClientYamlTestSuite( + candidate.getApi(), + candidate.getName(), + new SetupSection(candidate.getSetupSection().getSkipSection(), setup), + candidate.getTeardownSection(), + org.elasticsearch.common.collect.List.of() + ); + } + + /** + * Replace field configuration in {@code indices.create} with scripts + * that load from the source. + * + * @return true if the section is appropriate for testing with runtime fields + */ + public boolean modifySections(List executables) { + for (ExecutableSection section : executables) { + if (false == (section instanceof DoSection)) { + continue; + } + DoSection doSection = (DoSection) section; + String api = doSection.getApiCallSection().getApi(); + switch (api) { + case "indices.create": + if (false == modifyCreateIndex(doSection.getApiCallSection())) { + return false; + } + break; + case "search": + case "async_search.submit": + if (false == modifySearch(doSection.getApiCallSection())) { + return false; + } + break; + case "bulk": + if (false == handleBulk(doSection.getApiCallSection())) { + return false; + } + break; + case "index": + if (false == handleIndex(doSection.getApiCallSection())) { + return false; + } + break; + default: + continue; + } + } + return true; + } + + /** + * Modify a test search request. + */ + protected abstract boolean modifySearch(ApiCallSection search); + + private boolean modifyCreateIndex(ApiCallSection createIndex) { + String index = createIndex.getParams().get("index"); + for (Map body : createIndex.getBodies()) { + Object settings = body.get("settings"); + if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { + /* + * You can't sort the index on a runtime_keyword and it is + * hard to figure out if the sort was a runtime_keyword so + * let's just skip this test. + */ + continue; + } + Object mapping = body.get("mappings"); + if (false == (mapping instanceof Map)) { + continue; + } + Object properties = ((Map) mapping).get("properties"); + if (false == (properties instanceof Map)) { + continue; + } + @SuppressWarnings("unchecked") + Map propertiesMap = (Map) properties; + if (false == modifyMappingProperties(index, propertiesMap)) { + return false; + } + } + return true; + } + + /** + * Modify the mapping defined in the test. + */ + protected abstract boolean modifyMappingProperties(String index, Map properties); + + /** + * Modify the provided map in place, translating all fields into + * runtime fields that load from source. + * @return true if this mapping supports runtime fields, false otherwise + */ + protected final boolean runtimeifyMappingProperties( + Map properties, + Map untouchedProperties, + Map> runtimeProperties + ) { + for (Map.Entry property : properties.entrySet()) { + if (false == property.getValue() instanceof Map) { + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + @SuppressWarnings("unchecked") + Map propertyMap = (Map) property.getValue(); + String name = property.getKey().toString(); + String type = Objects.toString(propertyMap.get("type")); + if ("nested".equals(type)) { + // Our loading scripts can't be made to manage nested fields so we have to skip those tests. + return false; + } + if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { + // If doc_values is false we can't emulate with scripts. So we keep the old definition. `null` and `true` are fine. + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if ("false".equals(Objects.toString(propertyMap.get("index")))) { + // If index is false we can't emulate with scripts + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if ("true".equals(Objects.toString(propertyMap.get("store")))) { + // If store is true we can't emulate with scripts + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if (propertyMap.containsKey("ignore_above")) { + // Scripts don't support ignore_above so we skip those fields + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + if (propertyMap.containsKey("ignore_malformed")) { + // Our source reading script doesn't emulate ignore_malformed + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + String toLoad = painlessToLoadFromSource(name, type); + if (toLoad == null) { + untouchedProperties.put(property.getKey(), property.getValue()); + continue; + } + Map runtimeConfig = new HashMap<>(propertyMap); + runtimeConfig.put("script", toLoad); + runtimeConfig.remove("store"); + runtimeConfig.remove("index"); + runtimeConfig.remove("doc_values"); + runtimeProperties.put(property.getKey(), runtimeConfig); + } + /* + * Its tempting to return false here if we didn't make any runtime + * fields, skipping the test. But that would cause us to skip any + * test uses dynamic mappings. Disaster! Instead we use a dynamic + * template to make the dynamic mappings into runtime fields too. + * The downside is that we can run tests that don't use runtime + * fields at all. That's unfortunate, but its ok. + */ + return true; + } + + private boolean handleBulk(ApiCallSection bulk) { + String defaultIndex = bulk.getParams().get("index"); + String defaultRouting = bulk.getParams().get("routing"); + String defaultPipeline = bulk.getParams().get("pipeline"); + BytesStreamOutput bos = new BytesStreamOutput(); + try { + for (Map body : bulk.getBodies()) { + try (XContentBuilder b = new XContentBuilder(JsonXContent.jsonXContent, bos)) { + b.map(body); + } + bos.write(JsonXContent.jsonXContent.streamSeparator()); + } + List indexRequests = new ArrayList<>(); + new BulkRequestParser(false).parse( + bos.bytes(), + defaultIndex, + defaultRouting, + null, + defaultPipeline, + null, + true, + XContentType.JSON, + index -> indexRequests.add(index), + u -> {}, + d -> {} + ); + for (IndexRequest index : indexRequests) { + if (false == handleIndex(index)) { + return false; + } + } + } catch (IOException e) { + throw new AssertionError(e); + } + return true; + } + + private boolean handleIndex(ApiCallSection indexRequest) { + String index = indexRequest.getParams().get("index"); + String pipeline = indexRequest.getParams().get("pipeline"); + assert indexRequest.getBodies().size() == 1; + try { + return handleIndex(new IndexRequest(index).setPipeline(pipeline).source(indexRequest.getBodies().get(0))); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + protected abstract boolean handleIndex(IndexRequest index) throws IOException; + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index ab71ddb06190d..bc50c6b6d1170 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -286,6 +286,7 @@ import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_FORMAT_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.API_KEY_SERVICE_ENABLED_SETTING; @@ -765,7 +766,10 @@ public void onIndexModule(IndexModule module) { () -> { throw new IllegalArgumentException("permission filters are not allowed to use the current timestamp"); - }, null), + }, + null, + // Don't use runtime mappings in the security query + emptyMap()), dlsBitsetCache.get(), securityContext.get(), getLicenseState(), diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index 840a0906dc1db..733c2f0b50190 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -360,3 +360,110 @@ setup: - match: { aggregations.to-users.users.hits.hits.2._index: test } - match: { aggregations.to-users.users.hits.hits.2._nested.field: users } - match: { aggregations.to-users.users.hits.hits.2._nested.offset: 1 } + +--- +"fetch defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage_rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + fields: [voltage_rating] + sort: timestamp + - match: {hits.total.value: 6} + - match: {hits.hits.0._source.voltage: 4.0} + - match: {hits.hits.0.fields.voltage_rating: [low]} + +--- +"match defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage.rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + query: + match: + voltage.rating: ok + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 5.1} + +--- +"search glob defined on search request": + - do: + search: + index: sensor + body: + runtime_mappings: + voltage.rating: + type: keyword + script: | + double v = doc['voltage'].value; + if (v < 4.8) { + emit('low'); + } else if (v > 5.2) { + emit('high'); + } else { + emit('ok'); + } + query: + simple_query_string: + fields: [voltage.*] + query: ok + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 5.1} + + +--- +"replace object field on search request": + - do: + bulk: + index: student + refresh: true + body: | + {"index":{}} + {"name": {"first": "Andrew", "last": "Wiggin"}} + {"index":{}} + {"name": {"first": "Julian", "last": "Delphiki", "suffix": "II"}} + + - do: + search: + index: student + body: + runtime_mappings: + name.first: + type: keyword + script: | + if ('Wiggin'.equals(doc['name.last.keyword'].value)) { + emit('Ender'); + } else if ('Delphiki'.equals(doc['name.last.keyword'].value)) { + emit('Bean'); + } + query: + match: + name.first: Bean + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.name.first: Julian} From 9edcfc44d08b2f4c016a11b84d9c67c4edf75a69 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 10 Nov 2020 14:15:55 -0500 Subject: [PATCH 2/5] Fixup --- .../search/builder/SearchSourceBuilder.java | 2 +- .../search/builder/SearchSourceBuilderTests.java | 9 +++++++-- .../test/search/CoreTestsWithSearchRuntimeFieldsIT.java | 5 ++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 0835b55defa53..5f66552afdf19 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -363,7 +363,7 @@ public void writeTo(StreamOutput out) throws IOException { } else { if (false == runtimeMappings.isEmpty()) { throw new IllegalArgumentException( - "Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + out.getVersion() + "]" + "Versions before 7.11.0 don't support [runtime_mappings] and search was sent to [" + out.getVersion() + "]" ); } } diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index f88da07130530..7e724bed19770 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.util.Map; @@ -101,9 +102,13 @@ public void testSerialization() throws IOException { public void testSerializingWithRuntimeFieldsBeforeSupportedThrows() { SearchSourceBuilder original = new SearchSourceBuilder().runtimeMappings(randomRuntimeMappings()); - Version v = Version.V_7_11_0.minimumCompatibilityVersion(); + Version v = VersionUtils.randomVersionBetween( + random(), + Version.V_7_11_0.minimumCompatibilityVersion(), + VersionUtils.getPreviousVersion(Version.V_7_11_0) + ); Exception e = expectThrows(IllegalArgumentException.class, () -> copyBuilder(original, v)); - assertThat(e.getMessage(), equalTo("Versions before 8.0.0 don't support [runtime_mappings] and search was sent to [" + v + "]")); + assertThat(e.getMessage(), equalTo("Versions before 7.11.0 don't support [runtime_mappings] and search was sent to [" + v + "]")); } public void testShallowCopy() { diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index 02092699ecc02..69325a5729dd0 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -168,7 +168,10 @@ protected boolean handleIndex(IndexRequest index) { index.index(), i -> new HashMap<>() ); - Set indexMappedfields = mappedFields.computeIfAbsent(index.index(), i -> Set.of()); + Set indexMappedfields = mappedFields.computeIfAbsent( + index.index(), + i -> org.elasticsearch.common.collect.Set.of() + ); for (Map.Entry e : map.entrySet()) { String name = e.getKey(); if (indexRuntimeMappings.containsKey(name) || indexMappedfields.contains(name)) { From 161797e61667a5b499ed7ce70a6c651f1d6797dc Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 10 Nov 2020 15:05:49 -0500 Subject: [PATCH 3/5] drop broken --- .../search/internal/ShardSearchRequestTests.java | 4 ++++ x-pack/plugin/runtime-fields/qa/build.gradle | 2 ++ 2 files changed, 6 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java index b0dd5adde4316..5e105b9a02f32 100644 --- a/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/search/internal/ShardSearchRequestTests.java @@ -47,6 +47,7 @@ import java.io.IOException; import java.io.InputStream; +import static java.util.Collections.emptyMap; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.hamcrest.Matchers.containsString; @@ -76,6 +77,9 @@ public void testClone() throws Exception { public void testAllowPartialResultsSerializationPre7_0_0() throws IOException { Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, VersionUtils.getPreviousVersion(Version.V_7_0_0)); ShardSearchRequest shardSearchTransportRequest = createShardSearchRequest(); + if(shardSearchTransportRequest.source() != null) { + shardSearchTransportRequest.source().runtimeMappings(emptyMap()); + } ShardSearchRequest deserializedRequest = copyWriteable(shardSearchTransportRequest, namedWriteableRegistry, ShardSearchRequest::new, version); if (version.before(Version.V_6_3_0)) { diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle index 61e9797913849..68193426b68ef 100644 --- a/x-pack/plugin/runtime-fields/qa/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -67,6 +67,8 @@ subprojects { 'search.aggregation/20_terms/string profiler via global ordinals', 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', 'search.aggregation/170_cardinality_metric/profiler string', + 'search/171_terms_query_with_types/*', + 'msearch/12_basic_with_types/*' /////// NOT SUPPORTED /////// ].join(',') } From 7a6d7b21733150f9ded8559cf8eb7b842c7e9fd6 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 10 Nov 2020 15:13:03 -0500 Subject: [PATCH 4/5] Skip other tests --- x-pack/plugin/runtime-fields/qa/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle index 68193426b68ef..aed46ee888998 100644 --- a/x-pack/plugin/runtime-fields/qa/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -67,6 +67,8 @@ subprojects { 'search.aggregation/20_terms/string profiler via global ordinals', 'search.aggregation/20_terms/Global ordinals are loaded with the global_ordinals execution hint', 'search.aggregation/170_cardinality_metric/profiler string', + //dynamic template causes a type _doc to be created, these tests use another type but only one type is allowed + 'search.aggregation/51_filter_with_types/*', 'search/171_terms_query_with_types/*', 'msearch/12_basic_with_types/*' /////// NOT SUPPORTED /////// From fdb85449f61dc9ff6ece2d718b4caee9307126f8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 10 Nov 2020 17:53:33 -0500 Subject: [PATCH 5/5] Add missing skip --- .../main/resources/rest-api-spec/test/index/80_date_nanos.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml index ada085ebb148e..fe9adb7e42b03 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/80_date_nanos.yml @@ -1,5 +1,8 @@ --- "date_nanos requires dates after 1970 and before 2262": + - skip: + version: " - 6.99.99" + reason: "Implemented in 7.0" - do: indices.create: