From ddf615272b91d4bf99a38f0217195c52728e298c Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 26 Mar 2021 16:24:16 +0100 Subject: [PATCH 01/35] transformation working --- .../compat/RestCompatTestTransformTask.java | 5 + .../rest/transform/RestTestTransformer.java | 9 +- .../rest/transform/match/ReplaceTextual.java | 76 +++++++++++ .../transform/match/ReplaceTextualTests.java | 121 ++++++++++++++++++ .../rest/transform/match/is_true.yml | 110 ++++++++++++++++ rest-api-spec/build.gradle | 12 +- .../admin/indices/get/GetIndexResponse.java | 22 +++- .../mapping/get/GetMappingsResponse.java | 4 + .../elasticsearch/rest/BaseRestHandler.java | 21 ++- .../admin/indices/RestCreateIndexAction.java | 65 +++++++++- .../admin/indices/RestGetIndicesAction.java | 27 ++++ .../indices/RestCreateIndexActionTests.java | 69 ++++++++++ .../org/elasticsearch/test/ESTestCase.java | 4 + 13 files changed, 532 insertions(+), 13 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java create mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java create mode 100644 buildSrc/src/test/resources/rest/transform/match/is_true.yml diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 26cf4664d3ac7..ab7bf3ae218c2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -23,6 +23,7 @@ import org.elasticsearch.gradle.test.rest.transform.headers.InjectHeaders; import org.elasticsearch.gradle.test.rest.transform.match.AddMatch; import org.elasticsearch.gradle.test.rest.transform.match.RemoveMatch; +import org.elasticsearch.gradle.test.rest.transform.match.ReplaceTextual; import org.elasticsearch.gradle.test.rest.transform.match.ReplaceMatch; import org.elasticsearch.gradle.test.rest.transform.warnings.InjectAllowedWarnings; import org.elasticsearch.gradle.test.rest.transform.warnings.InjectWarnings; @@ -101,6 +102,10 @@ public void replaceMatch(String subKey, Object value) { transformations.add(new ReplaceMatch(subKey, MAPPER.convertValue(value, JsonNode.class))); } + public void replaceIsTrue(String subKey, Object value) { + transformations.add(new ReplaceTextual(subKey, MAPPER.convertValue(value, JsonNode.class))); + } + /** * Replaces the values of a match assertion for the given REST test. For example "match":{"_type": "foo"} to "match":{"_type": "bar"} * diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java index 5f276d75257dc..2003397acd8af 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java @@ -144,9 +144,16 @@ private void traverseTest( } else { if (entry.getValue().isObject()) { ObjectNode child = (ObjectNode) entry.getValue(); - if (child.has(transform.requiredChildKey())) { +// if (child.has(transform.requiredChildKey())) { + if(transform.matches(child)) { transform.transformTest((ObjectNode) currentNode); } + } else if(entry.getValue().isTextual()){ + String fieldValue = entry.getValue().asText(); + if(transform.matches(fieldValue)) {///fieldValue.equals(transform.requiredChildKey())){ + transform.transformTest((ObjectNode) currentNode); + } + } } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java new file mode 100644 index 0000000000000..86a06d4df9158 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.test.rest.transform.match; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.test.rest.transform.RestTestContext; +import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentObject; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Optional; + +/** + * A transformation to replace the flat fields + */ +public class ReplaceTextual implements RestTestTransformByParentObject { + private final String replaceKey; + private final JsonNode replacementNode; + private final String testName; + + public ReplaceTextual(String replaceKey, JsonNode replacementNode) { + this.replaceKey = replaceKey; + this.replacementNode = replacementNode; + this.testName = null; + } + + public ReplaceTextual(String replaceKey, JsonNode replacementNode, String testName) { + this.replaceKey = replaceKey; + this.replacementNode = replacementNode; + this.testName = testName; + } + + @Override + @Internal + public String getKeyToFind() { + return "is_true"; + } + + @Override + public String requiredChildKey() { + return replaceKey; + } + + @Override + public boolean shouldApply(RestTestContext testContext) { + return testName == null || testContext.getTestName().equals(testName); + } + + @Override + public void transformTest(ObjectNode matchParent) { +// ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); + matchParent.set(getKeyToFind(), replacementNode); + } + + @Input + public String getReplaceKey() { + return replaceKey; + } + + @Input + public JsonNode getReplacementNode() { + return replacementNode; + } + + @Input + @Optional + public String getTestName() { + return testName; + } +} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java new file mode 100644 index 0000000000000..07aa62adc82dc --- /dev/null +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.test.rest.transform.match; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.elasticsearch.gradle.test.rest.transform.TransformTests; +import org.hamcrest.CoreMatchers; +import org.junit.Test; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +public class ReplaceTextualTests extends TransformTests { + + private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); + private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY); + + @Test + public void testReplaceAll() throws Exception { + String testName = "/rest/transform/match/is_true.yml"; + List tests = getTests(testName); + JsonNode replacementNode = MAPPER.convertValue("test_index.mappings._doc", JsonNode.class); + validateTest(tests, true, true); + List transformedTests = transformTests( + new LinkedList<>(tests), + Collections.singletonList(new ReplaceTextual("test_index.mappings.type_1", replacementNode, null)) + ); + printTest(testName, transformedTests); + validateTest(tests, false, true); + + } + + private void validateTest(List tests, boolean beforeTransformation, boolean allTests) { + validateSetupAndTearDownForMatchTests(tests); + // first test + JsonNode firstTestChild = tests.get(2).get("First test"); + assertThat(firstTestChild, CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode firstTestParentArray = (ArrayNode) firstTestChild; + + AtomicBoolean firstTestHasMatchObject = new AtomicBoolean(false); + AtomicBoolean firstTestHasTypeMatch = new AtomicBoolean(false); + + firstTestParentArray.elements().forEachRemaining(node -> { + assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode childObject = (ObjectNode) node; + JsonNode matchObject = childObject.get("is_true"); + if (matchObject != null) { + firstTestHasMatchObject.set(true); + if (firstTestHasTypeMatch.get() == false ) { + firstTestHasTypeMatch.set(true); + }/*&& matchObject.asText().equals("test_index.mappings.type_1")*/ + if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { + assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); + } + } + }); + assertTrue(firstTestHasMatchObject.get()); + assertTrue(firstTestHasTypeMatch.get()); + + // last test + JsonNode lastTestChild = tests.get(tests.size() - 1).get("Last test"); + assertThat(lastTestChild, CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode lastTestParentArray = (ArrayNode) lastTestChild; + + AtomicBoolean lastTestHasMatchObject = new AtomicBoolean(false); + AtomicBoolean lastTestHasTypeMatch = new AtomicBoolean(false); + lastTestParentArray.elements().forEachRemaining(node -> { + assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode childObject = (ObjectNode) node; + JsonNode matchObject = childObject.get("is_true"); + if (matchObject != null) { + lastTestHasMatchObject.set(true); + if (lastTestHasTypeMatch.get() == false ) { + lastTestHasTypeMatch.set(true); + }/*&& matchObject.asText().equals("test_index.mappings.type_1")*/ + if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { + assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); + } + } + }); + assertTrue(lastTestHasMatchObject.get()); + assertTrue(lastTestHasTypeMatch.get()); + +// // exclude setup, teardown, first test, and last test +// for (int i = 3; i <= tests.size() - 2; i++) { +// ObjectNode otherTest = tests.get(i); +// JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); +// assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); +// ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; +// otherTestParentArray.elements().forEachRemaining(node -> { +// assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); +// ObjectNode childObject = (ObjectNode) node; +// JsonNode matchObject = childObject.get("match"); +// if (matchObject != null) { +// if (matchObject.get("_type") != null) { +// if (matchObject.get("_type") != null && beforeTransformation == false && allTests) { +// assertThat(matchObject.get("_type").asText(), CoreMatchers.is("_replaced_type")); +// } +// } +// } +// }); +// } + } + + @Override + protected boolean getHumanDebug() { + return true; + } +} diff --git a/buildSrc/src/test/resources/rest/transform/match/is_true.yml b/buildSrc/src/test/resources/rest/transform/match/is_true.yml new file mode 100644 index 0000000000000..a23d8ad0349bd --- /dev/null +++ b/buildSrc/src/test/resources/rest/transform/match/is_true.yml @@ -0,0 +1,110 @@ +--- +setup: + - do: + something: + here: ok +--- +teardown: + - do: + something_else: + here: true +--- +"First test": + + - do: + something: + that_is: true + + - do: + and: again + + - match: { copied.from.real.test.total: 1 } + - match: { hits.hits.0._index: "single_doc_index"} + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 0} + - match: { _shards.failed: 0 } + + - do: + and: again + + - match: { hits.total: 1 } + - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"} + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 0} + - match: { _below_is_target_for_tests: 0 } + - is_true: "test_index.mappings.type_1" + +--- +"Also has _type match": + + - do: + something: + that_is: true + + - do: + and: again + + - match: { hits.total.value: 0 } + - is_true: "test_index.mappings.type_1" + - match: { _below_is_target_for_tests: 0 } + - match: { _type: foo } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + + - do: + not_random_but_representive: "of actual test" + + - match: { hits.total.value: 0 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + +--- +"Does not have _type match ": + + - do: + something: + that_is: true + + - do: + and: again + + - match: { hits.total.value: 0 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + + - do: + it: again + + - match: { _type: 0 } + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } +--- +"Last test": + + - do: + something: 中文 + + - match: { _index: test_1 } + - is_true: "test_index.mappings.type_1" + - match: { _id: 中文 } + - match: { _source: { foo: "Hello: 中文" } } + + - do: + something_else: 中文 + + - match: { _index: test_1 } + - match: { _type: "the value does not matter" } + - match: { _id: 中文 } + - match: { _source: { foo: "Hello: 中文" } } + + diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index e8101d069f359..a4e86eff1a95e 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -143,9 +143,8 @@ tasks.named("yamlRestCompatTest").configure { 'explain/40_mix_typeless_typeful/Explain with typeless API on an index that has types', 'field_caps/10_basic/Get date_nanos field caps', 'field_caps/30_filter/Field caps with index filter', - 'get/100_mix_typeless_typeful/GET with typeless API on an index that has types',// failing due to include_type_name #48632 - 'get/21_stored_fields_with_types/Stored fields', // failing due to include_type_name #48632 - 'get/71_source_filtering_with_types/Source filtering',// failing due to include_type_name #48632 + // WILL NOT BE FIXED - failing due to not recognising missing type (the type path param is ignored) + 'get/100_mix_typeless_typeful/GET with typeless API on an index that has types', 'get_source/11_basic_with_types/Basic with types', 'get_source/16_default_values_with_types/Default values', 'get_source/41_routing_with_types/Routing', @@ -155,8 +154,6 @@ tasks.named("yamlRestCompatTest").configure { 'get_source/81_missing_with_types/Missing document with ignore', 'get_source/86_source_missing_with_types/Missing document source with catch', 'get_source/86_source_missing_with_types/Missing document source with ignore', - 'index/70_mix_typeless_typeful/Index call that introduces new field mappings', // failing due to include_type_name #48632 - 'index/70_mix_typeless_typeful/Index with typeless API on an index that has types', // failing due to include_type_name #48632 'indices.clone/10_basic/Clone index via API', 'indices.create/10_basic/Create index with explicit _doc type', 'indices.create/10_basic/Create index without soft deletes', @@ -173,8 +170,8 @@ tasks.named("yamlRestCompatTest").configure { 'indices.flush/10_basic/Flush stats', 'indices.flush/10_basic/Index synced flush rest test', 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', - 'indices.get/11_basic_with_types/Test include_type_name', - 'indices.get/11_basic_with_types/Test include_type_name dafaults to false', +// 'indices.get/11_basic_with_types/Test include_type_name', +// 'indices.get/11_basic_with_types/Test include_type_name dafaults to false', 'indices.get_field_mapping/10_basic/Get field mapping with local is deprecated', 'indices.get_field_mapping/11_basic_with_types/Get field mapping by index only', 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field', @@ -355,4 +352,5 @@ tasks.named("yamlRestCompatTest").configure { tasks.named("transformV7RestTests").configure({ task -> task.replaceMatch("_type", "_doc") task.addAllowedWarningRegex("\\[types removal\\].*") + task.replaceIsTrue("test_index.mappings.type_1", "test_index.mappings._doc") }) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index cc9aaf74ef970..f4c996885b00c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; @@ -26,6 +27,9 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; + /** * A response for a get index action. */ @@ -192,7 +196,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (indexMappings == null) { builder.startObject("mappings").endObject(); } else { - builder.field("mappings", indexMappings.sourceAsMap()); + boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName) { + builder.startObject("mappings"); + builder.field(MapperService.SINGLE_MAPPING_NAME, indexMappings.sourceAsMap()); + builder.endObject(); + } else { + builder.field("mappings", indexMappings.sourceAsMap()); + } } builder.startObject("settings"); @@ -221,6 +233,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + private void addMappings(XContentBuilder builder, MappingMetadata indexMappings) throws IOException { + if (indexMappings == null) { + builder.startObject("mappings").endObject(); + } else { + builder.field("mappings", indexMappings.sourceAsMap()); + } + } + @Override public String toString() { return Strings.toString(this); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index c7ed7ebaad560..c699a194286e7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; @@ -23,6 +24,9 @@ import java.io.IOException; +import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; + public class GetMappingsResponse extends ActionResponse implements ToXContentFragment { private static final ParseField MAPPINGS = new ParseField("mappings"); diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 4bd7272b3c440..59d4db3dc5a38 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -12,6 +12,7 @@ import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -40,6 +41,13 @@ */ public abstract class BaseRestHandler implements RestHandler { + /** + * Parameter that controls whether certain REST apis should include type names in their requests or responses. + * Note: This parameter is only available through compatible rest api. + */ + public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name"; + public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false; + public static final Setting MULTI_ALLOW_EXPLICIT_INDEX = Setting.boolSetting("rest.action.multi.allow_explicit_index", true, Property.NodeScope); @@ -72,14 +80,14 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl // use a sorted set so the unconsumed parameters appear in a reliable sorted order final SortedSet unconsumedParams = request.unconsumedParams() .stream() - .filter(p -> responseParams().contains(p) == false) + .filter(p -> responseParams(request.getRestApiVersion()).contains(p) == false) .collect(Collectors.toCollection(TreeSet::new)); // validate the non-response params if (unconsumedParams.isEmpty() == false) { final Set candidateParams = new HashSet<>(); candidateParams.addAll(request.consumedParams()); - candidateParams.addAll(responseParams()); + candidateParams.addAll(responseParams(request.getRestApiVersion())); throw new IllegalArgumentException(unrecognized(request, unconsumedParams, candidateParams, "parameter")); } @@ -174,6 +182,10 @@ protected Set responseParams() { return Collections.emptySet(); } + protected Set responseParams(RestApiVersion restApiVersion) { + return responseParams(); + } + public static class Wrapper extends BaseRestHandler { protected final BaseRestHandler delegate; @@ -202,6 +214,11 @@ protected Set responseParams() { return delegate.responseParams(); } + @Override + protected Set responseParams(RestApiVersion restApiVersion) { + return delegate.responseParams(restApiVersion); + } + @Override public boolean canTripCircuitBreaker() { return delegate.canTripCircuitBreaker(); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index 78801211488f9..621cdc18b1094 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -11,6 +11,9 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.mapper.MapperService; @@ -19,6 +22,7 @@ import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,6 +32,10 @@ public class RestCreateIndexAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestCreateIndexAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in create " + + "index requests is deprecated. The parameter will be removed in the next major version."; + @Override public List routes() { return List.of(new Route(PUT, "/{index}")); @@ -40,7 +48,60 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + CreateIndexRequest createIndexRequest; + if (request.getRestApiVersion() == RestApiVersion.V_7) { + createIndexRequest = prepareRequestV7(request); + } else { + createIndexRequest = prepareRequest(request); + } + return channel -> client.admin().indices().create(createIndexRequest, new RestToXContentListener<>(channel)); + } + // default scope for testing types in mapping + CreateIndexRequest prepareRequestV7(RestRequest request) { + CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.compatibleApiWarning("create_index_with_types", TYPES_DEPRECATION_MESSAGE); + } + + if (request.hasContent()) { + Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); + + + request.param(INCLUDE_TYPE_NAME_PARAMETER);// just consume, it is always replaced with _doc + sourceAsMap = prepareMappingsV7(sourceAsMap, request); + + createIndexRequest.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); + } + + createIndexRequest.timeout(request.paramAsTime("timeout", createIndexRequest.timeout())); + createIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", createIndexRequest.masterNodeTimeout())); + createIndexRequest.waitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards"))); + return createIndexRequest; + } + + static Map prepareMappingsV7(Map source, RestRequest request) { + final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, false); + + @SuppressWarnings("unchecked") + Map mappings = (Map) source.get("mappings"); + + if (includeTypeName && mappings.size() == 1) { + Map newSource = new HashMap<>(); + + String typeName = mappings.keySet().iterator().next(); + @SuppressWarnings("unchecked") + Map typedMappings = (Map) mappings.get(typeName); + + // no matter what the type was, replace it with _doc, because the internal representation still uses single type `_doc`. + newSource.put("mappings", Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, typedMappings)); + return newSource; + } else { + return prepareMappings(source); + } + } + + CreateIndexRequest prepareRequest(RestRequest request) { CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); if (request.hasContent()) { @@ -53,9 +114,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC createIndexRequest.timeout(request.paramAsTime("timeout", createIndexRequest.timeout())); createIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", createIndexRequest.masterNodeTimeout())); createIndexRequest.waitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards"))); - return channel -> client.admin().indices().create(createIndexRequest, new RestToXContentListener<>(channel)); - } + return createIndexRequest; + } static Map prepareMappings(Map source) { if (source.containsKey("mappings") == false diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java index 6394666bd98e0..0b13c6620198c 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java @@ -12,15 +12,20 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.HEAD; @@ -29,6 +34,13 @@ * The REST handler for get index and head index APIs. */ public class RestGetIndicesAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetIndicesAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using `include_type_name` in get indices requests" + + " is deprecated. The parameter will be removed in the next major version."; + + private static final Set COMPATIBLE_RESPONSE_PARAMS = Collections + .unmodifiableSet(Stream.concat(Collections.singleton(INCLUDE_TYPE_NAME_PARAMETER).stream(), Settings.FORMAT_PARAMS.stream()) + .collect(Collectors.toSet())); @Override public List routes() { @@ -44,6 +56,12 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + // starting with 7.0 we don't include types by default in the response to GET requests + if (request.getRestApiVersion() == RestApiVersion.V_7 && + request.hasParam(INCLUDE_TYPE_NAME_PARAMETER) && request.method().equals(GET)) { + deprecationLogger.compatibleApiWarning("get_indices_with_types", TYPES_DEPRECATION_MESSAGE); + } + String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final GetIndexRequest getIndexRequest = new GetIndexRequest(); getIndexRequest.indices(indices); @@ -63,4 +81,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC protected Set responseParams() { return Settings.FORMAT_PARAMS; } + + @Override + protected Set responseParams(RestApiVersion restApiVersion) { + if(restApiVersion == RestApiVersion.V_7){ + return COMPATIBLE_RESPONSE_PARAMS; + } else { + return responseParams(); + } + } } diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java index fe89d90405777..b38c5d69b6fb8 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java @@ -8,15 +8,29 @@ package org.elasticsearch.rest.action.admin.indices; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; + public class RestCreateIndexActionTests extends ESTestCase { public void testPrepareTypelessRequest() throws IOException { @@ -69,4 +83,59 @@ public void testMalformedMappings() throws IOException { Map source = RestCreateIndexAction.prepareMappings(contentAsMap); assertEquals(contentAsMap, source); } + + public void testIncludeTypeName() throws IOException { + RestCreateIndexAction action = new RestCreateIndexAction(); + List compatibleMediaType = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); + + Map params = new HashMap<>(); + params.put(INCLUDE_TYPE_NAME_PARAMETER, randomFrom("true", "false")); + RestRequest deprecatedRequest = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Accept", compatibleMediaType)) + .withMethod(RestRequest.Method.PUT) + .withPath("/some_index") + .withParams(params) + .build(); + + action.prepareRequest(deprecatedRequest, mock(NodeClient.class)); + assertWarnings(RestCreateIndexAction.TYPES_DEPRECATION_MESSAGE); + + RestRequest validRequest = new FakeRestRequest.Builder(xContentRegistry()) + .withMethod(RestRequest.Method.PUT) + .withPath("/some_index") + .build(); + action.prepareRequest(validRequest, mock(NodeClient.class)); + } + + public void testTypeInMapping() throws IOException { + RestCreateIndexAction action = new RestCreateIndexAction(); + + List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); + + String content = "{\n" + + " \"mappings\": {\n" + + " \"some_type\": {\n" + + " \"properties\": {\n" + + " \"field1\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + Map params = new HashMap<>(); + params.put(RestCreateIndexAction.INCLUDE_TYPE_NAME_PARAMETER, "true"); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withPath("/some_index") + .withParams(params) + .withContent(new BytesArray(content), null) + .build(); + + CreateIndexRequest createIndexRequest = action.prepareRequestV7(request); + // some_type is replaced with _doc + assertThat(createIndexRequest.mappings(), equalTo("{\"_doc\":{\"properties\":{\"field1\":{\"type\":\"text\"}}}}")); + assertWarnings(RestCreateIndexAction.TYPES_DEPRECATION_MESSAGE); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 0d654136b23b1..81a23048c9790 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -1144,6 +1144,10 @@ public static Class getTestTransportPlugin() { public String randomCompatibleMediaType(RestApiVersion version) { XContentType type = randomFrom(XContentType.VND_JSON, XContentType.VND_SMILE, XContentType.VND_CBOR, XContentType.VND_YAML); + return compatibleMediaType(type, version); + } + + public String compatibleMediaType(XContentType type, RestApiVersion version) { return type.toParsedMediaType() .responseContentTypeHeader(Map.of(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, String.valueOf(version.major))); } From 51ef24abdfea63bce5fd1d17bed301a425a0dd98 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 29 Mar 2021 10:54:45 +0200 Subject: [PATCH 02/35] style --- .../rest/transform/RestTestTransformer.java | 7 ++- .../rest/transform/match/ReplaceTextual.java | 2 +- .../transform/match/ReplaceTextualTests.java | 46 +++++++++---------- .../mapping/get/GetMappingsResponse.java | 4 -- .../admin/indices/RestCreateIndexAction.java | 1 - 5 files changed, 27 insertions(+), 33 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java index 2003397acd8af..9b6ef3f5eaa24 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformer.java @@ -144,13 +144,12 @@ private void traverseTest( } else { if (entry.getValue().isObject()) { ObjectNode child = (ObjectNode) entry.getValue(); -// if (child.has(transform.requiredChildKey())) { - if(transform.matches(child)) { + if (child.has(transform.requiredChildKey())) { transform.transformTest((ObjectNode) currentNode); } - } else if(entry.getValue().isTextual()){ + } else if (entry.getValue().isTextual()) { String fieldValue = entry.getValue().asText(); - if(transform.matches(fieldValue)) {///fieldValue.equals(transform.requiredChildKey())){ + if (fieldValue.equals(transform.requiredChildKey())) { transform.transformTest((ObjectNode) currentNode); } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java index 86a06d4df9158..a6182dc0bee9d 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java @@ -54,7 +54,7 @@ public boolean shouldApply(RestTestContext testContext) { @Override public void transformTest(ObjectNode matchParent) { -// ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); + // ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); matchParent.set(getKeyToFind(), replacementNode); } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java index 07aa62adc82dc..10b1fa5a0dc32 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -58,9 +58,9 @@ private void validateTest(List tests, boolean beforeTransformation, JsonNode matchObject = childObject.get("is_true"); if (matchObject != null) { firstTestHasMatchObject.set(true); - if (firstTestHasTypeMatch.get() == false ) { + if (firstTestHasTypeMatch.get() == false) { firstTestHasTypeMatch.set(true); - }/*&& matchObject.asText().equals("test_index.mappings.type_1")*/ + } /*&& matchObject.asText().equals("test_index.mappings.type_1")*/ if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); } @@ -82,9 +82,9 @@ private void validateTest(List tests, boolean beforeTransformation, JsonNode matchObject = childObject.get("is_true"); if (matchObject != null) { lastTestHasMatchObject.set(true); - if (lastTestHasTypeMatch.get() == false ) { + if (lastTestHasTypeMatch.get() == false) { lastTestHasTypeMatch.set(true); - }/*&& matchObject.asText().equals("test_index.mappings.type_1")*/ + } /*&& matchObject.asText().equals("test_index.mappings.type_1")*/ if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); } @@ -93,25 +93,25 @@ private void validateTest(List tests, boolean beforeTransformation, assertTrue(lastTestHasMatchObject.get()); assertTrue(lastTestHasTypeMatch.get()); -// // exclude setup, teardown, first test, and last test -// for (int i = 3; i <= tests.size() - 2; i++) { -// ObjectNode otherTest = tests.get(i); -// JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); -// assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); -// ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; -// otherTestParentArray.elements().forEachRemaining(node -> { -// assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); -// ObjectNode childObject = (ObjectNode) node; -// JsonNode matchObject = childObject.get("match"); -// if (matchObject != null) { -// if (matchObject.get("_type") != null) { -// if (matchObject.get("_type") != null && beforeTransformation == false && allTests) { -// assertThat(matchObject.get("_type").asText(), CoreMatchers.is("_replaced_type")); -// } -// } -// } -// }); -// } + // // exclude setup, teardown, first test, and last test + // for (int i = 3; i <= tests.size() - 2; i++) { + // ObjectNode otherTest = tests.get(i); + // JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); + // assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); + // ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; + // otherTestParentArray.elements().forEachRemaining(node -> { + // assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); + // ObjectNode childObject = (ObjectNode) node; + // JsonNode matchObject = childObject.get("match"); + // if (matchObject != null) { + // if (matchObject.get("_type") != null) { + // if (matchObject.get("_type") != null && beforeTransformation == false && allTests) { + // assertThat(matchObject.get("_type").asText(), CoreMatchers.is("_replaced_type")); + // } + // } + // } + // }); + // } } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index c699a194286e7..c7ed7ebaad560 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; @@ -24,9 +23,6 @@ import java.io.IOException; -import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; -import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; - public class GetMappingsResponse extends ActionResponse implements ToXContentFragment { private static final ParseField MAPPINGS = new ParseField("mappings"); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index 621cdc18b1094..891af2db58287 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.RestApiVersion; -import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; From 1bbc802c71fcdde1bdd3db69c173e02603c90289 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 29 Mar 2021 12:17:21 +0200 Subject: [PATCH 03/35] test fix --- .../rest/compat/RestCompatTestTransformTask.java | 3 ++- .../transform/RestTestTransformByParentObject.java | 5 +++++ .../test/rest/transform/RestTestTransformer.java | 7 ++++--- .../test/rest/transform/match/ReplaceTextual.java | 14 ++++++++++---- .../rest/transform/match/ReplaceTextualTests.java | 3 ++- .../admin/indices/RestCreateIndexAction.java | 4 ++++ 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index ab7bf3ae218c2..d7e38ec01353e 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLParser; import org.elasticsearch.gradle.Version; @@ -103,7 +104,7 @@ public void replaceMatch(String subKey, Object value) { } public void replaceIsTrue(String subKey, Object value) { - transformations.add(new ReplaceTextual(subKey, MAPPER.convertValue(value, JsonNode.class))); + transformations.add(new ReplaceTextual(subKey, MAPPER.convertValue(value, TextNode.class))); } /** diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentObject.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentObject.java index 531530654f0c5..219630a62880f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentObject.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/RestTestTransformByParentObject.java @@ -8,6 +8,7 @@ package org.elasticsearch.gradle.test.rest.transform; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; /** @@ -27,4 +28,8 @@ public interface RestTestTransformByParentObject extends RestTestTransform tests = getTests(testName); - JsonNode replacementNode = MAPPER.convertValue("test_index.mappings._doc", JsonNode.class); + TextNode replacementNode = MAPPER.convertValue("test_index.mappings._doc", TextNode.class); validateTest(tests, true, true); List transformedTests = transformTests( new LinkedList<>(tests), diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index 891af2db58287..04437b0ff3137 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; @@ -89,6 +90,9 @@ static Map prepareMappingsV7(Map source, RestReq Map newSource = new HashMap<>(); String typeName = mappings.keySet().iterator().next(); + if (Strings.hasText(typeName) == false) { + throw new IllegalArgumentException("name cannot be empty string"); + } @SuppressWarnings("unchecked") Map typedMappings = (Map) mappings.get(typeName); From 9592c74f88679e29fa1ccc6148d974984d77dc02 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 29 Mar 2021 14:57:40 +0200 Subject: [PATCH 04/35] is true as a parameter --- .../compat/RestCompatTestTransformTask.java | 2 +- .../rest/transform/match/ReplaceTextual.java | 23 +++--- .../transform/match/ReplaceTextualTests.java | 76 ++++++++----------- .../match/{is_true.yml => text_replace.yml} | 6 +- 4 files changed, 49 insertions(+), 58 deletions(-) rename buildSrc/src/test/resources/rest/transform/match/{is_true.yml => text_replace.yml} (94%) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index d7e38ec01353e..9ed6554d26263 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -104,7 +104,7 @@ public void replaceMatch(String subKey, Object value) { } public void replaceIsTrue(String subKey, Object value) { - transformations.add(new ReplaceTextual(subKey, MAPPER.convertValue(value, TextNode.class))); + transformations.add(new ReplaceTextual("is_true", subKey, MAPPER.convertValue(value, TextNode.class))); } /** diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java index 0432805f8b174..ad32bc3bb86c5 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java @@ -18,21 +18,24 @@ import org.gradle.api.tasks.Optional; /** - * A transformation to replace the flat fields + * A transformation to replace the flat textual fields. */ public class ReplaceTextual implements RestTestTransformByParentObject { - private final String replaceKey; + private final String keyToReplaceName; + private final String valueToBeReplaced; private final TextNode replacementNode; private final String testName; - public ReplaceTextual(String replaceKey, TextNode replacementNode) { - this.replaceKey = replaceKey; + public ReplaceTextual(String keyToReplaceName, String valueToBeReplaced, TextNode replacementNode) { + this.keyToReplaceName = keyToReplaceName; + this.valueToBeReplaced = valueToBeReplaced; this.replacementNode = replacementNode; this.testName = null; } - public ReplaceTextual(String replaceKey, TextNode replacementNode, String testName) { - this.replaceKey = replaceKey; + public ReplaceTextual(String keyToReplaceName, String valueToBeReplaced, TextNode replacementNode, String testName) { + this.keyToReplaceName = keyToReplaceName; + this.valueToBeReplaced = valueToBeReplaced; this.replacementNode = replacementNode; this.testName = testName; } @@ -40,12 +43,12 @@ public ReplaceTextual(String replaceKey, TextNode replacementNode, String testNa @Override @Internal public String getKeyToFind() { - return "is_true"; + return keyToReplaceName; } @Override public String requiredChildKey() { - return replaceKey; + return valueToBeReplaced; } @Override @@ -59,8 +62,8 @@ public void transformTest(ObjectNode matchParent) { } @Input - public String getReplaceKey() { - return replaceKey; + public String getValueToBeReplaced() { + return valueToBeReplaced; } @Input diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java index 11e6dd7cffaa9..7e6788cb75eac 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -30,13 +30,13 @@ public class ReplaceTextualTests extends TransformTests { @Test public void testReplaceAll() throws Exception { - String testName = "/rest/transform/match/is_true.yml"; + String testName = "/rest/transform/match/text_replace.yml"; List tests = getTests(testName); - TextNode replacementNode = MAPPER.convertValue("test_index.mappings._doc", TextNode.class); + TextNode replacementNode = MAPPER.convertValue("_replaced_value", TextNode.class); validateTest(tests, true, true); List transformedTests = transformTests( new LinkedList<>(tests), - Collections.singletonList(new ReplaceTextual("test_index.mappings.type_1", replacementNode, null)) + Collections.singletonList(new ReplaceTextual("key_to_replace", "key_to_replace", replacementNode, null)) ); printTest(testName, transformedTests); validateTest(tests, false, true); @@ -50,69 +50,57 @@ private void validateTest(List tests, boolean beforeTransformation, assertThat(firstTestChild, CoreMatchers.instanceOf(ArrayNode.class)); ArrayNode firstTestParentArray = (ArrayNode) firstTestChild; - AtomicBoolean firstTestHasMatchObject = new AtomicBoolean(false); - AtomicBoolean firstTestHasTypeMatch = new AtomicBoolean(false); + AtomicBoolean firstTestOccurrenceFound = new AtomicBoolean(false); firstTestParentArray.elements().forEachRemaining(node -> { assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("is_true"); + JsonNode matchObject = childObject.get("key_to_replace"); if (matchObject != null) { - firstTestHasMatchObject.set(true); - if (firstTestHasTypeMatch.get() == false) { - firstTestHasTypeMatch.set(true); - } /*&& matchObject.asText().equals("test_index.mappings.type_1")*/ - if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { - assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); + firstTestOccurrenceFound.set(true); + if (beforeTransformation == false && allTests) { + assertThat(matchObject.asText(), CoreMatchers.is("_replaced_value")); } } }); - assertTrue(firstTestHasMatchObject.get()); - assertTrue(firstTestHasTypeMatch.get()); + assertTrue(firstTestOccurrenceFound.get()); // last test JsonNode lastTestChild = tests.get(tests.size() - 1).get("Last test"); assertThat(lastTestChild, CoreMatchers.instanceOf(ArrayNode.class)); ArrayNode lastTestParentArray = (ArrayNode) lastTestChild; - AtomicBoolean lastTestHasMatchObject = new AtomicBoolean(false); - AtomicBoolean lastTestHasTypeMatch = new AtomicBoolean(false); + AtomicBoolean lastTestOccurrenceFound = new AtomicBoolean(false); lastTestParentArray.elements().forEachRemaining(node -> { assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("is_true"); + JsonNode matchObject = childObject.get("key_to_replace"); if (matchObject != null) { - lastTestHasMatchObject.set(true); - if (lastTestHasTypeMatch.get() == false) { - lastTestHasTypeMatch.set(true); - } /*&& matchObject.asText().equals("test_index.mappings.type_1")*/ - if (/*matchObject.get("_type") != null && */beforeTransformation == false && allTests) { - assertThat(matchObject.asText(), CoreMatchers.is("test_index.mappings._doc")); + lastTestOccurrenceFound.set(true); + if (beforeTransformation == false && allTests) { + assertThat(matchObject.asText(), CoreMatchers.is("_replaced_value")); } } }); - assertTrue(lastTestHasMatchObject.get()); - assertTrue(lastTestHasTypeMatch.get()); + assertTrue(lastTestOccurrenceFound.get()); - // // exclude setup, teardown, first test, and last test - // for (int i = 3; i <= tests.size() - 2; i++) { - // ObjectNode otherTest = tests.get(i); - // JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); - // assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); - // ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; - // otherTestParentArray.elements().forEachRemaining(node -> { - // assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - // ObjectNode childObject = (ObjectNode) node; - // JsonNode matchObject = childObject.get("match"); - // if (matchObject != null) { - // if (matchObject.get("_type") != null) { - // if (matchObject.get("_type") != null && beforeTransformation == false && allTests) { - // assertThat(matchObject.get("_type").asText(), CoreMatchers.is("_replaced_type")); - // } - // } - // } - // }); - // } + // exclude setup, teardown, first test, and last test + for (int i = 3; i <= tests.size() - 2; i++) { + ObjectNode otherTest = tests.get(i); + JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); + assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); + ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; + otherTestParentArray.elements().forEachRemaining(node -> { + assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); + ObjectNode childObject = (ObjectNode) node; + JsonNode matchObject = childObject.get("key_to_replace"); + if (matchObject != null) { + if (beforeTransformation == false && allTests) { + assertThat(matchObject.get("key_to_replace").asText(), CoreMatchers.is("_replaced_value")); + } + } + }); + } } @Override diff --git a/buildSrc/src/test/resources/rest/transform/match/is_true.yml b/buildSrc/src/test/resources/rest/transform/match/text_replace.yml similarity index 94% rename from buildSrc/src/test/resources/rest/transform/match/is_true.yml rename to buildSrc/src/test/resources/rest/transform/match/text_replace.yml index a23d8ad0349bd..3016d113d7220 100644 --- a/buildSrc/src/test/resources/rest/transform/match/is_true.yml +++ b/buildSrc/src/test/resources/rest/transform/match/text_replace.yml @@ -34,7 +34,7 @@ teardown: - match: { _shards.successful: 2 } - match: { _shards.skipped : 0} - match: { _below_is_target_for_tests: 0 } - - is_true: "test_index.mappings.type_1" + - key_to_replace: "_to_be_replaced" --- "Also has _type match": @@ -47,7 +47,7 @@ teardown: and: again - match: { hits.total.value: 0 } - - is_true: "test_index.mappings.type_1" + - key_to_replace: "_to_be_replaced" - match: { _below_is_target_for_tests: 0 } - match: { _type: foo } - match: { _shards.total: 2 } @@ -95,7 +95,7 @@ teardown: something: 中文 - match: { _index: test_1 } - - is_true: "test_index.mappings.type_1" + - key_to_replace: "_to_be_replaced" - match: { _id: 中文 } - match: { _source: { foo: "Hello: 中文" } } From b6abb3f67f5ee4c7e6081b75e205a65c32c52c4e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 29 Mar 2021 17:47:15 +0200 Subject: [PATCH 05/35] create index passing except for get mappings api --- .../compat/RestCompatTestTransformTask.java | 3 +++ rest-api-spec/build.gradle | 12 ++++------- .../admin/indices/RestCreateIndexAction.java | 10 ++++----- .../admin/indices/RestGetMappingAction.java | 9 ++++++++ .../indices/RestPutIndexTemplateAction.java | 21 +++++++++++++++++-- 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 9ed6554d26263..89fedbad549c0 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -107,6 +107,9 @@ public void replaceIsTrue(String subKey, Object value) { transformations.add(new ReplaceTextual("is_true", subKey, MAPPER.convertValue(value, TextNode.class))); } + public void replaceIsFalse(String subKey, Object value) { + transformations.add(new ReplaceTextual("is_false", subKey, MAPPER.convertValue(value, TextNode.class))); + } /** * Replaces the values of a match assertion for the given REST test. For example "match":{"_type": "foo"} to "match":{"_type": "bar"} * diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index a4e86eff1a95e..8206c6bd2646e 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -155,23 +155,17 @@ tasks.named("yamlRestCompatTest").configure { 'get_source/86_source_missing_with_types/Missing document source with catch', 'get_source/86_source_missing_with_types/Missing document source with ignore', 'indices.clone/10_basic/Clone index via API', - 'indices.create/10_basic/Create index with explicit _doc type', 'indices.create/10_basic/Create index without soft deletes', - 'indices.create/11_basic_with_types/Create index', - 'indices.create/11_basic_with_types/Create index with aliases', + // 5 below await retrofitting Removes typed URLs from mapping APIs #41676 'indices.create/11_basic_with_types/Create index with mappings', - 'indices.create/11_basic_with_types/Create index with settings', - 'indices.create/11_basic_with_types/Create index with wait_for_active_shards set to all', - 'indices.create/11_basic_with_types/Create index with write aliases', 'indices.create/20_mix_typeless_typeful/Create a typed index while there is a typeless template', 'indices.create/20_mix_typeless_typeful/Create a typeless index while there is a typed template', 'indices.create/20_mix_typeless_typeful/Implicitly create a typed index while there is a typeless template', 'indices.create/20_mix_typeless_typeful/Implicitly create a typeless index while there is a typed template', + // 'indices.flush/10_basic/Flush stats', 'indices.flush/10_basic/Index synced flush rest test', 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', -// 'indices.get/11_basic_with_types/Test include_type_name', -// 'indices.get/11_basic_with_types/Test include_type_name dafaults to false', 'indices.get_field_mapping/10_basic/Get field mapping with local is deprecated', 'indices.get_field_mapping/11_basic_with_types/Get field mapping by index only', 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field', @@ -353,4 +347,6 @@ tasks.named("transformV7RestTests").configure({ task -> task.replaceMatch("_type", "_doc") task.addAllowedWarningRegex("\\[types removal\\].*") task.replaceIsTrue("test_index.mappings.type_1", "test_index.mappings._doc") + task.replaceIsFalse("test_index.mappings.type_1", "test_index.mappings._doc") + task.replaceIsFalse("test-1.mappings.my_type", "test-1.mappings._doc") }) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index 04437b0ff3137..d5be27fca0d6d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -61,14 +61,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC CreateIndexRequest prepareRequestV7(RestRequest request) { CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.param(INCLUDE_TYPE_NAME_PARAMETER);// just consume, it is always replaced with _doc deprecationLogger.compatibleApiWarning("create_index_with_types", TYPES_DEPRECATION_MESSAGE); } if (request.hasContent()) { Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); - - request.param(INCLUDE_TYPE_NAME_PARAMETER);// just consume, it is always replaced with _doc sourceAsMap = prepareMappingsV7(sourceAsMap, request); createIndexRequest.source(sourceAsMap, LoggingDeprecationHandler.INSTANCE); @@ -86,9 +85,9 @@ static Map prepareMappingsV7(Map source, RestReq @SuppressWarnings("unchecked") Map mappings = (Map) source.get("mappings"); - if (includeTypeName && mappings.size() == 1) { + if (includeTypeName && mappings != null && mappings.size() == 1) { Map newSource = new HashMap<>(); - + newSource.putAll(source); // mappings will be overridden. Aliases, settings stay the same String typeName = mappings.keySet().iterator().next(); if (Strings.hasText(typeName) == false) { throw new IllegalArgumentException("name cannot be empty string"); @@ -132,7 +131,8 @@ static Map prepareMappings(Map source) { @SuppressWarnings("unchecked") Map mappings = (Map) source.get("mappings"); if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { - throw new IllegalArgumentException("The mapping definition cannot be nested under a type"); + throw new IllegalArgumentException("The mapping definition cannot be nested under a type " + + "[" + MapperService.SINGLE_MAPPING_NAME + "] unless include_type_name is set to true."); } newSource.put("mappings", singletonMap(MapperService.SINGLE_MAPPING_NAME, mappings)); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index 53585818546f4..a6d271991a2bc 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.BaseRestHandler; @@ -32,6 +33,9 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; public class RestGetMappingAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetMappingAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get" + + " mapping requests is deprecated. The parameter will be removed in the next major version."; private final ThreadPool threadPool; @@ -55,6 +59,11 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.param(INCLUDE_TYPE_NAME_PARAMETER); + deprecationLogger.compatibleApiWarning("get_mapping_with_types", TYPES_DEPRECATION_MESSAGE); + } + final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index 05b8bfc998cf0..12ab2346e2107 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -10,7 +10,9 @@ import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -25,7 +27,10 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT; public class RestPutIndexTemplateAction extends BaseRestHandler { - + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestPutIndexTemplateAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" + + " Specifying include_type_name in put index template requests is deprecated."+ + " The parameter will be removed in the next major version."; @Override public List routes() { return List.of( @@ -50,7 +55,19 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); - sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap); + if(request.getRestApiVersion() == RestApiVersion.V_7) { + boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER) ) { + deprecationLogger.compatibleApiWarning("put_index_template_with_types", TYPES_DEPRECATION_MESSAGE); + } + if(includeTypeName) { + sourceAsMap = RestCreateIndexAction.prepareMappingsV7(sourceAsMap, request); + } else { + sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap); + } + } else { + sourceAsMap = RestCreateIndexAction.prepareMappings(sourceAsMap); + } putRequest.source(sourceAsMap); return channel -> client.admin().indices().putTemplate(putRequest, new RestToXContentListener<>(channel)); From 4ead4002c9eb38b6d3d79a32f2e7f1c759b2b64c Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 30 Mar 2021 09:31:53 +0200 Subject: [PATCH 06/35] code style --- .../gradle/internal/rest/compat/RestCompatTestTransformTask.java | 1 + 1 file changed, 1 insertion(+) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 89fedbad549c0..3f5247e244c6f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -110,6 +110,7 @@ public void replaceIsTrue(String subKey, Object value) { public void replaceIsFalse(String subKey, Object value) { transformations.add(new ReplaceTextual("is_false", subKey, MAPPER.convertValue(value, TextNode.class))); } + /** * Replaces the values of a match assertion for the given REST test. For example "match":{"_type": "foo"} to "match":{"_type": "bar"} * From adead4d082b0d453d64f4b0fa2e53a8f47b1640e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 30 Mar 2021 10:10:33 +0200 Subject: [PATCH 07/35] test fix --- .../test/rest/transform/match/ReplaceTextualTests.java | 3 +-- .../test/resources/rest/transform/match/text_replace.yml | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java index 7e6788cb75eac..fd710b0bc497e 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -36,11 +36,10 @@ public void testReplaceAll() throws Exception { validateTest(tests, true, true); List transformedTests = transformTests( new LinkedList<>(tests), - Collections.singletonList(new ReplaceTextual("key_to_replace", "key_to_replace", replacementNode, null)) + Collections.singletonList(new ReplaceTextual("key_to_replace", "value_to_replace", replacementNode, null)) ); printTest(testName, transformedTests); validateTest(tests, false, true); - } private void validateTest(List tests, boolean beforeTransformation, boolean allTests) { diff --git a/buildSrc/src/test/resources/rest/transform/match/text_replace.yml b/buildSrc/src/test/resources/rest/transform/match/text_replace.yml index 3016d113d7220..a3f71bc33ffd0 100644 --- a/buildSrc/src/test/resources/rest/transform/match/text_replace.yml +++ b/buildSrc/src/test/resources/rest/transform/match/text_replace.yml @@ -34,7 +34,7 @@ teardown: - match: { _shards.successful: 2 } - match: { _shards.skipped : 0} - match: { _below_is_target_for_tests: 0 } - - key_to_replace: "_to_be_replaced" + - key_to_replace: "value_to_replace" --- "Also has _type match": @@ -47,7 +47,7 @@ teardown: and: again - match: { hits.total.value: 0 } - - key_to_replace: "_to_be_replaced" + - key_to_replace: "value_to_replace" - match: { _below_is_target_for_tests: 0 } - match: { _type: foo } - match: { _shards.total: 2 } @@ -95,7 +95,7 @@ teardown: something: 中文 - match: { _index: test_1 } - - key_to_replace: "_to_be_replaced" + - key_to_replace: "value_to_replace" - match: { _id: 中文 } - match: { _source: { foo: "Hello: 中文" } } From c8f7413e25ebb7e0401b5ff4f479ea5d74d5c2dd Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 30 Mar 2021 13:42:55 +0200 Subject: [PATCH 08/35] fix test --- .../test/rest/transform/match/ReplaceTextualTests.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java index fd710b0bc497e..ac6ed772b5576 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -93,10 +93,9 @@ private void validateTest(List tests, boolean beforeTransformation, assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); ObjectNode childObject = (ObjectNode) node; JsonNode matchObject = childObject.get("key_to_replace"); - if (matchObject != null) { - if (beforeTransformation == false && allTests) { - assertThat(matchObject.get("key_to_replace").asText(), CoreMatchers.is("_replaced_value")); - } + if (matchObject != null && matchObject.get("key_to_replace") != null + && beforeTransformation == false && allTests) { + assertThat(matchObject.get("key_to_replace").asText(), CoreMatchers.is("_replaced_value")); } }); } From 57e63fcd2dffb56e9866950d100679304da26186 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 30 Mar 2021 14:12:34 +0200 Subject: [PATCH 09/35] style check --- .../gradle/test/rest/transform/match/ReplaceTextualTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java index ac6ed772b5576..0f05a4ba2cc7d 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java @@ -93,8 +93,7 @@ private void validateTest(List tests, boolean beforeTransformation, assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); ObjectNode childObject = (ObjectNode) node; JsonNode matchObject = childObject.get("key_to_replace"); - if (matchObject != null && matchObject.get("key_to_replace") != null - && beforeTransformation == false && allTests) { + if (matchObject != null && matchObject.get("key_to_replace") != null && beforeTransformation == false && allTests) { assertThat(matchObject.get("key_to_replace").asText(), CoreMatchers.is("_replaced_value")); } }); From 539b4217969902964db67b9a11b705b4d78c9d3f Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 30 Mar 2021 17:01:43 +0200 Subject: [PATCH 10/35] add all changes --- rest-api-spec/build.gradle | 51 ++++++++----------- .../admin/indices/get/GetIndexResponse.java | 8 --- .../mapping/get/GetFieldMappingsResponse.java | 16 +++++- .../indices/rollover/RolloverRequest.java | 26 ++++++++-- .../get/GetIndexTemplatesResponse.java | 15 +++++- .../metadata/IndexTemplateMetadata.java | 5 +- .../elasticsearch/rest/BaseRestHandler.java | 8 +++ .../indices/RestGetFieldMappingAction.java | 21 ++++++++ .../indices/RestGetIndexTemplateAction.java | 23 +++++++++ .../indices/RestRolloverIndexAction.java | 22 +++++++- .../rollover/RolloverRequestTests.java | 8 +-- x-pack/plugin/build.gradle | 13 +++++ 12 files changed, 164 insertions(+), 52 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index b6c14a05cc1f9..40538c28a2ff4 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -145,7 +145,6 @@ tasks.named("yamlRestCompatTest").configure { 'explain/21_source_filtering_with_types/Source filtering', 'explain/31_query_string_with_types/explain with query_string parameters', 'explain/40_mix_typeless_typeful/Explain with typeless API on an index that has types', - 'field_caps/10_basic/Get date_nanos field caps', 'field_caps/30_filter/Field caps with index filter', // WILL NOT BE FIXED - failing due to not recognising missing type (the type path param is ignored) 'get/100_mix_typeless_typeful/GET with typeless API on an index that has types', @@ -170,23 +169,23 @@ tasks.named("yamlRestCompatTest").configure { 'indices.flush/10_basic/Flush stats', 'indices.flush/10_basic/Index synced flush rest test', 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', - 'indices.get_field_mapping/10_basic/Get field mapping with local is deprecated', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping by index only', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field, with another field that doesn\'t exist', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping should work without index specifying type and fields', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping with include_defaults', - 'indices.get_field_mapping/11_basic_with_types/Get field mapping with no index and type', - 'indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn\'t exist, but type and index do', - 'indices.get_field_mapping/30_missing_type/Raise 404 when type doesn\'t exist', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'*\' for indices and types', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'_all\' for indices and types', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using comma_separated values for indices and types', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with * for fields', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with *t1 for fields', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with t* for fields', - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with wildcarded relative names', - 'indices.get_field_mapping/60_mix_typeless_typeful/GET mapping with typeless API on an index that has types', + 'indices.get_field_mapping/10_basic/Get field mapping with local is deprecated',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping by index only',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field, with another field that doesn\'t exist',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping should work without index specifying type and fields',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping with include_defaults',// awaits #41676 + 'indices.get_field_mapping/11_basic_with_types/Get field mapping with no index and type',// awaits #41676 + 'indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn\'t exist, but type and index do',// awaits #41676 + 'indices.get_field_mapping/30_missing_type/Raise 404 when type doesn\'t exist',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'*\' for indices and types',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'_all\' for indices and types',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using comma_separated values for indices and types',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with * for fields',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with *t1 for fields',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with t* for fields',// awaits #41676 + 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with wildcarded relative names',// awaits #41676 + 'indices.get_field_mapping/60_mix_typeless_typeful/GET mapping with typeless API on an index that has types',// awaits #41676 'indices.get_mapping/11_basic_with_types/Get /*/_mapping/{type}', 'indices.get_mapping/11_basic_with_types/Get /_all/_mapping/{type}', 'indices.get_mapping/11_basic_with_types/Get /_mapping', @@ -207,8 +206,8 @@ tasks.named("yamlRestCompatTest").configure { 'indices.get_mapping/40_aliases/Getting mapping for aliases should return the real index as key', 'indices.get_mapping/61_empty_with_types/Check empty mapping when getting all mappings via /_mapping', 'indices.get_mapping/70_mix_typeless_typeful/GET mapping with typeless API on an index that has types', - 'indices.get_template/11_basic_with_types/Get template', - 'indices.get_template/11_basic_with_types/Get template with no mappings', +// 'indices.get_template/11_basic_with_types/Get template', +// 'indices.get_template/11_basic_with_types/Get template with no mappings', 'indices.open/10_basic/?wait_for_active_shards default is deprecated', 'indices.open/10_basic/?wait_for_active_shards=index-setting', 'indices.put_mapping/10_basic/Put mappings with explicit _doc type', @@ -223,15 +222,9 @@ tasks.named("yamlRestCompatTest").configure { 'indices.put_mapping/all_path_options_with_types/put mapping in prefix* index', 'indices.put_mapping/all_path_options_with_types/put mapping with blank index', 'indices.put_mapping/all_path_options_with_types/put one mapping per index', - 'indices.put_template/10_basic/Put template with explicit _doc type', - 'indices.put_template/11_basic_with_types/Put multiple template', - 'indices.put_template/11_basic_with_types/Put template', + // needs discussion about IndexTemplateMetadata#toInnerXContent and reducingMappings when includeTypeName and empty mappings etc 'indices.put_template/11_basic_with_types/Put template with empty mappings', - 'indices.rollover/10_basic/Rollover index via API', - 'indices.rollover/20_max_doc_condition/Max docs rollover conditions matches only primary shards', - 'indices.rollover/30_max_size_condition/Rollover with max_size condition', - 'indices.rollover/40_mapping/Mappings with explicit _doc type', - 'indices.rollover/41_mapping_with_types/Typeless mapping', + // 'indices.segments/10_basic/basic segments test', 'indices.segments/10_basic/closed segments test', 'indices.shard_stores/10_basic/basic index test', @@ -292,8 +285,6 @@ tasks.named("yamlRestCompatTest").configure { 'search.aggregation/10_histogram/Deprecated _time order', 'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers', 'search.aggregation/20_terms/Deprecated _term order', - 'search.aggregation/280_geohash_grid/Basic test', - 'search.aggregation/290_geotile_grid/Basic test', 'search.aggregation/51_filter_with_types/Filter aggs with terms lookup and ensure it\'s cached', 'search.inner_hits/10_basic/Nested doc version and seqIDs', 'search.inner_hits/10_basic/Nested inner hits', diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index f4c996885b00c..b6d9beb29b4ba 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -233,14 +233,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - private void addMappings(XContentBuilder builder, MappingMetadata indexMappings) throws IOException { - if (indexMappings == null) { - builder.startObject("mappings").endObject(); - } else { - builder.field("mappings", indexMappings.sourceAsMap()); - } - } - @Override public String toString() { return Strings.toString(this); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index e91492c1344e1..254dbed30c26f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -22,6 +23,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.rest.BaseRestHandler; import java.io.IOException; import java.io.InputStream; @@ -30,6 +32,7 @@ import java.util.Objects; import static java.util.Collections.unmodifiableMap; +import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; /** * Response object for {@link GetFieldMappingsRequest} API @@ -84,15 +87,24 @@ public FieldMappingMetadata fieldMappings(String index, String field) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + boolean includeTypeName = params.paramAsBoolean(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); + builder.startObject(); for (Map.Entry> indexEntry : mappings.entrySet()) { builder.startObject(indexEntry.getKey()); builder.startObject(MAPPINGS.getPreferredName()); - if (indexEntry.getValue() != null) { - addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); + if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName) { + builder.startObject(MapperService.SINGLE_MAPPING_NAME); + addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); + builder.endObject(); + } else { + addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); + } } + builder.endObject(); builder.endObject(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 248c8c162d16c..44626a6af41bd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.ByteSizeValue; @@ -37,7 +38,7 @@ */ public class RolloverRequest extends AcknowledgedRequest implements IndicesRequest { - private static final ObjectParser PARSER = new ObjectParser<>("rollover"); + private static final ObjectParser PARSER = new ObjectParser<>("rollover"); private static final ObjectParser>, Void> CONDITION_PARSER = new ObjectParser<>("conditions"); private static final ParseField CONDITIONS = new ParseField("conditions"); @@ -67,6 +68,22 @@ public class RolloverRequest extends AcknowledgedRequest implem CONDITIONS, ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.settings(parser.map()), CreateIndexRequest.SETTINGS, ObjectParser.ValueType.OBJECT); + PARSER.declareField((parser, request, includeTypeName) -> { + if (includeTypeName) { + //expecting one type only + for (Map.Entry mappingsEntry : parser.map().entrySet()) { + request.createIndexRequest.mapping((Map) mappingsEntry.getValue()); + } + } else { + // a type is not included, add a dummy _doc type + Map mappings = parser.map(); + if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { + throw new IllegalArgumentException("The mapping definition cannot be nested under a type " + + "[" + MapperService.SINGLE_MAPPING_NAME + "] unless include_type_name is set to true."); + } + request.createIndexRequest.mapping(mappings); + } + }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> { // a type is not included, add a dummy _doc type Map mappings = parser.map(); @@ -74,7 +91,8 @@ public class RolloverRequest extends AcknowledgedRequest implem throw new IllegalArgumentException("The mapping definition cannot be nested under a type"); } request.createIndexRequest.mapping(mappings); - }, CreateIndexRequest.MAPPINGS, ObjectParser.ValueType.OBJECT); + }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.V_8)), ObjectParser.ValueType.OBJECT); + PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()), CreateIndexRequest.ALIASES, ObjectParser.ValueType.OBJECT); } @@ -238,7 +256,7 @@ public CreateIndexRequest getCreateIndexRequest() { } // param isTypeIncluded decides how mappings should be parsed from XContent - public void fromXContent(XContentParser parser) throws IOException { - PARSER.parse(parser, this, null); + public void fromXContent(boolean isTypeIncluded, XContentParser parser) throws IOException { + PARSER.parse(parser, this, isTypeIncluded); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index 9b95c146b79a7..f3332b734080f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; @@ -20,6 +21,8 @@ import java.util.Objects; import static java.util.Collections.singletonMap; +import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject { @@ -63,9 +66,19 @@ public int hashCode() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { params = new ToXContent.DelegatingMapParams(singletonMap("reduce_mappings", "true"), params); + boolean includeTypeName = false; + if(builder.getRestApiVersion() == RestApiVersion.V_7) { + includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); + } + builder.startObject(); for (IndexTemplateMetadata indexTemplateMetadata : getIndexTemplates()) { - IndexTemplateMetadata.Builder.toXContent(indexTemplateMetadata, builder, params); + if (includeTypeName) { + IndexTemplateMetadata.Builder.toXContentWithTypes(indexTemplateMetadata, builder, params); + } else { + IndexTemplateMetadata.Builder.toXContent(indexTemplateMetadata, builder, params); + } } builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java index 855e357bb6f93..ef592c48b2a14 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; @@ -378,7 +379,9 @@ private static void toInnerXContent(IndexTemplateMetadata indexTemplateMetadata, indexTemplateMetadata.settings().toXContent(builder, params); builder.endObject(); - includeTypeName &= (params.paramAsBoolean("reduce_mappings", false) == false); + if(builder.getRestApiVersion() != RestApiVersion.V_7) { + includeTypeName &= (params.paramAsBoolean("reduce_mappings", false) == false); + } CompressedXContent m = indexTemplateMetadata.mappings(); if (m != null) { diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 59d4db3dc5a38..2d2d0cc83a0d3 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -182,6 +182,14 @@ protected Set responseParams() { return Collections.emptySet(); } + /** + * Parameters used for controlling the response and thus might not be consumed during + * preparation of the request execution. The value depends on the RestApiVersion provided + * by a user on a request. + * Used in RestHandlers with Compatible Rest Api + * @param restApiVersion - a version provided by a user on a request + * @return a set of parameters used to control the response, depending on a restApiVersion + */ protected Set responseParams(RestApiVersion restApiVersion) { return responseParams(); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index 4e192bb996ffa..25283a235ef33 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -8,12 +8,16 @@ package org.elasticsearch.rest.action.admin.indices; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.BytesRestResponse; @@ -32,6 +36,11 @@ public class RestGetFieldMappingAction extends BaseRestHandler { + private static final Logger logger = LogManager.getLogger(RestGetFieldMappingAction.class); + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(logger.getName()); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get " + + "field mapping requests is deprecated. The parameter will be removed in the next major version."; + @Override public List routes() { return List.of( @@ -49,6 +58,18 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final String[] fields = Strings.splitStringByCommaToArray(request.param("fields")); + if(request.getRestApiVersion() == RestApiVersion.V_7){ + boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); +// if (includeTypeName == false /*&& types.length > 0*/) { +// throw new IllegalArgumentException("Types cannot be specified unless include_type_name" + +// " is set to true."); +// } + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.compatibleApiWarning("get_field_mapping_with_types", TYPES_DEPRECATION_MESSAGE); + } + } + + GetFieldMappingsRequest getMappingsRequest = new GetFieldMappingsRequest(); getMappingsRequest.indices(indices).fields(fields).includeDefaults(request.paramAsBoolean("include_defaults", false)); getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index 10b97915f1e7e..fa51cd4bf96f2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -11,14 +11,18 @@ import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Set; @@ -31,6 +35,11 @@ * The REST handler for get template and head template APIs. */ public class RestGetIndexTemplateAction extends BaseRestHandler { + private static final Set COMPATIBLE_RESPONSE_PARAMS = Collections.unmodifiableSet(Sets.union( + Collections.singleton(INCLUDE_TYPE_NAME_PARAMETER), Settings.FORMAT_PARAMS)); + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetIndexTemplateAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get " + + "index template requests is deprecated. The parameter will be removed in the next major version."; @Override public List routes() { @@ -47,6 +56,12 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + if(request.getRestApiVersion() == RestApiVersion.V_7) { + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.param(INCLUDE_TYPE_NAME_PARAMETER); + deprecationLogger.compatibleApiWarning("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE); + } + } final String[] names = Strings.splitStringByCommaToArray(request.param("name")); final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names); @@ -73,4 +88,12 @@ protected Set responseParams() { return Settings.FORMAT_PARAMS; } + @Override + protected Set responseParams(RestApiVersion restApiVersion) { + if(restApiVersion == RestApiVersion.V_7){ + return COMPATIBLE_RESPONSE_PARAMS; + } else { + return responseParams(); + } + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index 025e291176ef4..f6f8e450e4838 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -11,6 +11,8 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; @@ -22,6 +24,10 @@ public class RestRolloverIndexAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestRolloverIndexAction.class); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in rollover " + + "index requests is deprecated. The parameter will be removed in the next major version."; + @Override public List routes() { return List.of( @@ -36,13 +42,25 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final boolean includeTypeName = isIncludeTypeName(request); RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index")); - request.applyContentParser(rolloverIndexRequest::fromXContent); + request.applyContentParser(parser -> rolloverIndexRequest.fromXContent(includeTypeName, parser)); rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false)); rolloverIndexRequest.timeout(request.paramAsTime("timeout", rolloverIndexRequest.timeout())); rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout())); rolloverIndexRequest.getCreateIndexRequest().waitForActiveShards( - ActiveShardCount.parseString(request.param("wait_for_active_shards"))); + ActiveShardCount.parseString(request.param("wait_for_active_shards"))); return channel -> client.admin().indices().rolloverIndex(rolloverIndexRequest, new RestToXContentListener<>(channel)); } + + private boolean isIncludeTypeName(RestRequest request) { + boolean includeTypeName = false; + if (request.getRestApiVersion() == RestApiVersion.V_7) { + includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.compatibleApiWarning("index_rollover_with_types", TYPES_DEPRECATION_MESSAGE); + } + } + return includeTypeName; + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index ced6901b6daaa..71156785b66ca 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -60,7 +60,7 @@ public void testConditionsParsing() throws Exception { .field("max_primary_shard_size", "55gb") .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(false, createParser(builder)); Map> conditions = request.getConditions(); assertThat(conditions.size(), equalTo(4)); MaxAgeCondition maxAgeCondition = (MaxAgeCondition)conditions.get(MaxAgeCondition.NAME); @@ -97,7 +97,7 @@ public void testParsingWithIndexSettings() throws Exception { .startObject("alias1").endObject() .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(false, createParser(builder)); Map> conditions = request.getConditions(); assertThat(conditions.size(), equalTo(2)); assertThat(request.getCreateIndexRequest().mappings(), containsString("not_analyzed")); @@ -118,7 +118,7 @@ public void testTypelessMappingParsing() throws Exception { .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(false, createParser(builder)); CreateIndexRequest createIndexRequest = request.getCreateIndexRequest(); String mapping = createIndexRequest.mappings(); @@ -167,7 +167,7 @@ public void testUnknownFields() throws IOException { } builder.endObject(); BytesReference mutated = XContentTestUtils.insertRandomFields(xContentType, BytesReference.bytes(builder), null, random()); - expectThrows(XContentParseException.class, () -> request.fromXContent(createParser(xContentType.xContent(), mutated))); + expectThrows(XContentParseException.class, () -> request.fromXContent(false, createParser(xContentType.xContent(), mutated))); } public void testSameConditionCanOnlyBeAddedOnce() { diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 973d46e6406e1..b09b791c579fa 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -4,6 +4,7 @@ import org.elasticsearch.gradle.util.GradleUtils apply plugin: 'elasticsearch.yaml-rest-test' apply plugin: 'elasticsearch.validate-rest-spec' apply plugin: 'elasticsearch.internal-test-artifact' +apply plugin: 'elasticsearch.yaml-rest-compat-test' archivesBaseName = 'x-pack' @@ -130,3 +131,15 @@ tasks.register('enforceYamlTestConvention').configure { tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } +//tasks.named("yamlRestCompatTest").configure { +// onlyIf { +// // Skip these tests on Windows since the blacklist exceeds Windows CLI limits +// OS.current() != OS.WINDOWS +// } +// systemProperty 'tests.rest.blacklist', [] +//} +// +//tasks.named("transformV7RestTests").configure({ task -> +// task.replaceMatch("_type", "_doc") +// task.addAllowedWarningRegex("\\[types removal\\].*") +//}) From 27c80c04f327bd0e91a1cb8eb42923be59bcc7a6 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 09:53:06 +0200 Subject: [PATCH 11/35] disable compat xpack tests --- x-pack/plugin/build.gradle | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index b09b791c579fa..ffdf1924bdef3 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -4,7 +4,7 @@ import org.elasticsearch.gradle.util.GradleUtils apply plugin: 'elasticsearch.yaml-rest-test' apply plugin: 'elasticsearch.validate-rest-spec' apply plugin: 'elasticsearch.internal-test-artifact' -apply plugin: 'elasticsearch.yaml-rest-compat-test' +//apply plugin: 'elasticsearch.yaml-rest-compat-test' archivesBaseName = 'x-pack' @@ -15,6 +15,7 @@ dependencies { // let the yamlRestTest see the classpath of test GradleUtils.extendSourceSet(project, "test", "yamlRestTest", tasks.named("yamlRestTest")) +//GradleUtils.extendSourceSet(project, "test", "yamlRestCompatTest", tasks.named("yamlRestCompatTest")) restResources { restApi { @@ -132,11 +133,13 @@ tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } //tasks.named("yamlRestCompatTest").configure { -// onlyIf { -// // Skip these tests on Windows since the blacklist exceeds Windows CLI limits -// OS.current() != OS.WINDOWS -// } -// systemProperty 'tests.rest.blacklist', [] +//// onlyIf { +//// // Skip these tests on Windows since the blacklist exceeds Windows CLI limits +//// OS.current() != OS.WINDOWS +//// } +// systemProperty 'tests.rest.blacklist', [ +// +// ] //} // //tasks.named("transformV7RestTests").configure({ task -> From 4c707bb455142554520c367ab67c8ee354771d10 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 12:49:07 +0200 Subject: [PATCH 12/35] test --- .../get/GetFieldMappingsResponseTests.java | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index b967195eea83b..d04a565827892 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -9,18 +9,33 @@ package org.elasticsearch.action.admin.indices.mapping.get; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; + public class GetFieldMappingsResponseTests extends AbstractWireSerializingTestCase { public void testManualSerialization() throws IOException { @@ -47,6 +62,55 @@ public void testNullFieldMappingToXContent() { assertEquals("{\"index\":{\"mappings\":{}}}", Strings.toString(response)); } + public void testToXContentIncludesType() throws Exception { + Map> mappings = new HashMap<>(); + FieldMappingMetadata fieldMappingMetadata = new FieldMappingMetadata("my field", new BytesArray("{}")); + mappings.put("index", Collections.singletonMap("field", fieldMappingMetadata)); + GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings); + ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, + "true")); + + // v7 with include_type_name attaches _doc + try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) { + response.toXContent(builder, params); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map>> index = (Map>>) parser.map().get("index"); + assertThat(index.get("mappings"), hasKey(MapperService.SINGLE_MAPPING_NAME)); + assertThat(index.get("mappings").get(MapperService.SINGLE_MAPPING_NAME), hasKey("field")); + } + } + + // v7 with no include_type_name do not attach _doc + try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_7)) { + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map> index = (Map>) parser.map().get("index"); + assertThat(index.get("mappings"), hasKey("field")); + } + } + //v8 does not have _doc, een when include_type_name is present + // (although this throws unconsumed parameter exception in RestGetFieldMappings) + try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) { + response.toXContent(builder, params); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map> index = (Map>) parser.map().get("index"); + assertThat(index.get("mappings"), hasKey("field")); + } + } + + try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) { + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map> index = (Map>) parser.map().get("index"); + assertThat(index.get("mappings"), hasKey("field")); + } + } + } + @Override protected GetFieldMappingsResponse createTestInstance() { return new GetFieldMappingsResponse(randomMapping()); @@ -61,7 +125,7 @@ private Map> randomMapping() { Map> mappings = new HashMap<>(); int indices = randomInt(10); - for(int i = 0; i < indices; i++) { + for (int i = 0; i < indices; i++) { Map fieldMappings = new HashMap<>(); int fields = randomInt(10); for (int k = 0; k < fields; k++) { From 4dcd63ca39272620dd44cc4c78b81b79c19a4d0a Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 13:54:39 +0200 Subject: [PATCH 13/35] unit tests --- .../indices/rollover/RolloverRequest.java | 4 +- .../index/mapper/MapperService.java | 5 ++ .../get/GetFieldMappingsResponseTests.java | 16 ++--- .../rollover/RolloverRequestTests.java | 45 ++++++++++++ .../indices/RestGetIndicesActionTests.java | 71 +++++++++++++++++++ .../RestPutIndexTemplateActionTests.java | 69 ++++++++++++++++++ 6 files changed, 198 insertions(+), 12 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java create mode 100644 server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 44626a6af41bd..b2d49b85411e0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -77,7 +77,7 @@ public class RolloverRequest extends AcknowledgedRequest implem } else { // a type is not included, add a dummy _doc type Map mappings = parser.map(); - if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { + if (MapperService.isMappingSourceTyped(mappings)) { throw new IllegalArgumentException("The mapping definition cannot be nested under a type " + "[" + MapperService.SINGLE_MAPPING_NAME + "] unless include_type_name is set to true."); } @@ -87,7 +87,7 @@ public class RolloverRequest extends AcknowledgedRequest implem PARSER.declareField((parser, request, context) -> { // a type is not included, add a dummy _doc type Map mappings = parser.map(); - if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { + if (MapperService.isMappingSourceTyped(mappings)) { throw new IllegalArgumentException("The mapping definition cannot be nested under a type"); } request.createIndexRequest.mapping(mappings); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 83937f19468ed..6735281819fd1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -333,6 +333,11 @@ public static boolean isMappingSourceTyped(String type, Map mapp return mapping.size() == 1 && mapping.keySet().iterator().next().equals(type); } + public static boolean isMappingSourceTyped(Map mapping) { + return mapping.size() == 1 && + mapping.keySet().iterator().next().equals("properties") == false; + } + /** * Resolves a type from a mapping-related request into the type that should be used when * merging and updating mappings. diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index d04a565827892..1b235c50e3a0a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.admin.indices.mapping.get; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata; -import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -17,23 +16,19 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; public class GetFieldMappingsResponseTests extends AbstractWireSerializingTestCase { @@ -75,7 +70,8 @@ public void testToXContentIncludesType() throws Exception { response.toXContent(builder, params); try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - Map>> index = (Map>>) parser.map().get("index"); + Map>> index = + (Map>>) parser.map().get("index"); assertThat(index.get("mappings"), hasKey(MapperService.SINGLE_MAPPING_NAME)); assertThat(index.get("mappings").get(MapperService.SINGLE_MAPPING_NAME), hasKey("field")); } @@ -90,8 +86,8 @@ public void testToXContentIncludesType() throws Exception { assertThat(index.get("mappings"), hasKey("field")); } } - //v8 does not have _doc, een when include_type_name is present - // (although this throws unconsumed parameter exception in RestGetFieldMappings) + //v8 does not have _doc, even when include_type_name is present + // (although this throws unconsumed parameter exception in RestGetFieldMappingsAction) try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) { response.toXContent(builder, params); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index 71156785b66ca..4dc413b5fb834 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -23,7 +24,9 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParseException; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.test.ESTestCase; @@ -186,6 +189,48 @@ public void testValidation() { assertEquals("rollover target is missing", validationException.validationErrors().get(0)); } + public void testParsingWithType() throws Exception { + final XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("conditions") + .field("max_age", "10d") + .field("max_docs", 100) + .endObject() + .startObject("mappings") + .startObject("type1") //TO ASK did we allow types or only _doc? + .startObject("properties") + .startObject("field1") + .field("type", "string") + .field("index", "not_analyzed") + .endObject() + .endObject() + .endObject() + .endObject() + .startObject("settings") + .field("number_of_shards", 10) + .endObject() + .startObject("aliases") + .startObject("alias1").endObject() + .endObject() + .endObject(); + + try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, + BytesReference.bytes(builder).utf8ToString(), RestApiVersion.V_7)) { + final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); + request.fromXContent(true, parser); + Map> conditions = request.getConditions(); + assertThat(conditions.size(), equalTo(2)); + assertThat(request.getCreateIndexRequest().mappings(), + equalTo("{\"_doc\":{\"properties\":{\"field1\":{\"index\":\"not_analyzed\",\"type\":\"string\"}}}}")); + } + + try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, + BytesReference.bytes(builder).utf8ToString(), RestApiVersion.V_7)) { + final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); + expectThrows(IllegalArgumentException.class, () -> request.fromXContent(false, parser)); + } + } + private static List> conditionsGenerator = new ArrayList<>(); static { conditionsGenerator.add((request) -> request.addMaxIndexDocsCondition(randomNonNegativeLong())); diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java new file mode 100644 index 0000000000000..38e1c6d6cc13d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesActionTests.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; +import static org.mockito.Mockito.mock; + +public class RestGetIndicesActionTests extends ESTestCase { + final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); + + /** + * Test that setting the "include_type_name" parameter raises a warning for the GET request + */ + public void testIncludeTypeNamesWarning() throws IOException { + Map params = new HashMap<>(); + params.put(INCLUDE_TYPE_NAME_PARAMETER, randomFrom("true", "false")); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.GET) + .withPath("/some_index") + .withParams(params) + .build(); + + RestGetIndicesAction handler = new RestGetIndicesAction(); + handler.prepareRequest(request, mock(NodeClient.class)); + assertWarnings(RestGetIndicesAction.TYPES_DEPRECATION_MESSAGE); + + // the same request without the parameter should pass without warning + request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.GET) + .withPath("/some_index") + .build(); + handler.prepareRequest(request, mock(NodeClient.class)); + } + + /** + * Test that setting the "include_type_name" parameter doesn't raises a warning if the HEAD method is used (indices.exists) + */ + public void testIncludeTypeNamesWarningExists() throws IOException { + Map params = new HashMap<>(); + params.put(INCLUDE_TYPE_NAME_PARAMETER, randomFrom("true", "false")); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.HEAD) + .withPath("/some_index") + .withParams(params) + .build(); + + RestGetIndicesAction handler = new RestGetIndicesAction(); + handler.prepareRequest(request, mock(NodeClient.class)); + } +} diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java new file mode 100644 index 0000000000000..1308d5762f8e3 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.rest.action.admin.indices; + + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; +import static org.mockito.Mockito.mock; + +public class RestPutIndexTemplateActionTests extends ESTestCase { + final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); + + private RestPutIndexTemplateAction action; + + @Before + public void setUpAction() { + action = new RestPutIndexTemplateAction(); + } + + public void testIncludeTypeName() throws IOException { + XContentBuilder typedContent = XContentFactory.jsonBuilder().startObject() + .startObject("mappings") + .startObject("my_doc") + .startObject("properties") + .startObject("field1").field("type", "keyword").endObject() + .startObject("field2").field("type", "text").endObject() + .endObject() + .endObject() + .endObject() + .startObject("aliases") + .startObject("read_alias").endObject() + .endObject() + .endObject(); + + Map params = new HashMap<>(); + params.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) + .withHeaders(Map.of("Content-Type", contentTypeHeader, "Accept", contentTypeHeader)) + .withMethod(RestRequest.Method.PUT) + .withParams(params) + .withPath("/_template/_some_template") + .withContent(BytesReference.bytes(typedContent), XContentType.JSON) + .build(); + action.prepareRequest(request, mock(NodeClient.class)); + assertWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE); + } +} From b384677fa88da43c180921562e4d3bfe75774a39 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 14:20:41 +0200 Subject: [PATCH 14/35] enable xpack compat tests --- x-pack/plugin/build.gradle | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index ffdf1924bdef3..380d0f8320963 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -4,7 +4,7 @@ import org.elasticsearch.gradle.util.GradleUtils apply plugin: 'elasticsearch.yaml-rest-test' apply plugin: 'elasticsearch.validate-rest-spec' apply plugin: 'elasticsearch.internal-test-artifact' -//apply plugin: 'elasticsearch.yaml-rest-compat-test' +apply plugin: 'elasticsearch.yaml-rest-compat-test' archivesBaseName = 'x-pack' @@ -15,7 +15,7 @@ dependencies { // let the yamlRestTest see the classpath of test GradleUtils.extendSourceSet(project, "test", "yamlRestTest", tasks.named("yamlRestTest")) -//GradleUtils.extendSourceSet(project, "test", "yamlRestCompatTest", tasks.named("yamlRestCompatTest")) +GradleUtils.extendSourceSet(project, "test", "yamlRestCompatTest") restResources { restApi { @@ -132,17 +132,3 @@ tasks.register('enforceYamlTestConvention').configure { tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } -//tasks.named("yamlRestCompatTest").configure { -//// onlyIf { -//// // Skip these tests on Windows since the blacklist exceeds Windows CLI limits -//// OS.current() != OS.WINDOWS -//// } -// systemProperty 'tests.rest.blacklist', [ -// -// ] -//} -// -//tasks.named("transformV7RestTests").configure({ task -> -// task.replaceMatch("_type", "_doc") -// task.addAllowedWarningRegex("\\[types removal\\].*") -//}) From 0cc442767002652c955418837c79caeb01b6eb84 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 15:00:08 +0200 Subject: [PATCH 15/35] small cleanups --- rest-api-spec/build.gradle | 2 -- .../action/admin/indices/get/GetIndexResponse.java | 5 ++--- .../indices/mapping/get/GetFieldMappingsResponse.java | 5 ++--- .../indices/template/get/GetIndexTemplatesResponse.java | 9 ++------- .../org/elasticsearch/index/mapper/MapperService.java | 5 ++++- .../action/admin/indices/RestGetFieldMappingAction.java | 6 +----- .../action/admin/indices/RestGetIndexTemplateAction.java | 1 - .../action/admin/indices/RestPutIndexTemplateAction.java | 2 +- .../action/admin/indices/RestRolloverIndexAction.java | 2 +- 9 files changed, 13 insertions(+), 24 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 40538c28a2ff4..b74cd818ae799 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -206,8 +206,6 @@ tasks.named("yamlRestCompatTest").configure { 'indices.get_mapping/40_aliases/Getting mapping for aliases should return the real index as key', 'indices.get_mapping/61_empty_with_types/Check empty mapping when getting all mappings via /_mapping', 'indices.get_mapping/70_mix_typeless_typeful/GET mapping with typeless API on an index that has types', -// 'indices.get_template/11_basic_with_types/Get template', -// 'indices.get_template/11_basic_with_types/Get template with no mappings', 'indices.open/10_basic/?wait_for_active_shards default is deprecated', 'indices.open/10_basic/?wait_for_active_shards=index-setting', 'indices.put_mapping/10_basic/Put mappings with explicit _doc type', diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index b6d9beb29b4ba..74fae9eaacfbd 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -196,9 +196,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (indexMappings == null) { builder.startObject("mappings").endObject(); } else { - boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, - DEFAULT_INCLUDE_TYPE_NAME_POLICY); - if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName) { + if (builder.getRestApiVersion() == RestApiVersion.V_7 && + params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) { builder.startObject("mappings"); builder.field(MapperService.SINGLE_MAPPING_NAME, indexMappings.sourceAsMap()); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index 254dbed30c26f..bb1443d9d76f3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -87,15 +87,14 @@ public FieldMappingMetadata fieldMappings(String index, String field) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - boolean includeTypeName = params.paramAsBoolean(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, - DEFAULT_INCLUDE_TYPE_NAME_POLICY); builder.startObject(); for (Map.Entry> indexEntry : mappings.entrySet()) { builder.startObject(indexEntry.getKey()); builder.startObject(MAPPINGS.getPreferredName()); if (indexEntry.getValue() != null) { - if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName) { + if (builder.getRestApiVersion() == RestApiVersion.V_7 && + params.paramAsBoolean(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) { builder.startObject(MapperService.SINGLE_MAPPING_NAME); addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index f3332b734080f..f54fc070a3425 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -66,15 +66,10 @@ public int hashCode() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { params = new ToXContent.DelegatingMapParams(singletonMap("reduce_mappings", "true"), params); - boolean includeTypeName = false; - if(builder.getRestApiVersion() == RestApiVersion.V_7) { - includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, - DEFAULT_INCLUDE_TYPE_NAME_POLICY); - } - builder.startObject(); for (IndexTemplateMetadata indexTemplateMetadata : getIndexTemplates()) { - if (includeTypeName) { + if(builder.getRestApiVersion() == RestApiVersion.V_7 && + params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) { IndexTemplateMetadata.Builder.toXContentWithTypes(indexTemplateMetadata, builder, params); } else { IndexTemplateMetadata.Builder.toXContent(indexTemplateMetadata, builder, params); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 6735281819fd1..82d7a49531dd0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -332,7 +332,10 @@ public DocumentMapper documentMapper() { public static boolean isMappingSourceTyped(String type, Map mapping) { return mapping.size() == 1 && mapping.keySet().iterator().next().equals(type); } - + /** + * Returns {@code true} if the given {@code mappingSource} includes any type (anything other than properties) + * as a top-level object. + */ public static boolean isMappingSourceTyped(Map mapping) { return mapping.size() == 1 && mapping.keySet().iterator().next().equals("properties") == false; diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index 25283a235ef33..530198a5b7136 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -59,12 +59,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] fields = Strings.splitStringByCommaToArray(request.param("fields")); if(request.getRestApiVersion() == RestApiVersion.V_7){ - boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); -// if (includeTypeName == false /*&& types.length > 0*/) { -// throw new IllegalArgumentException("Types cannot be specified unless include_type_name" + -// " is set to true."); -// } if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); deprecationLogger.compatibleApiWarning("get_field_mapping_with_types", TYPES_DEPRECATION_MESSAGE); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index fa51cd4bf96f2..b3379899a6f40 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -58,7 +58,6 @@ public String getName() { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { if(request.getRestApiVersion() == RestApiVersion.V_7) { if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - request.param(INCLUDE_TYPE_NAME_PARAMETER); deprecationLogger.compatibleApiWarning("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index 12ab2346e2107..337aedc7ea696 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -56,10 +56,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); if(request.getRestApiVersion() == RestApiVersion.V_7) { - boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER) ) { deprecationLogger.compatibleApiWarning("put_index_template_with_types", TYPES_DEPRECATION_MESSAGE); } + boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); if(includeTypeName) { sourceAsMap = RestCreateIndexAction.prepareMappingsV7(sourceAsMap, request); } else { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index f6f8e450e4838..0c4cf0c9e2f5a 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -56,10 +56,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC private boolean isIncludeTypeName(RestRequest request) { boolean includeTypeName = false; if (request.getRestApiVersion() == RestApiVersion.V_7) { - includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { deprecationLogger.compatibleApiWarning("index_rollover_with_types", TYPES_DEPRECATION_MESSAGE); } + includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); } return includeTypeName; } From 64c7e90bd2c9d86ea4603fc881e41b4d5e065838 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 31 Mar 2021 15:03:58 +0200 Subject: [PATCH 16/35] fix test --- .../action/admin/indices/RestPutIndexTemplateActionTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java index 1308d5762f8e3..4fc39dde435b8 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java @@ -30,7 +30,7 @@ import static org.mockito.Mockito.mock; public class RestPutIndexTemplateActionTests extends ESTestCase { - final List contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7)); + final List contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7)); private RestPutIndexTemplateAction action; @@ -61,7 +61,7 @@ public void testIncludeTypeName() throws IOException { .withMethod(RestRequest.Method.PUT) .withParams(params) .withPath("/_template/_some_template") - .withContent(BytesReference.bytes(typedContent), XContentType.JSON) + .withContent(BytesReference.bytes(typedContent), null) .build(); action.prepareRequest(request, mock(NodeClient.class)); assertWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE); From 734e8b8df910372e16e63d23624b5d63e5ffe7cf Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 2 Apr 2021 10:02:07 +0200 Subject: [PATCH 17/35] add blacklist --- x-pack/plugin/build.gradle | 72 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 380d0f8320963..17a481ec32494 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -132,3 +132,75 @@ tasks.register('enforceYamlTestConvention').configure { tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } + +tasks.named("yamlRestCompatTest").configure { + + systemProperty 'tests.rest.blacklist', [ + 'vectors/30_sparse_vector_basic/Cosine Similarity', + 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', + 'ml/datafeeds_crud/Test update datafeed to point to different job', + 'roles/11_idx_arrays/Test put role api using as array of index names', + 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', + 'sql/sql/Clean cursor', + 'ml/jobs_get_stats/Test no exception on get job stats with missing index', + 'sql/sql/Paging through results', + 'privileges/11_builtin/Test get builtin privileges', + 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', + 'ml/datafeeds_crud/Test update datafeed to point to missing job', + 'sql/translate/Translate SQL', + 'vectors/15_dense_vector_l1l2/L2 norm', + 'ml/datafeed_cat_apis/Test cat datafeeds', + 'transform/transforms_cat_apis/Test cat transform stats hiding headers', + 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', + 'rollup/get_jobs/Test basic get_jobs', + 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', + 'vectors/30_sparse_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Dot Product', + 'vectors/10_dense_vector_basic/Cosine Similarity', + 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', + 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', + 'ml/post_data/Test flush with skip_time', + 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', + 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', + 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', + 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', + 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', + 'rollup/rollup_search/Obsolete BWC Timezone', + 'transform/transforms_cat_apis/Test cat transform stats with column selection', + 'security/authz/14_cat_indices/Test empty request while no-authorized index', + 'vectors/35_sparse_vector_l1l2/L1 norm', + 'rollup/put_job/Test put_job in non-rollup index', + 'security/authz/14_cat_indices/Test empty request while single authorized index', + 'vectors/20_dense_vector_special_cases/Documents missing a vector field', + 'vectors/10_dense_vector_basic/Dot Product', + 'rollup/rollup_search/Obsolete Timezone', + 'rollup/put_job/Test basic put_job', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', + 'sql/sql/Getting textual representation', + 'ml/trained_model_cat_apis/Test cat trained models', + 'rollup/start_job/Test start job twice', + 'ml/job_cat_apis/Test cat anomaly detector jobs', + 'vectors/50_vector_stats/Usage stats on vector fields', + 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', + 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', + 'vectors/10_dense_vector_basic/Deprecated function signature', + 'vectors/15_dense_vector_l1l2/L1 norm', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', + 'rollup/delete_job/Test delete running job', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', + 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', + 'rollup/delete_job/Test basic delete_job', + 'license/30_enterprise_license/Installing enterprise license', + 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', + 'sql/sql/Execute some SQL', + 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', + 'rollup/delete_job/Test delete job twice', + 'vectors/35_sparse_vector_l1l2/L2 norm', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', + 'roles/30_prohibited_role_query/Test use prohibited query inside role query', + 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', + 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', + 'ml/jobs_get_stats/Test get job stats for closed job', + 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS' + ] +} From 276fc88be5705170eb7177ad48d7504cd2e1e10f Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 2 Apr 2021 10:05:05 +0200 Subject: [PATCH 18/35] sort and join --- x-pack/plugin/build.gradle | 110 ++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 17a481ec32494..f88be8afa01d9 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -136,71 +136,71 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTest").configure { systemProperty 'tests.rest.blacklist', [ - 'vectors/30_sparse_vector_basic/Cosine Similarity', - 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', + 'license/30_enterprise_license/Installing enterprise license', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', + 'ml/datafeed_cat_apis/Test cat datafeeds', 'ml/datafeeds_crud/Test update datafeed to point to different job', - 'roles/11_idx_arrays/Test put role api using as array of index names', - 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', - 'sql/sql/Clean cursor', - 'ml/jobs_get_stats/Test no exception on get job stats with missing index', - 'sql/sql/Paging through results', - 'privileges/11_builtin/Test get builtin privileges', - 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', + 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', 'ml/datafeeds_crud/Test update datafeed to point to missing job', - 'sql/translate/Translate SQL', - 'vectors/15_dense_vector_l1l2/L2 norm', - 'ml/datafeed_cat_apis/Test cat datafeeds', - 'transform/transforms_cat_apis/Test cat transform stats hiding headers', - 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', - 'rollup/get_jobs/Test basic get_jobs', + 'ml/job_cat_apis/Test cat anomaly detector jobs', 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', - 'vectors/30_sparse_vector_basic/Deprecated function signature', - 'vectors/30_sparse_vector_basic/Dot Product', - 'vectors/10_dense_vector_basic/Cosine Similarity', - 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', - 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', + 'ml/jobs_get_stats/Test get job stats for closed job', + 'ml/jobs_get_stats/Test no exception on get job stats with missing index', + 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', 'ml/post_data/Test flush with skip_time', + 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', + 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', - 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', - 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', - 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', - 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', - 'rollup/rollup_search/Obsolete BWC Timezone', - 'transform/transforms_cat_apis/Test cat transform stats with column selection', - 'security/authz/14_cat_indices/Test empty request while no-authorized index', - 'vectors/35_sparse_vector_l1l2/L1 norm', + 'ml/trained_model_cat_apis/Test cat trained models', + 'privileges/11_builtin/Test get builtin privileges', + 'roles/11_idx_arrays/Test put role api using as array of index names', + 'roles/30_prohibited_role_query/Test use prohibited query inside role query', + 'rollup/delete_job/Test basic delete_job', + 'rollup/delete_job/Test delete job twice', + 'rollup/delete_job/Test delete running job', + 'rollup/get_jobs/Test basic get_jobs', + 'rollup/put_job/Test basic put_job', 'rollup/put_job/Test put_job in non-rollup index', - 'security/authz/14_cat_indices/Test empty request while single authorized index', - 'vectors/20_dense_vector_special_cases/Documents missing a vector field', - 'vectors/10_dense_vector_basic/Dot Product', + 'rollup/rollup_search/Obsolete BWC Timezone', 'rollup/rollup_search/Obsolete Timezone', - 'rollup/put_job/Test basic put_job', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', - 'sql/sql/Getting textual representation', - 'ml/trained_model_cat_apis/Test cat trained models', 'rollup/start_job/Test start job twice', - 'ml/job_cat_apis/Test cat anomaly detector jobs', - 'vectors/50_vector_stats/Usage stats on vector fields', - 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', - 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', - 'vectors/10_dense_vector_basic/Deprecated function signature', - 'vectors/15_dense_vector_l1l2/L1 norm', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', - 'rollup/delete_job/Test delete running job', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', + 'security/authz/14_cat_indices/Test empty request while no-authorized index', + 'security/authz/14_cat_indices/Test empty request while single authorized index', 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', - 'rollup/delete_job/Test basic delete_job', - 'license/30_enterprise_license/Installing enterprise license', - 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', + 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', + 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', + 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS' + 'sql/sql/Clean cursor', 'sql/sql/Execute some SQL', + 'sql/sql/Getting textual representation', + 'sql/sql/Paging through results', + 'sql/translate/Translate SQL', + 'transform/transforms_cat_apis/Test cat transform stats hiding headers', + 'transform/transforms_cat_apis/Test cat transform stats with column selection', 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', - 'rollup/delete_job/Test delete job twice', + 'vectors/10_dense_vector_basic/Cosine Similarity', + 'vectors/10_dense_vector_basic/Deprecated function signature', + 'vectors/10_dense_vector_basic/Dot Product', + 'vectors/15_dense_vector_l1l2/L1 norm', + 'vectors/15_dense_vector_l1l2/L2 norm', + 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', + 'vectors/20_dense_vector_special_cases/Documents missing a vector field', + 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', + 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', + 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', + 'vectors/30_sparse_vector_basic/Cosine Similarity', + 'vectors/30_sparse_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Dot Product', + 'vectors/35_sparse_vector_l1l2/L1 norm', 'vectors/35_sparse_vector_l1l2/L2 norm', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', - 'roles/30_prohibited_role_query/Test use prohibited query inside role query', - 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', - 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', - 'ml/jobs_get_stats/Test get job stats for closed job', - 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS' - ] + 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', + 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', + 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', + 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', + 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', + 'vectors/50_vector_stats/Usage stats on vector fields', + ].join(',') } From 07db905c54d9b3f9598825321808694e3af49788 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 2 Apr 2021 11:21:45 +0200 Subject: [PATCH 19/35] fix build.gradle --- x-pack/plugin/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index f88be8afa01d9..77b285921b285 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -172,7 +172,7 @@ tasks.named("yamlRestCompatTest").configure { 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', - 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS' + 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS', 'sql/sql/Clean cursor', 'sql/sql/Execute some SQL', 'sql/sql/Getting textual representation', From bc0e5df6026c74f934c7aade20c3c0deb2cf766c Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 2 Apr 2021 12:34:03 +0200 Subject: [PATCH 20/35] enable all tests --- x-pack/plugin/build.gradle | 147 +++++++++++++++++++------------------ 1 file changed, 76 insertions(+), 71 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 77b285921b285..4f263f3de0631 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -133,74 +133,79 @@ tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } -tasks.named("yamlRestCompatTest").configure { - - systemProperty 'tests.rest.blacklist', [ - 'license/30_enterprise_license/Installing enterprise license', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', - 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', - 'ml/datafeed_cat_apis/Test cat datafeeds', - 'ml/datafeeds_crud/Test update datafeed to point to different job', - 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', - 'ml/datafeeds_crud/Test update datafeed to point to missing job', - 'ml/job_cat_apis/Test cat anomaly detector jobs', - 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', - 'ml/jobs_get_stats/Test get job stats for closed job', - 'ml/jobs_get_stats/Test no exception on get job stats with missing index', - 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', - 'ml/post_data/Test flush with skip_time', - 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', - 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', - 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', - 'ml/trained_model_cat_apis/Test cat trained models', - 'privileges/11_builtin/Test get builtin privileges', - 'roles/11_idx_arrays/Test put role api using as array of index names', - 'roles/30_prohibited_role_query/Test use prohibited query inside role query', - 'rollup/delete_job/Test basic delete_job', - 'rollup/delete_job/Test delete job twice', - 'rollup/delete_job/Test delete running job', - 'rollup/get_jobs/Test basic get_jobs', - 'rollup/put_job/Test basic put_job', - 'rollup/put_job/Test put_job in non-rollup index', - 'rollup/rollup_search/Obsolete BWC Timezone', - 'rollup/rollup_search/Obsolete Timezone', - 'rollup/start_job/Test start job twice', - 'security/authz/14_cat_indices/Test empty request while no-authorized index', - 'security/authz/14_cat_indices/Test empty request while single authorized index', - 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', - 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', - 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', - 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS', - 'sql/sql/Clean cursor', - 'sql/sql/Execute some SQL', - 'sql/sql/Getting textual representation', - 'sql/sql/Paging through results', - 'sql/translate/Translate SQL', - 'transform/transforms_cat_apis/Test cat transform stats hiding headers', - 'transform/transforms_cat_apis/Test cat transform stats with column selection', - 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', - 'vectors/10_dense_vector_basic/Cosine Similarity', - 'vectors/10_dense_vector_basic/Deprecated function signature', - 'vectors/10_dense_vector_basic/Dot Product', - 'vectors/15_dense_vector_l1l2/L1 norm', - 'vectors/15_dense_vector_l1l2/L2 norm', - 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', - 'vectors/20_dense_vector_special_cases/Documents missing a vector field', - 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', - 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', - 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', - 'vectors/30_sparse_vector_basic/Cosine Similarity', - 'vectors/30_sparse_vector_basic/Deprecated function signature', - 'vectors/30_sparse_vector_basic/Dot Product', - 'vectors/35_sparse_vector_l1l2/L1 norm', - 'vectors/35_sparse_vector_l1l2/L2 norm', - 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', - 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', - 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', - 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', - 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', - 'vectors/50_vector_stats/Usage stats on vector fields', - ].join(',') -} +//tasks.named("yamlRestCompatTest").configure { +// +// systemProperty 'tests.rest.blacklist', [ +// 'license/30_enterprise_license/Installing enterprise license', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', +// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', +// 'ml/datafeed_cat_apis/Test cat datafeeds', +// 'ml/datafeeds_crud/Test update datafeed to point to different job', +// 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', +// 'ml/datafeeds_crud/Test update datafeed to point to missing job', +// 'ml/job_cat_apis/Test cat anomaly detector jobs', +// 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', +// 'ml/jobs_get_stats/Test get job stats for closed job', +// 'ml/jobs_get_stats/Test no exception on get job stats with missing index', +// 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', +// 'ml/post_data/Test flush with skip_time', +// 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', +// 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', +// 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', +// 'ml/trained_model_cat_apis/Test cat trained models', +// 'privileges/11_builtin/Test get builtin privileges', +// 'roles/11_idx_arrays/Test put role api using as array of index names', +// 'roles/30_prohibited_role_query/Test use prohibited query inside role query', +// 'rollup/delete_job/Test basic delete_job', +// 'rollup/delete_job/Test delete job twice', +// 'rollup/delete_job/Test delete running job', +// 'rollup/get_jobs/Test basic get_jobs', +// 'rollup/put_job/Test basic put_job', +// 'rollup/put_job/Test put_job in non-rollup index', +// 'rollup/rollup_search/Obsolete BWC Timezone', +// 'rollup/rollup_search/Obsolete Timezone', +// 'rollup/start_job/Test start job twice', +// 'security/authz/14_cat_indices/Test empty request while no-authorized index', +// 'security/authz/14_cat_indices/Test empty request while single authorized index', +// 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', +// 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', +// 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', +// 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS', +// 'sql/sql/Clean cursor', +// 'sql/sql/Execute some SQL', +// 'sql/sql/Getting textual representation', +// 'sql/sql/Paging through results', +// 'sql/translate/Translate SQL', +// 'transform/transforms_cat_apis/Test cat transform stats hiding headers', +// 'transform/transforms_cat_apis/Test cat transform stats with column selection', +// 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', +// 'vectors/10_dense_vector_basic/Cosine Similarity', +// 'vectors/10_dense_vector_basic/Deprecated function signature', +// 'vectors/10_dense_vector_basic/Dot Product', +// 'vectors/15_dense_vector_l1l2/L1 norm', +// 'vectors/15_dense_vector_l1l2/L2 norm', +//// 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', +//// 'vectors/20_dense_vector_special_cases/Documents missing a vector field', +//// 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', +//// 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', +//// 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', +// 'vectors/30_sparse_vector_basic/Cosine Similarity', +// 'vectors/30_sparse_vector_basic/Deprecated function signature', +// 'vectors/30_sparse_vector_basic/Dot Product', +// 'vectors/35_sparse_vector_l1l2/L1 norm', +// 'vectors/35_sparse_vector_l1l2/L2 norm', +// 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', +// 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', +// 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', +// 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', +// 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', +// 'vectors/50_vector_stats/Usage stats on vector fields', +// ].join(',') +//} + +tasks.named("transformV7RestTests").configure({ task -> + task.replaceMatch("_type", "_doc") + task.addAllowedWarningRegex("\\[types removal\\].*") +}) From 0835b800c8a013550eb4fba58b2f3add2ffbe9f3 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 2 Apr 2021 12:58:54 +0200 Subject: [PATCH 21/35] fix block list --- x-pack/plugin/build.gradle | 132 +++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 71 deletions(-) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 4f263f3de0631..c5fb427a1ead4 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -133,77 +133,67 @@ tasks.named("precommit").configure { dependsOn 'enforceYamlTestConvention', 'enforceApiSpecsConvention' } -//tasks.named("yamlRestCompatTest").configure { -// -// systemProperty 'tests.rest.blacklist', [ -// 'license/30_enterprise_license/Installing enterprise license', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', -// 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', -// 'ml/datafeed_cat_apis/Test cat datafeeds', -// 'ml/datafeeds_crud/Test update datafeed to point to different job', -// 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', -// 'ml/datafeeds_crud/Test update datafeed to point to missing job', -// 'ml/job_cat_apis/Test cat anomaly detector jobs', -// 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', -// 'ml/jobs_get_stats/Test get job stats for closed job', -// 'ml/jobs_get_stats/Test no exception on get job stats with missing index', -// 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', -// 'ml/post_data/Test flush with skip_time', -// 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', -// 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', -// 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', -// 'ml/trained_model_cat_apis/Test cat trained models', -// 'privileges/11_builtin/Test get builtin privileges', -// 'roles/11_idx_arrays/Test put role api using as array of index names', -// 'roles/30_prohibited_role_query/Test use prohibited query inside role query', -// 'rollup/delete_job/Test basic delete_job', -// 'rollup/delete_job/Test delete job twice', -// 'rollup/delete_job/Test delete running job', -// 'rollup/get_jobs/Test basic get_jobs', -// 'rollup/put_job/Test basic put_job', -// 'rollup/put_job/Test put_job in non-rollup index', -// 'rollup/rollup_search/Obsolete BWC Timezone', -// 'rollup/rollup_search/Obsolete Timezone', -// 'rollup/start_job/Test start job twice', -// 'security/authz/14_cat_indices/Test empty request while no-authorized index', -// 'security/authz/14_cat_indices/Test empty request while single authorized index', -// 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', -// 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', -// 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', -// 'set_security_user/10_small_users_one_index/Test shared index separating user by using DLS', -// 'sql/sql/Clean cursor', -// 'sql/sql/Execute some SQL', -// 'sql/sql/Getting textual representation', -// 'sql/sql/Paging through results', -// 'sql/translate/Translate SQL', -// 'transform/transforms_cat_apis/Test cat transform stats hiding headers', -// 'transform/transforms_cat_apis/Test cat transform stats with column selection', -// 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', -// 'vectors/10_dense_vector_basic/Cosine Similarity', -// 'vectors/10_dense_vector_basic/Deprecated function signature', -// 'vectors/10_dense_vector_basic/Dot Product', -// 'vectors/15_dense_vector_l1l2/L1 norm', -// 'vectors/15_dense_vector_l1l2/L2 norm', -//// 'vectors/20_dense_vector_special_cases/Dense vectors should error with sparse vector functions', -//// 'vectors/20_dense_vector_special_cases/Documents missing a vector field', -//// 'vectors/20_dense_vector_special_cases/Functions with query vectors with dims different from docs vectors should error', -//// 'vectors/20_dense_vector_special_cases/Indexing of Dense vectors should error when dims don\'t match defined in the mapping', -//// 'vectors/20_dense_vector_special_cases/Vectors of mixed integers and floats', -// 'vectors/30_sparse_vector_basic/Cosine Similarity', -// 'vectors/30_sparse_vector_basic/Deprecated function signature', -// 'vectors/30_sparse_vector_basic/Dot Product', -// 'vectors/35_sparse_vector_l1l2/L1 norm', -// 'vectors/35_sparse_vector_l1l2/L2 norm', -// 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', -// 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', -// 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', -// 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', -// 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', -// 'vectors/50_vector_stats/Usage stats on vector fields', -// ].join(',') -//} +tasks.named("yamlRestCompatTest").configure { + + systemProperty 'tests.rest.blacklist', [ + 'license/30_enterprise_license/Installing enterprise license', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header and column selection', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics all jobs with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header', + 'ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job', + 'ml/datafeed_cat_apis/Test cat datafeeds', + 'ml/datafeeds_crud/Test update datafeed to point to different job', + 'ml/datafeeds_crud/Test update datafeed to point to job already attached to another datafeed', + 'ml/datafeeds_crud/Test update datafeed to point to missing job', + 'ml/job_cat_apis/Test cat anomaly detector jobs', + 'ml/jobs_get_stats/Test get job stats after uploading data prompting the creation of some stats', + 'ml/jobs_get_stats/Test get job stats for closed job', + 'ml/jobs_get_stats/Test no exception on get job stats with missing index', + 'ml/post_data/Test POST data job api, flush, close and verify DataCounts doc', + 'ml/post_data/Test flush with skip_time', + 'ml/set_upgrade_mode/Setting upgrade_mode to enabled', + 'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false', + 'ml/trained_model_cat_apis/Test cat trained models', + 'privileges/11_builtin/Test get builtin privileges', + 'roles/11_idx_arrays/Test put role api using as array of index names', + 'roles/30_prohibited_role_query/Test use prohibited query inside role query', + 'rollup/delete_job/Test basic delete_job', + 'rollup/delete_job/Test delete job twice', + 'rollup/delete_job/Test delete running job', + 'rollup/get_jobs/Test basic get_jobs', + 'rollup/put_job/Test basic put_job', + 'rollup/put_job/Test put_job in non-rollup index', + 'rollup/rollup_search/Obsolete BWC Timezone', + 'rollup/rollup_search/Obsolete Timezone', + 'rollup/start_job/Test start job twice', + 'security/authz/14_cat_indices/Test empty request while no-authorized index', + 'security/authz/14_cat_indices/Test empty request while single authorized index', + 'security/authz/14_cat_indices/Test explicit request while multiple authorized indices', + 'security/authz/14_cat_indices/Test explicit request while multiple opened/closed authorized indices', + 'security/authz/14_cat_indices/Test wildcard request with multiple authorized indices', + 'sql/sql/Clean cursor', + 'sql/sql/Execute some SQL', + 'sql/sql/Getting textual representation', + 'sql/sql/Paging through results', + 'sql/translate/Translate SQL', + 'transform/transforms_cat_apis/Test cat transform stats hiding headers', + 'transform/transforms_cat_apis/Test cat transform stats with column selection', + 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', + 'vectors/10_dense_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Cosine Similarity', + 'vectors/30_sparse_vector_basic/Deprecated function signature', + 'vectors/30_sparse_vector_basic/Dot Product', + 'vectors/35_sparse_vector_l1l2/L1 norm', + 'vectors/35_sparse_vector_l1l2/L2 norm', + 'vectors/40_sparse_vector_special_cases/Dimensions can be sorted differently', + 'vectors/40_sparse_vector_special_cases/Documents missing a vector field', + 'vectors/40_sparse_vector_special_cases/Query vector has different dimensions from documents\' vectors', + 'ml/set_upgrade_mode/Setting upgrade mode to disabled from enabled', + 'vectors/40_sparse_vector_special_cases/Sparse vectors should error with dense vector functions', + 'vectors/40_sparse_vector_special_cases/Vectors of different dimensions and data types', + 'vectors/50_vector_stats/Usage stats on vector fields' + ].join(',') +} tasks.named("transformV7RestTests").configure({ task -> task.replaceMatch("_type", "_doc") From 9e1644f8d4f2db1da95d85843d503ddc64a498b4 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 12 Apr 2021 15:19:53 +0200 Subject: [PATCH 22/35] review follow up. remove the type checks in rollover request --- .../rest/compat/RestCompatTestTransformTask.java | 9 --------- .../admin/indices/rollover/RolloverRequest.java | 14 ++------------ .../elasticsearch/index/mapper/MapperService.java | 8 -------- .../indices/rollover/RolloverRequestTests.java | 8 +------- 4 files changed, 3 insertions(+), 36 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 2476721ca1e36..1128417b9dc00 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -24,7 +24,6 @@ import org.elasticsearch.gradle.test.rest.transform.headers.InjectHeaders; import org.elasticsearch.gradle.test.rest.transform.match.AddMatch; import org.elasticsearch.gradle.test.rest.transform.match.RemoveMatch; -import org.elasticsearch.gradle.test.rest.transform.match.ReplaceTextual; import org.elasticsearch.gradle.test.rest.transform.match.ReplaceMatch; import org.elasticsearch.gradle.test.rest.transform.text.ReplaceIsFalse; import org.elasticsearch.gradle.test.rest.transform.text.ReplaceIsTrue; @@ -106,14 +105,6 @@ public void replaceMatch(String subKey, Object value) { transformations.add(new ReplaceMatch(subKey, MAPPER.convertValue(value, JsonNode.class))); } - public void replaceIsTrue(String subKey, Object value) { - transformations.add(new ReplaceTextual("is_true", subKey, MAPPER.convertValue(value, TextNode.class))); - } - - public void replaceIsFalse(String subKey, Object value) { - transformations.add(new ReplaceTextual("is_false", subKey, MAPPER.convertValue(value, TextNode.class))); - } - /** * Replaces all the values of a is_true assertion for all project REST tests. * For example "is_true": "value_to_replace" to "match": "value_replaced" diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index b2d49b85411e0..a5b9f4c98f3d8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.util.HashMap; @@ -76,21 +75,12 @@ public class RolloverRequest extends AcknowledgedRequest implem } } else { // a type is not included, add a dummy _doc type - Map mappings = parser.map(); - if (MapperService.isMappingSourceTyped(mappings)) { - throw new IllegalArgumentException("The mapping definition cannot be nested under a type " + - "[" + MapperService.SINGLE_MAPPING_NAME + "] unless include_type_name is set to true."); - } - request.createIndexRequest.mapping(mappings); + request.createIndexRequest.mapping(parser.map()); } }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> { // a type is not included, add a dummy _doc type - Map mappings = parser.map(); - if (MapperService.isMappingSourceTyped(mappings)) { - throw new IllegalArgumentException("The mapping definition cannot be nested under a type"); - } - request.createIndexRequest.mapping(mappings); + request.createIndexRequest.mapping(parser.map()); }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.V_8)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 82d7a49531dd0..83937f19468ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -332,14 +332,6 @@ public DocumentMapper documentMapper() { public static boolean isMappingSourceTyped(String type, Map mapping) { return mapping.size() == 1 && mapping.keySet().iterator().next().equals(type); } - /** - * Returns {@code true} if the given {@code mappingSource} includes any type (anything other than properties) - * as a top-level object. - */ - public static boolean isMappingSourceTyped(Map mapping) { - return mapping.size() == 1 && - mapping.keySet().iterator().next().equals("properties") == false; - } /** * Resolves a type from a mapping-related request into the type that should be used when diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index 4dc413b5fb834..8b4525d58a155 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -197,7 +197,7 @@ public void testParsingWithType() throws Exception { .field("max_docs", 100) .endObject() .startObject("mappings") - .startObject("type1") //TO ASK did we allow types or only _doc? + .startObject("type1") .startObject("properties") .startObject("field1") .field("type", "string") @@ -223,12 +223,6 @@ public void testParsingWithType() throws Exception { assertThat(request.getCreateIndexRequest().mappings(), equalTo("{\"_doc\":{\"properties\":{\"field1\":{\"index\":\"not_analyzed\",\"type\":\"string\"}}}}")); } - - try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, - BytesReference.bytes(builder).utf8ToString(), RestApiVersion.V_7)) { - final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); - expectThrows(IllegalArgumentException.class, () -> request.fromXContent(false, parser)); - } } private static List> conditionsGenerator = new ArrayList<>(); From 2049d0e95cb41a890fb8eef2b3902bb0263d2a4c Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 13 Apr 2021 09:27:15 +0200 Subject: [PATCH 23/35] bring back a check against type without include_type_name --- .../indices/rollover/RolloverRequest.java | 8 ++++++- .../rollover/RolloverRequestTests.java | 21 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index a5b9f4c98f3d8..22ff5921917c9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.util.HashMap; @@ -75,7 +76,12 @@ public class RolloverRequest extends AcknowledgedRequest implem } } else { // a type is not included, add a dummy _doc type - request.createIndexRequest.mapping(parser.map()); + Map mappings = parser.map(); + if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { + throw new IllegalArgumentException("The mapping definition cannot be nested under a type " + + "[" + MapperService.SINGLE_MAPPING_NAME + "] unless include_type_name is set to true."); + } + request.createIndexRequest.mapping(mappings); } }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index 8b4525d58a155..7269c68a3d3a8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -225,6 +225,27 @@ public void testParsingWithType() throws Exception { } } + public void testTypedRequestWithoutIncludeTypeName() throws IOException { + final XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("mappings") + .startObject("_doc") + .startObject("properties") + .startObject("field1") + .field("type", "string") + .field("index", "not_analyzed") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + try (XContentParser parser = createParserWithCompatibilityFor(JsonXContent.jsonXContent, + BytesReference.bytes(builder).utf8ToString(), RestApiVersion.V_7)) { + final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); + expectThrows(IllegalArgumentException.class, () -> request.fromXContent(false, parser)); + } + } + private static List> conditionsGenerator = new ArrayList<>(); static { conditionsGenerator.add((request) -> request.addMaxIndexDocsCondition(randomNonNegativeLong())); From a1ea94cb8080d6ce00cc95f57d11cd004704427b Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 13 Apr 2021 16:09:53 +0200 Subject: [PATCH 24/35] remove stale gradle transformations --- .../rest/transform/match/ReplaceTextual.java | 85 -------------- .../transform/match/ReplaceTextualTests.java | 107 ----------------- .../rest/transform/match/text_replace.yml | 110 ------------------ 3 files changed, 302 deletions(-) delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java delete mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java delete mode 100644 buildSrc/src/test/resources/rest/transform/match/text_replace.yml diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java deleted file mode 100644 index ad32bc3bb86c5..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextual.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.test.rest.transform.match; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import org.elasticsearch.gradle.test.rest.transform.RestTestContext; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentObject; -import org.gradle.api.tasks.Input; -import org.gradle.api.tasks.Internal; -import org.gradle.api.tasks.Optional; - -/** - * A transformation to replace the flat textual fields. - */ -public class ReplaceTextual implements RestTestTransformByParentObject { - private final String keyToReplaceName; - private final String valueToBeReplaced; - private final TextNode replacementNode; - private final String testName; - - public ReplaceTextual(String keyToReplaceName, String valueToBeReplaced, TextNode replacementNode) { - this.keyToReplaceName = keyToReplaceName; - this.valueToBeReplaced = valueToBeReplaced; - this.replacementNode = replacementNode; - this.testName = null; - } - - public ReplaceTextual(String keyToReplaceName, String valueToBeReplaced, TextNode replacementNode, String testName) { - this.keyToReplaceName = keyToReplaceName; - this.valueToBeReplaced = valueToBeReplaced; - this.replacementNode = replacementNode; - this.testName = testName; - } - - @Override - @Internal - public String getKeyToFind() { - return keyToReplaceName; - } - - @Override - public String requiredChildKey() { - return valueToBeReplaced; - } - - @Override - public boolean shouldApply(RestTestContext testContext) { - return testName == null || testContext.getTestName().equals(testName); - } - - @Override - public void transformTest(ObjectNode matchParent) { - matchParent.set(getKeyToFind(), replacementNode); - } - - @Input - public String getValueToBeReplaced() { - return valueToBeReplaced; - } - - @Input - public JsonNode getReplacementNode() { - return replacementNode; - } - - @Input - @Optional - public String getTestName() { - return testName; - } - - @Override - public boolean matches(JsonNode child) { - return child.asText().equals(requiredChildKey()); - } - -} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java deleted file mode 100644 index 0f05a4ba2cc7d..0000000000000 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceTextualTests.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.gradle.test.rest.transform.match; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.elasticsearch.gradle.test.rest.transform.TransformTests; -import org.hamcrest.CoreMatchers; -import org.junit.Test; - -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -public class ReplaceTextualTests extends TransformTests { - - private static final YAMLFactory YAML_FACTORY = new YAMLFactory(); - private static final ObjectMapper MAPPER = new ObjectMapper(YAML_FACTORY); - - @Test - public void testReplaceAll() throws Exception { - String testName = "/rest/transform/match/text_replace.yml"; - List tests = getTests(testName); - TextNode replacementNode = MAPPER.convertValue("_replaced_value", TextNode.class); - validateTest(tests, true, true); - List transformedTests = transformTests( - new LinkedList<>(tests), - Collections.singletonList(new ReplaceTextual("key_to_replace", "value_to_replace", replacementNode, null)) - ); - printTest(testName, transformedTests); - validateTest(tests, false, true); - } - - private void validateTest(List tests, boolean beforeTransformation, boolean allTests) { - validateSetupAndTearDownForMatchTests(tests); - // first test - JsonNode firstTestChild = tests.get(2).get("First test"); - assertThat(firstTestChild, CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode firstTestParentArray = (ArrayNode) firstTestChild; - - AtomicBoolean firstTestOccurrenceFound = new AtomicBoolean(false); - - firstTestParentArray.elements().forEachRemaining(node -> { - assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("key_to_replace"); - if (matchObject != null) { - firstTestOccurrenceFound.set(true); - if (beforeTransformation == false && allTests) { - assertThat(matchObject.asText(), CoreMatchers.is("_replaced_value")); - } - } - }); - assertTrue(firstTestOccurrenceFound.get()); - - // last test - JsonNode lastTestChild = tests.get(tests.size() - 1).get("Last test"); - assertThat(lastTestChild, CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode lastTestParentArray = (ArrayNode) lastTestChild; - - AtomicBoolean lastTestOccurrenceFound = new AtomicBoolean(false); - lastTestParentArray.elements().forEachRemaining(node -> { - assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("key_to_replace"); - if (matchObject != null) { - lastTestOccurrenceFound.set(true); - if (beforeTransformation == false && allTests) { - assertThat(matchObject.asText(), CoreMatchers.is("_replaced_value")); - } - } - }); - assertTrue(lastTestOccurrenceFound.get()); - - // exclude setup, teardown, first test, and last test - for (int i = 3; i <= tests.size() - 2; i++) { - ObjectNode otherTest = tests.get(i); - JsonNode otherTestChild = otherTest.get(otherTest.fields().next().getKey()); - assertThat(otherTestChild, CoreMatchers.instanceOf(ArrayNode.class)); - ArrayNode otherTestParentArray = (ArrayNode) otherTestChild; - otherTestParentArray.elements().forEachRemaining(node -> { - assertThat(node, CoreMatchers.instanceOf(ObjectNode.class)); - ObjectNode childObject = (ObjectNode) node; - JsonNode matchObject = childObject.get("key_to_replace"); - if (matchObject != null && matchObject.get("key_to_replace") != null && beforeTransformation == false && allTests) { - assertThat(matchObject.get("key_to_replace").asText(), CoreMatchers.is("_replaced_value")); - } - }); - } - } - - @Override - protected boolean getHumanDebug() { - return true; - } -} diff --git a/buildSrc/src/test/resources/rest/transform/match/text_replace.yml b/buildSrc/src/test/resources/rest/transform/match/text_replace.yml deleted file mode 100644 index a3f71bc33ffd0..0000000000000 --- a/buildSrc/src/test/resources/rest/transform/match/text_replace.yml +++ /dev/null @@ -1,110 +0,0 @@ ---- -setup: - - do: - something: - here: ok ---- -teardown: - - do: - something_else: - here: true ---- -"First test": - - - do: - something: - that_is: true - - - do: - and: again - - - match: { copied.from.real.test.total: 1 } - - match: { hits.hits.0._index: "single_doc_index"} - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 0} - - match: { _shards.failed: 0 } - - - do: - and: again - - - match: { hits.total: 1 } - - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"} - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 0} - - match: { _below_is_target_for_tests: 0 } - - key_to_replace: "value_to_replace" - ---- -"Also has _type match": - - - do: - something: - that_is: true - - - do: - and: again - - - match: { hits.total.value: 0 } - - key_to_replace: "value_to_replace" - - match: { _below_is_target_for_tests: 0 } - - match: { _type: foo } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } - - - do: - not_random_but_representive: "of actual test" - - - match: { hits.total.value: 0 } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } - ---- -"Does not have _type match ": - - - do: - something: - that_is: true - - - do: - and: again - - - match: { hits.total.value: 0 } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } - - - do: - it: again - - - match: { _type: 0 } - - match: { _shards.total: 2 } - - match: { _shards.successful: 2 } - - match: { _shards.skipped : 1} - - match: { _shards.failed: 0 } ---- -"Last test": - - - do: - something: 中文 - - - match: { _index: test_1 } - - key_to_replace: "value_to_replace" - - match: { _id: 中文 } - - match: { _source: { foo: "Hello: 中文" } } - - - do: - something_else: 中文 - - - match: { _index: test_1 } - - match: { _type: "the value does not matter" } - - match: { _id: 中文 } - - match: { _source: { foo: "Hello: 中文" } } - - From e7b7ab0584762fe428b9f0fd80a6909a2dfab217 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 14 Apr 2021 13:05:07 +0200 Subject: [PATCH 25/35] draft --- rest-api-spec/build.gradle | 98 +++++++++---------- .../admin/indices/RestGetMappingAction.java | 10 +- 2 files changed, 58 insertions(+), 50 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 079359fa5e71f..2d43e5f61fcec 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -170,57 +170,57 @@ tasks.named("yamlRestCompatTest").configure { 'indices.flush/10_basic/Flush stats', 'indices.flush/10_basic/Index synced flush rest test', 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', - 'indices.get_field_mapping/10_basic/Get field mapping with local is deprecated',// awaits #41676 - 'indices.get_field_mapping/11_basic_with_types/Get field mapping by index only',// awaits #41676 - 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field',// awaits #41676 - 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field, with another field that doesn\'t exist',// awaits #41676 - 'indices.get_field_mapping/11_basic_with_types/Get field mapping should work without index specifying type and fields',// awaits #41676 - 'indices.get_field_mapping/11_basic_with_types/Get field mapping with include_defaults',// awaits #41676 - 'indices.get_field_mapping/11_basic_with_types/Get field mapping with no index and type',// awaits #41676 - 'indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn\'t exist, but type and index do',// awaits #41676 - 'indices.get_field_mapping/30_missing_type/Raise 404 when type doesn\'t exist',// awaits #41676 - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'*\' for indices and types',// awaits #41676 - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'_all\' for indices and types',// awaits #41676 - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using comma_separated values for indices and types',// awaits #41676 - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with * for fields',// awaits #41676 - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with *t1 for fields',// awaits #41676 - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with t* for fields',// awaits #41676 - 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with wildcarded relative names',// awaits #41676 - 'indices.get_field_mapping/60_mix_typeless_typeful/GET mapping with typeless API on an index that has types',// awaits #41676 - 'indices.get_mapping/11_basic_with_types/Get /*/_mapping/{type}', - 'indices.get_mapping/11_basic_with_types/Get /_all/_mapping/{type}', - 'indices.get_mapping/11_basic_with_types/Get /_mapping', - 'indices.get_mapping/11_basic_with_types/Get /_mapping/{type}', - 'indices.get_mapping/11_basic_with_types/Get /index*/_mapping/{type}', - 'indices.get_mapping/11_basic_with_types/Get /index,index/_mapping/{type}', - 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping', - 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping with empty mappings', - 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/*', - 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/_all', - 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/{type*}', - 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/{type}', - 'indices.get_mapping/20_missing_type/Existent and non-existent type returns 404 and the existing type', - 'indices.get_mapping/20_missing_type/Existent and non-existent types returns 404 and the existing type', - 'indices.get_mapping/20_missing_type/No type matching pattern returns 404', - 'indices.get_mapping/20_missing_type/Non-existent type returns 404', - 'indices.get_mapping/20_missing_type/Type missing when no types exist', - 'indices.get_mapping/40_aliases/Getting mapping for aliases should return the real index as key', - 'indices.get_mapping/61_empty_with_types/Check empty mapping when getting all mappings via /_mapping', - 'indices.get_mapping/70_mix_typeless_typeful/GET mapping with typeless API on an index that has types', +// 'indices.get_field_mapping/10_basic/Get field mapping with local is deprecated',// awaits #41676 +// 'indices.get_field_mapping/11_basic_with_types/Get field mapping by index only',// awaits #41676 +// 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field',// awaits #41676 +// 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field, with another field that doesn\'t exist',// awaits #41676 +// 'indices.get_field_mapping/11_basic_with_types/Get field mapping should work without index specifying type and fields',// awaits #41676 +// 'indices.get_field_mapping/11_basic_with_types/Get field mapping with include_defaults',// awaits #41676 +// 'indices.get_field_mapping/11_basic_with_types/Get field mapping with no index and type',// awaits #41676 +// 'indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn\'t exist, but type and index do',// awaits #41676 +// 'indices.get_field_mapping/30_missing_type/Raise 404 when type doesn\'t exist',// awaits #41676 +// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'*\' for indices and types',// awaits #41676 +// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'_all\' for indices and types',// awaits #41676 +// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using comma_separated values for indices and types',// awaits #41676 +// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with * for fields',// awaits #41676 +// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with *t1 for fields',// awaits #41676 +// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with t* for fields',// awaits #41676 +// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with wildcarded relative names',// awaits #41676 +// 'indices.get_field_mapping/60_mix_typeless_typeful/GET mapping with typeless API on an index that has types',// awaits #41676 +// 'indices.get_mapping/11_basic_with_types/Get /*/_mapping/{type}', +// 'indices.get_mapping/11_basic_with_types/Get /_all/_mapping/{type}', +// 'indices.get_mapping/11_basic_with_types/Get /_mapping', +// 'indices.get_mapping/11_basic_with_types/Get /_mapping/{type}', +// 'indices.get_mapping/11_basic_with_types/Get /index*/_mapping/{type}', +// 'indices.get_mapping/11_basic_with_types/Get /index,index/_mapping/{type}', +// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping', +// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping with empty mappings', +// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/*', +// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/_all', +// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/{type*}', +// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/{type}', +// 'indices.get_mapping/20_missing_type/Existent and non-existent type returns 404 and the existing type', +// 'indices.get_mapping/20_missing_type/Existent and non-existent types returns 404 and the existing type', +// 'indices.get_mapping/20_missing_type/No type matching pattern returns 404', +// 'indices.get_mapping/20_missing_type/Non-existent type returns 404', +// 'indices.get_mapping/20_missing_type/Type missing when no types exist', +// 'indices.get_mapping/40_aliases/Getting mapping for aliases should return the real index as key', +// 'indices.get_mapping/61_empty_with_types/Check empty mapping when getting all mappings via /_mapping', +// 'indices.get_mapping/70_mix_typeless_typeful/GET mapping with typeless API on an index that has types', 'indices.open/10_basic/?wait_for_active_shards default is deprecated', 'indices.open/10_basic/?wait_for_active_shards=index-setting', - 'indices.put_mapping/10_basic/Put mappings with explicit _doc type', - 'indices.put_mapping/11_basic_with_types/Create index with invalid mappings', - 'indices.put_mapping/11_basic_with_types/Test Create and update mapping', - 'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with _doc on an index that has types', - 'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with typeless API on an index that has types', - 'indices.put_mapping/all_path_options_with_types/post a mapping with default analyzer twice', - 'indices.put_mapping/all_path_options_with_types/put mapping in * index', - 'indices.put_mapping/all_path_options_with_types/put mapping in _all index', - 'indices.put_mapping/all_path_options_with_types/put mapping in list of indices', - 'indices.put_mapping/all_path_options_with_types/put mapping in prefix* index', - 'indices.put_mapping/all_path_options_with_types/put mapping with blank index', - 'indices.put_mapping/all_path_options_with_types/put one mapping per index', +// 'indices.put_mapping/10_basic/Put mappings with explicit _doc type', +// 'indices.put_mapping/11_basic_with_types/Create index with invalid mappings', +// 'indices.put_mapping/11_basic_with_types/Test Create and update mapping', +// 'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with _doc on an index that has types', +// 'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with typeless API on an index that has types', +// 'indices.put_mapping/all_path_options_with_types/post a mapping with default analyzer twice', +// 'indices.put_mapping/all_path_options_with_types/put mapping in * index', +// 'indices.put_mapping/all_path_options_with_types/put mapping in _all index', +// 'indices.put_mapping/all_path_options_with_types/put mapping in list of indices', +// 'indices.put_mapping/all_path_options_with_types/put mapping in prefix* index', +// 'indices.put_mapping/all_path_options_with_types/put mapping with blank index', +// 'indices.put_mapping/all_path_options_with_types/put one mapping per index', // needs discussion about IndexTemplateMetadata#toInnerXContent and reducingMappings when includeTypeName and empty mappings etc 'indices.put_template/11_basic_with_types/Put template with empty mappings', // diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index a6d271991a2bc..2560391935afb 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.unit.TimeValue; @@ -31,11 +32,13 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.rest.RestRequest.Method.HEAD; public class RestGetMappingAction extends BaseRestHandler { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetMappingAction.class); public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get" + " mapping requests is deprecated. The parameter will be removed in the next major version."; + public static final String MSG = ""; private final ThreadPool threadPool; @@ -48,8 +51,13 @@ public List routes() { return List.of( new Route(GET, "/_mapping"), new Route(GET, "/_mappings"), + Route.builder(GET, "/{index}/{type}/_mapping").deprecated(MSG, RestApiVersion.V_7).build(), new Route(GET, "/{index}/_mapping"), - new Route(GET, "/{index}/_mappings")); + new Route(GET, "/{index}/_mappings"), + Route.builder(GET, "/{index}/_mappings/{type}").deprecated(MSG, RestApiVersion.V_7).build(), + Route.builder(GET, "/{index}/_mapping/{type}").deprecated(MSG, RestApiVersion.V_7).build(), + Route.builder(HEAD, "/{index}/_mapping/{type}").deprecated(MSG, RestApiVersion.V_7).build(), + Route.builder(GET, "/_mapping/{type}").deprecated(MSG, RestApiVersion.V_7).build()); } @Override From 5f1b95025d2a17d1d12e313e52377e5fc0ecf441 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 14 Apr 2021 14:09:40 +0200 Subject: [PATCH 26/35] missing version quard check --- .../action/admin/indices/RestGetFieldMappingAction.java | 8 +++----- .../action/admin/indices/RestGetIndexTemplateAction.java | 6 ++---- .../rest/action/admin/indices/RestGetMappingAction.java | 3 ++- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index 530198a5b7136..dbad6054d2023 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -58,11 +58,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final String[] fields = Strings.splitStringByCommaToArray(request.param("fields")); - if(request.getRestApiVersion() == RestApiVersion.V_7){ - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); - deprecationLogger.compatibleApiWarning("get_field_mapping_with_types", TYPES_DEPRECATION_MESSAGE); - } + if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + deprecationLogger.compatibleApiWarning("get_field_mapping_with_types", TYPES_DEPRECATION_MESSAGE); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index b3379899a6f40..4d42c371c591d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -56,10 +56,8 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if(request.getRestApiVersion() == RestApiVersion.V_7) { - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - deprecationLogger.compatibleApiWarning("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE); - } + if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.compatibleApiWarning("get_index_template_include_type_name", TYPES_DEPRECATION_MESSAGE); } final String[] names = Strings.splitStringByCommaToArray(request.param("name")); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index a6d271991a2bc..96bfb63d45abf 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.unit.TimeValue; @@ -59,7 +60,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { request.param(INCLUDE_TYPE_NAME_PARAMETER); deprecationLogger.compatibleApiWarning("get_mapping_with_types", TYPES_DEPRECATION_MESSAGE); } From a8f043b212074467547eec10f3cfe7e8e368f250 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 15 Apr 2021 09:04:00 +0200 Subject: [PATCH 27/35] get field mapping transformations --- .../compat/RestCompatTestTransformTask.java | 33 ++++++---- .../transform/match/ReplaceKeyInMatch.java | 45 ++++++++++++++ .../rest/transform/match/ReplaceLength.java | 34 +++++++++++ .../rest/transform/match/ReplaceMatch.java | 15 +++-- rest-api-spec/build.gradle | 37 +++++++++++ .../mapping/get/GetFieldMappingsResponse.java | 8 ++- .../mapping/get/GetMappingsResponse.java | 15 ++++- .../indices/RestGetFieldMappingAction.java | 30 +++++++-- .../admin/indices/RestGetMappingAction.java | 34 +++++++---- .../admin/indices/RestPutMappingAction.java | 61 +++++++++++++++++-- 10 files changed, 270 insertions(+), 42 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceKeyInMatch.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceLength.java diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 1128417b9dc00..595b11c8352c7 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -24,6 +24,8 @@ import org.elasticsearch.gradle.test.rest.transform.headers.InjectHeaders; import org.elasticsearch.gradle.test.rest.transform.match.AddMatch; import org.elasticsearch.gradle.test.rest.transform.match.RemoveMatch; +import org.elasticsearch.gradle.test.rest.transform.match.ReplaceKeyInMatch; +import org.elasticsearch.gradle.test.rest.transform.match.ReplaceLength; import org.elasticsearch.gradle.test.rest.transform.match.ReplaceMatch; import org.elasticsearch.gradle.test.rest.transform.text.ReplaceIsFalse; import org.elasticsearch.gradle.test.rest.transform.text.ReplaceIsTrue; @@ -94,6 +96,7 @@ public RestCompatTestTransformTask( transformations.add(new InjectHeaders(headers)); } + /** * Replaces all the values of a match assertion for all project REST tests. * For example "match":{"_type": "foo"} to "match":{"_type": "bar"} @@ -105,9 +108,26 @@ public void replaceMatch(String subKey, Object value) { transformations.add(new ReplaceMatch(subKey, MAPPER.convertValue(value, JsonNode.class))); } + /** + * Replaces the values of a match assertion for the given REST test. For example "match":{"_type": "foo"} to "match":{"_type": "bar"} + * + * @param subKey the key name directly under match to replace. For example "_type" + * @param value the value used in the replacement. For example "bar" + * @param testName the testName to apply replacement + */ + public void replaceMatch(String subKey, Object value, String testName) { + transformations.add(new ReplaceMatch(subKey, MAPPER.convertValue(value, JsonNode.class), testName)); + } + + public void replaceKeyInLength(String oldKeyName, String newKeyName) { + transformations.add(new ReplaceLength(oldKeyName,newKeyName, null)); + } + public void replaceKeyInMatch(String oldKeyName, String newKeyName) { + transformations.add(new ReplaceKeyInMatch(oldKeyName,newKeyName, null)); + } /** * Replaces all the values of a is_true assertion for all project REST tests. - * For example "is_true": "value_to_replace" to "match": "value_replaced" + * For example "is_true": "value_to_replace" to "is_true": "value_replaced" * * @param oldValue the value that has to match and will be replaced * @param newValue the value used in the replacement @@ -127,16 +147,7 @@ public void replaceIsFalse(String oldValue, Object newValue) { transformations.add(new ReplaceIsFalse(oldValue, MAPPER.convertValue(newValue, TextNode.class))); } - /** - * Replaces the values of a match assertion for the given REST test. For example "match":{"_type": "foo"} to "match":{"_type": "bar"} - * - * @param subKey the key name directly under match to replace. For example "_type" - * @param value the value used in the replacement. For example "bar" - * @param testName the testName to apply replacement - */ - public void replaceMatch(String subKey, Object value, String testName) { - transformations.add(new ReplaceMatch(subKey, MAPPER.convertValue(value, JsonNode.class), testName)); - } + /** * Removes the key/value of a match assertion all project REST tests for the matching subkey. diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceKeyInMatch.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceKeyInMatch.java new file mode 100644 index 0000000000000..1d2c73ae5098c --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceKeyInMatch.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.test.rest.transform.match; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.test.rest.transform.RestTestContext; +import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentObject; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Optional; + +/** + * A transformation to replace the value of a match. For example, change from "match":{"_type": "foo"} to "match":{"_type": "bar"} + */ +public class ReplaceKeyInMatch extends ReplaceMatch { + + private final String newKeyName; + + public ReplaceKeyInMatch(String replaceKey, String newKeyName, String testName) { + super(replaceKey, newKeyName, null, testName); + this.newKeyName = newKeyName; + } + + @Override + @Internal + public String getKeyToFind() { + return "match"; + } + + + @Override + public void transformTest(ObjectNode matchParent) { + ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); + JsonNode value = matchNode.get(getReplaceKey()); + matchNode.remove(getReplaceKey()); + matchNode.set(newKeyName, value); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceLength.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceLength.java new file mode 100644 index 0000000000000..55e87fb0e4941 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceLength.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.test.rest.transform.match; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.elasticsearch.gradle.test.rest.transform.RestTestContext; +import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentObject; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Optional; + +/** + * A transformation to replace the value of a match. For example, change from "match":{"_type": "foo"} to "match":{"_type": "bar"} + */ +public class ReplaceLength extends ReplaceKeyInMatch { + + + public ReplaceLength(String replaceKey, String newKeyName, String testName) { + super(replaceKey, newKeyName, testName); + } + + @Override + @Internal + public String getKeyToFind() { + return "length"; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatch.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatch.java index 69d234009f983..f7f171495b8e9 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatch.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatch.java @@ -21,22 +21,23 @@ */ public class ReplaceMatch implements RestTestTransformByParentObject { private final String replaceKey; + private final String newKeyName; private final JsonNode replacementNode; private final String testName; public ReplaceMatch(String replaceKey, JsonNode replacementNode) { - - this.replaceKey = replaceKey; - this.replacementNode = replacementNode; - this.testName = null; + this(replaceKey, replacementNode, null); } public ReplaceMatch(String replaceKey, JsonNode replacementNode, String testName) { + this(replaceKey, replaceKey, replacementNode, testName); + } + public ReplaceMatch(String replaceKey, String newKeyName, JsonNode replacementNode, String testName) { this.replaceKey = replaceKey; + this.newKeyName = newKeyName; this.replacementNode = replacementNode; this.testName = testName; } - @Override @Internal public String getKeyToFind() { @@ -56,7 +57,8 @@ public boolean shouldApply(RestTestContext testContext) { @Override public void transformTest(ObjectNode matchParent) { ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); - matchNode.set(replaceKey, replacementNode); + matchNode.remove(replaceKey); + matchNode.set(newKeyName, replacementNode); } @Input @@ -65,6 +67,7 @@ public String getReplaceKey() { } @Input + @Optional public JsonNode getReplacementNode() { return replacementNode; } diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 2d43e5f61fcec..87c39a99038a4 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -346,6 +346,43 @@ tasks.named("transformV7RestTests").configure({ task -> task.replaceIsTrue("test_index.mappings.type_1", "test_index.mappings._doc") task.replaceIsFalse("test_index.mappings.type_1", "test_index.mappings._doc") task.replaceIsFalse("test-1.mappings.my_type", "test-1.mappings._doc") + task.replaceKeyInLength("test_index.mappings.test_type.text.mapping.text.type", + "test_index.mappings._doc.text.mapping.text.type" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.text.mapping.text.analyzer", + "test_index.mappings._doc.text.mapping.text.analyzer" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.t1.full_name", + "test_index.mappings._doc.t1.full_name" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.t2.full_name", + "test_index.mappings._doc.t2.full_name" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.obj\\.t1.full_name", + "test_index.mappings._doc.obj\\.t1.full_name" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.obj\\.i_t1.full_name", + "test_index.mappings._doc.obj\\.i_t1.full_name" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.obj\\.i_t3.full_name", + "test_index.mappings._doc.obj\\.i_t3.full_name" + ) + task.replaceKeyInLength("test_index.mappings.test_type", + "test_index.mappings._doc" + ) + task.replaceKeyInMatch("test_index_2.mappings.test_type_2.t1.full_name", + "test_index.mappings._doc.t1.full_name" + ) + task.replaceKeyInMatch("test_index_2.mappings.test_type_2.t2.full_name", + "test_index.mappings._doc.t2.full_name" + ) + task.replaceKeyInLength("test_index_2.mappings.test_type_2", + "test_index.mappings._doc" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.text.mapping.text.type", + "test_index.mappings._doc.text.mapping.text.type" + ) +//TO ASK ON REVIEW - difference between 20_missing_field and 21_missing_field_with_types. Empty {} vs test_index.mappings: {} }) tasks.register('enforceYamlTestConvention').configure { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index bb1443d9d76f3..89cbe633f9de8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -95,9 +95,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (indexEntry.getValue() != null) { if (builder.getRestApiVersion() == RestApiVersion.V_7 && params.paramAsBoolean(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) { - builder.startObject(MapperService.SINGLE_MAPPING_NAME); - addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); - builder.endObject(); + if(indexEntry.getValue().size()>0){ + builder.startObject(MapperService.SINGLE_MAPPING_NAME); + addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); + builder.endObject(); + } } else { addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index c7ed7ebaad560..7a42524bdc43a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; @@ -23,6 +24,9 @@ import java.io.IOException; +import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; +import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; + public class GetMappingsResponse extends ActionResponse implements ToXContentFragment { private static final ParseField MAPPINGS = new ParseField("mappings"); @@ -65,7 +69,16 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { for (final ObjectObjectCursor indexEntry : getMappings()) { builder.startObject(indexEntry.key); - if (indexEntry.value != null) { + boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName) { + builder.startObject(MAPPINGS.getPreferredName()); + if (indexEntry.value.sourceAsMap().size() > 0) { + builder.field(MapperService.SINGLE_MAPPING_NAME, indexEntry.value.sourceAsMap()); + } + builder.endObject(); + + } else if (indexEntry.value != null) { builder.field(MAPPINGS.getPreferredName(), indexEntry.value.sourceAsMap()); } else { builder.startObject(MAPPINGS.getPreferredName()).endObject(); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index dbad6054d2023..6a649d62f2d90 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -38,14 +38,18 @@ public class RestGetFieldMappingAction extends BaseRestHandler { private static final Logger logger = LogManager.getLogger(RestGetFieldMappingAction.class); private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(logger.getName()); - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get " + + public static final String INCLUDE_TYPE_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get " + "field mapping requests is deprecated. The parameter will be removed in the next major version."; - + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in put mapping request is deprecated. " + + "Use typeless api instead"; @Override public List routes() { return List.of( new Route(GET, "/_mapping/field/{fields}"), - new Route(GET, "/{index}/_mapping/field/{fields}")); + new Route(GET, "/{index}/_mapping/field/{fields}"), + Route.builder(GET, "/_mapping/{type}/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(GET, "/{index}/{type}/_mapping/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(GET, "/{index}/_mapping/{type}/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()); } @Override @@ -58,9 +62,23 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final String[] fields = Strings.splitStringByCommaToArray(request.param("fields")); - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); - deprecationLogger.compatibleApiWarning("get_field_mapping_with_types", TYPES_DEPRECATION_MESSAGE); + if (request.getRestApiVersion() == RestApiVersion.V_7) { + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.compatibleApiWarning("get_field_mapping_with_types", INCLUDE_TYPE_DEPRECATION_MESSAGE); + } + boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + final String[] types = request.paramAsStringArrayOrEmptyIfAll("type"); + if (includeTypeName == false && types.length > 0) { + throw new IllegalArgumentException("Types cannot be specified unless include_type_name" + " is set to true."); + } + + if (request.hasParam("local")) { + request.param("local"); + deprecationLogger.compatibleApiWarning( + "get_field_mapping_local", + "Use [local] in get field mapping requests is deprecated. " + "The parameter will be removed in the next major version" + ); + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index 6b2162d46373a..a3bc4f709dcd5 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -36,9 +37,10 @@ public class RestGetMappingAction extends BaseRestHandler { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetMappingAction.class); - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get" + public static final String INCLUDE_TYPE_DEPRECATION_MSG = "[types removal] Using include_type_name in get" + " mapping requests is deprecated. The parameter will be removed in the next major version."; - public static final String MSG = ""; + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in get mapping request is deprecated. " + + "Use typeless api instead"; private final ThreadPool threadPool; @@ -51,13 +53,13 @@ public List routes() { return List.of( new Route(GET, "/_mapping"), new Route(GET, "/_mappings"), - Route.builder(GET, "/{index}/{type}/_mapping").deprecated(MSG, RestApiVersion.V_7).build(), + Route.builder(GET, "/{index}/{type}/_mapping").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), new Route(GET, "/{index}/_mapping"), new Route(GET, "/{index}/_mappings"), - Route.builder(GET, "/{index}/_mappings/{type}").deprecated(MSG, RestApiVersion.V_7).build(), - Route.builder(GET, "/{index}/_mapping/{type}").deprecated(MSG, RestApiVersion.V_7).build(), - Route.builder(HEAD, "/{index}/_mapping/{type}").deprecated(MSG, RestApiVersion.V_7).build(), - Route.builder(GET, "/_mapping/{type}").deprecated(MSG, RestApiVersion.V_7).build()); + Route.builder(GET, "/{index}/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(GET, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(HEAD, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(GET, "/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()); } @Override @@ -67,11 +69,21 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (request.getRestApiVersion() == RestApiVersion.V_7 && request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { - request.param(INCLUDE_TYPE_NAME_PARAMETER); - deprecationLogger.compatibleApiWarning("get_mapping_with_types", TYPES_DEPRECATION_MESSAGE); + if (request.getRestApiVersion() == RestApiVersion.V_7) { + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + request.param(INCLUDE_TYPE_NAME_PARAMETER); + deprecationLogger.compatibleApiWarning("get_mapping_with_types", INCLUDE_TYPE_DEPRECATION_MSG); + } + final String[] types = request.paramAsStringArrayOrEmptyIfAll("type"); + if (request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY) == false && types.length > 0) { + throw new IllegalArgumentException("Types cannot be provided in get mapping requests, unless" + + " include_type_name is set to true."); + } + if (request.method().equals(HEAD)) { + deprecationLogger.compatibleApiWarning("get_mapping_types_removal", + "Type exists requests are deprecated, as types have been deprecated."); + } } - final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java index 3a63d9937dfe2..3329018744557 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java @@ -11,7 +11,10 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.BaseRestHandler; @@ -19,14 +22,20 @@ import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import static org.elasticsearch.client.Requests.putMappingRequest; +import static org.elasticsearch.index.mapper.MapperService.isMappingSourceTyped; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; public class RestPutMappingAction extends BaseRestHandler { + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in put mapping request is deprecated. " + + "Use typeless api instead"; + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestPutMappingAction.class); @Override public List routes() { @@ -34,7 +43,19 @@ public List routes() { new Route(POST, "/{index}/_mapping/"), new Route(PUT, "/{index}/_mapping/"), new Route(POST, "/{index}/_mappings/"), - new Route(PUT, "/{index}/_mappings/")); + new Route(PUT, "/{index}/_mappings/"), + Route.builder(POST, "/{index}/{type}/_mapping").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(PUT, "/{index}/{type}/_mapping").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(POST, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(PUT, "/{index}/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(POST, "/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(PUT, "/_mapping/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(POST, "/{index}/{type}/_mappings").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(PUT, "/{index}/{type}/_mappings").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(POST, "/{index}/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(PUT, "/{index}/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(POST, "/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(PUT, "/_mappings/{type}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()); } @Override @@ -48,15 +69,47 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, request.getXContentType()).v2(); - if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap)) { - throw new IllegalArgumentException("Types cannot be provided in put mapping requests"); + if (request.getRestApiVersion() == RestApiVersion.V_7) { + final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, + DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (request.hasParam(INCLUDE_TYPE_NAME_PARAMETER)) { + deprecationLogger.compatibleApiWarning("put_mapping_with_types", TYPES_DEPRECATION_MESSAGE); + } + final String type = request.param("type"); + if (includeTypeName == false && + (type != null || isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap))) { + throw new IllegalArgumentException("Types cannot be provided in put mapping requests, unless " + + "the include_type_name parameter is set to true."); + } + // TODO what about the type in the source? it should be skipped? + // (different than in RestCreateIndexAction which expects the _doc) + Map mappingSource = prepareV7Mappings(includeTypeName, sourceAsMap); + putMappingRequest.source(mappingSource); + } else { + if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, sourceAsMap)) { + throw new IllegalArgumentException("Types cannot be provided in put mapping requests"); + } + putMappingRequest.source(sourceAsMap); } - putMappingRequest.source(sourceAsMap); putMappingRequest.timeout(request.paramAsTime("timeout", putMappingRequest.timeout())); putMappingRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putMappingRequest.masterNodeTimeout())); putMappingRequest.indicesOptions(IndicesOptions.fromRequest(request, putMappingRequest.indicesOptions())); putMappingRequest.writeIndexOnly(request.paramAsBoolean("write_index_only", false)); return channel -> client.admin().indices().putMapping(putMappingRequest, new RestToXContentListener<>(channel)); } + + private Map prepareV7Mappings(boolean includeTypeName, Map mappings) { + if (includeTypeName && mappings != null && mappings.size() == 1) { + String typeName = mappings.keySet().iterator().next(); + if (Strings.hasText(typeName) == false) { + throw new IllegalArgumentException("name cannot be empty string"); + } + @SuppressWarnings("unchecked") + Map typedMappings = (Map) mappings.get(typeName); + return typedMappings; + } + return mappings; + } + } From d4ef47081137ff9141dd7738f004d83b326769ab Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 15 Apr 2021 10:14:33 +0200 Subject: [PATCH 28/35] precommit --- .../rest/compat/RestCompatTestTransformTask.java | 9 ++++----- .../test/rest/transform/match/ReplaceKeyInMatch.java | 5 ----- .../gradle/test/rest/transform/match/ReplaceLength.java | 9 +-------- .../gradle/test/rest/transform/match/ReplaceMatch.java | 2 ++ .../action/admin/indices/RestGetFieldMappingAction.java | 9 ++++++--- .../rest/action/admin/indices/RestGetMappingAction.java | 1 - .../rest/action/admin/indices/RestPutMappingAction.java | 3 --- 7 files changed, 13 insertions(+), 25 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 595b11c8352c7..216259f76b61f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -96,7 +96,6 @@ public RestCompatTestTransformTask( transformations.add(new InjectHeaders(headers)); } - /** * Replaces all the values of a match assertion for all project REST tests. * For example "match":{"_type": "foo"} to "match":{"_type": "bar"} @@ -120,11 +119,13 @@ public void replaceMatch(String subKey, Object value, String testName) { } public void replaceKeyInLength(String oldKeyName, String newKeyName) { - transformations.add(new ReplaceLength(oldKeyName,newKeyName, null)); + transformations.add(new ReplaceLength(oldKeyName, newKeyName, null)); } + public void replaceKeyInMatch(String oldKeyName, String newKeyName) { - transformations.add(new ReplaceKeyInMatch(oldKeyName,newKeyName, null)); + transformations.add(new ReplaceKeyInMatch(oldKeyName, newKeyName, null)); } + /** * Replaces all the values of a is_true assertion for all project REST tests. * For example "is_true": "value_to_replace" to "is_true": "value_replaced" @@ -147,8 +148,6 @@ public void replaceIsFalse(String oldValue, Object newValue) { transformations.add(new ReplaceIsFalse(oldValue, MAPPER.convertValue(newValue, TextNode.class))); } - - /** * Removes the key/value of a match assertion all project REST tests for the matching subkey. * For example "match":{"_type": "foo"} to "match":{} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceKeyInMatch.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceKeyInMatch.java index 1d2c73ae5098c..0bc84330bd60a 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceKeyInMatch.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceKeyInMatch.java @@ -10,11 +10,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.elasticsearch.gradle.test.rest.transform.RestTestContext; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentObject; -import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Internal; -import org.gradle.api.tasks.Optional; /** * A transformation to replace the value of a match. For example, change from "match":{"_type": "foo"} to "match":{"_type": "bar"} @@ -34,7 +30,6 @@ public String getKeyToFind() { return "match"; } - @Override public void transformTest(ObjectNode matchParent) { ObjectNode matchNode = (ObjectNode) matchParent.get(getKeyToFind()); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceLength.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceLength.java index 55e87fb0e4941..39d9f779e43e8 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceLength.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceLength.java @@ -8,22 +8,15 @@ package org.elasticsearch.gradle.test.rest.transform.match; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.elasticsearch.gradle.test.rest.transform.RestTestContext; -import org.elasticsearch.gradle.test.rest.transform.RestTestTransformByParentObject; -import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Internal; -import org.gradle.api.tasks.Optional; /** * A transformation to replace the value of a match. For example, change from "match":{"_type": "foo"} to "match":{"_type": "bar"} */ public class ReplaceLength extends ReplaceKeyInMatch { - public ReplaceLength(String replaceKey, String newKeyName, String testName) { - super(replaceKey, newKeyName, testName); + super(replaceKey, newKeyName, testName); } @Override diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatch.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatch.java index f7f171495b8e9..402aeeb536ed1 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatch.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/match/ReplaceMatch.java @@ -32,12 +32,14 @@ public ReplaceMatch(String replaceKey, JsonNode replacementNode) { public ReplaceMatch(String replaceKey, JsonNode replacementNode, String testName) { this(replaceKey, replaceKey, replacementNode, testName); } + public ReplaceMatch(String replaceKey, String newKeyName, JsonNode replacementNode, String testName) { this.replaceKey = replaceKey; this.newKeyName = newKeyName; this.replacementNode = replacementNode; this.testName = testName; } + @Override @Internal public String getKeyToFind() { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index 6a649d62f2d90..7fe5edbe5235f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -47,9 +47,12 @@ public List routes() { return List.of( new Route(GET, "/_mapping/field/{fields}"), new Route(GET, "/{index}/_mapping/field/{fields}"), - Route.builder(GET, "/_mapping/{type}/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(GET, "/{index}/{type}/_mapping/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), - Route.builder(GET, "/{index}/_mapping/{type}/field/{fields}").deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()); + Route.builder(GET, "/_mapping/{type}/field/{fields}") + .deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(GET, "/{index}/{type}/_mapping/field/{fields}") + .deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build(), + Route.builder(GET, "/{index}/_mapping/{type}/field/{fields}") + .deprecated(TYPES_DEPRECATION_MESSAGE, RestApiVersion.V_7).build()); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index a3bc4f709dcd5..9f8c41bffae49 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java index 3329018744557..1fec5c6b5df38 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.mapper.MapperService; @@ -22,8 +21,6 @@ import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; From 95bba64fe338b320636709d1aa08c48cff5ed25d Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 15 Apr 2021 10:49:09 +0200 Subject: [PATCH 29/35] precommit --- .../action/admin/indices/rollover/RolloverRequest.java | 7 ++++++- .../cluster/metadata/IndexTemplateMetadata.java | 7 +++++-- .../main/java/org/elasticsearch/rest/BaseRestHandler.java | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 22ff5921917c9..31a40b22a9e4c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -86,7 +86,12 @@ public class RolloverRequest extends AcknowledgedRequest implem }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> { // a type is not included, add a dummy _doc type - request.createIndexRequest.mapping(parser.map()); + Map mappings = parser.map(); + if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) { + + throw new IllegalArgumentException("The mapping definition cannot be nested under a type"); + } + request.createIndexRequest.mapping(mappings); }, CreateIndexRequest.MAPPINGS.forRestApiVersion(RestApiVersion.onOrAfter(RestApiVersion.V_8)), ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()), diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java index ef592c48b2a14..fff997e4bc9ec 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetadata.java @@ -13,7 +13,6 @@ import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.RestApiVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; @@ -38,6 +37,10 @@ import java.util.Objects; import java.util.Set; +import static org.elasticsearch.common.RestApiVersion.V_8; +import static org.elasticsearch.common.RestApiVersion.onOrAfter; + + public class IndexTemplateMetadata extends AbstractDiffable { private final String name; @@ -379,7 +382,7 @@ private static void toInnerXContent(IndexTemplateMetadata indexTemplateMetadata, indexTemplateMetadata.settings().toXContent(builder, params); builder.endObject(); - if(builder.getRestApiVersion() != RestApiVersion.V_7) { + if(builder.getRestApiVersion().matches(onOrAfter(V_8))) { includeTypeName &= (params.paramAsBoolean("reduce_mappings", false) == false); } diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 2d2d0cc83a0d3..857e92bdbc18a 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -43,7 +43,7 @@ public abstract class BaseRestHandler implements RestHandler { /** * Parameter that controls whether certain REST apis should include type names in their requests or responses. - * Note: This parameter is only available through compatible rest api. + * Note: This parameter is only available through compatible rest api for {@link RestApiVersion#V_7}. */ public static final String INCLUDE_TYPE_NAME_PARAMETER = "include_type_name"; public static final boolean DEFAULT_INCLUDE_TYPE_NAME_POLICY = false; From 597a94b476b745d67e2bcc33fb7d6d1bf4d37433 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 15 Apr 2021 14:42:29 +0200 Subject: [PATCH 30/35] empty mappings handling --- rest-api-spec/build.gradle | 105 ++++++++++-------- .../mapping/get/GetMappingsResponse.java | 5 +- 2 files changed, 60 insertions(+), 50 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 87c39a99038a4..af79e27c3db82 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -170,59 +170,27 @@ tasks.named("yamlRestCompatTest").configure { 'indices.flush/10_basic/Flush stats', 'indices.flush/10_basic/Index synced flush rest test', 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', -// 'indices.get_field_mapping/10_basic/Get field mapping with local is deprecated',// awaits #41676 -// 'indices.get_field_mapping/11_basic_with_types/Get field mapping by index only',// awaits #41676 -// 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field',// awaits #41676 -// 'indices.get_field_mapping/11_basic_with_types/Get field mapping by type & field, with another field that doesn\'t exist',// awaits #41676 -// 'indices.get_field_mapping/11_basic_with_types/Get field mapping should work without index specifying type and fields',// awaits #41676 -// 'indices.get_field_mapping/11_basic_with_types/Get field mapping with include_defaults',// awaits #41676 -// 'indices.get_field_mapping/11_basic_with_types/Get field mapping with no index and type',// awaits #41676 -// 'indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn\'t exist, but type and index do',// awaits #41676 -// 'indices.get_field_mapping/30_missing_type/Raise 404 when type doesn\'t exist',// awaits #41676 -// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'*\' for indices and types',// awaits #41676 -// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using \'_all\' for indices and types',// awaits #41676 -// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping should work using comma_separated values for indices and types',// awaits #41676 -// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with * for fields',// awaits #41676 -// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with *t1 for fields',// awaits #41676 -// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with t* for fields',// awaits #41676 -// 'indices.get_field_mapping/51_field_wildcards_with_types/Get field mapping with wildcarded relative names',// awaits #41676 -// 'indices.get_field_mapping/60_mix_typeless_typeful/GET mapping with typeless API on an index that has types',// awaits #41676 -// 'indices.get_mapping/11_basic_with_types/Get /*/_mapping/{type}', -// 'indices.get_mapping/11_basic_with_types/Get /_all/_mapping/{type}', -// 'indices.get_mapping/11_basic_with_types/Get /_mapping', -// 'indices.get_mapping/11_basic_with_types/Get /_mapping/{type}', -// 'indices.get_mapping/11_basic_with_types/Get /index*/_mapping/{type}', -// 'indices.get_mapping/11_basic_with_types/Get /index,index/_mapping/{type}', -// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping', -// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping with empty mappings', -// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/*', -// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/_all', -// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/{type*}', -// 'indices.get_mapping/11_basic_with_types/Get /{index}/_mapping/{type}', -// 'indices.get_mapping/20_missing_type/Existent and non-existent type returns 404 and the existing type', -// 'indices.get_mapping/20_missing_type/Existent and non-existent types returns 404 and the existing type', -// 'indices.get_mapping/20_missing_type/No type matching pattern returns 404', -// 'indices.get_mapping/20_missing_type/Non-existent type returns 404', -// 'indices.get_mapping/20_missing_type/Type missing when no types exist', + // This test returns test_index.mappings:{} when {} was expected. difference between 20_missing_field and 21_missing_field_with_types? + 'indices.get_field_mapping/21_missing_field_with_types/Return empty object if field doesn\'t exist, but type and index do', + // The information about the type is not present in the index. hence it cannot know if the type exist or not. + 'indices.get_field_mapping/30_missing_type/Raise 404 when type doesn\'t exist', + // The information about the type is not present in the index. hence it cannot know if the type exist or not. + 'indices.get_mapping/20_missing_type/Existent and non-existent type returns 404 and the existing type', + 'indices.get_mapping/20_missing_type/Existent and non-existent types returns 404 and the existing type', + 'indices.get_mapping/20_missing_type/No type matching pattern returns 404', + 'indices.get_mapping/20_missing_type/Non-existent type returns 404', + 'indices.get_mapping/20_missing_type/Type missing when no types exist', // 'indices.get_mapping/40_aliases/Getting mapping for aliases should return the real index as key', // 'indices.get_mapping/61_empty_with_types/Check empty mapping when getting all mappings via /_mapping', // 'indices.get_mapping/70_mix_typeless_typeful/GET mapping with typeless API on an index that has types', 'indices.open/10_basic/?wait_for_active_shards default is deprecated', 'indices.open/10_basic/?wait_for_active_shards=index-setting', -// 'indices.put_mapping/10_basic/Put mappings with explicit _doc type', -// 'indices.put_mapping/11_basic_with_types/Create index with invalid mappings', -// 'indices.put_mapping/11_basic_with_types/Test Create and update mapping', -// 'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with _doc on an index that has types', -// 'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with typeless API on an index that has types', -// 'indices.put_mapping/all_path_options_with_types/post a mapping with default analyzer twice', -// 'indices.put_mapping/all_path_options_with_types/put mapping in * index', -// 'indices.put_mapping/all_path_options_with_types/put mapping in _all index', -// 'indices.put_mapping/all_path_options_with_types/put mapping in list of indices', -// 'indices.put_mapping/all_path_options_with_types/put mapping in prefix* index', -// 'indices.put_mapping/all_path_options_with_types/put mapping with blank index', -// 'indices.put_mapping/all_path_options_with_types/put one mapping per index', + + // The information about the type is not present in the index. hence it cannot know if the type was already used or not + 'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with _doc on an index that has types', + 'indices.put_mapping/20_mix_typeless_typeful/PUT mapping with typeless API on an index that has types', // needs discussion about IndexTemplateMetadata#toInnerXContent and reducingMappings when includeTypeName and empty mappings etc - 'indices.put_template/11_basic_with_types/Put template with empty mappings', +// 'indices.put_template/11_basic_with_types/Put template with empty mappings', // 'indices.segments/10_basic/basic segments test', 'indices.segments/10_basic/closed segments test', @@ -346,6 +314,7 @@ tasks.named("transformV7RestTests").configure({ task -> task.replaceIsTrue("test_index.mappings.type_1", "test_index.mappings._doc") task.replaceIsFalse("test_index.mappings.type_1", "test_index.mappings._doc") task.replaceIsFalse("test-1.mappings.my_type", "test-1.mappings._doc") + // overrides for indices.get_field_mapping task.replaceKeyInLength("test_index.mappings.test_type.text.mapping.text.type", "test_index.mappings._doc.text.mapping.text.type" ) @@ -382,7 +351,47 @@ tasks.named("transformV7RestTests").configure({ task -> task.replaceKeyInMatch("test_index.mappings.test_type.text.mapping.text.type", "test_index.mappings._doc.text.mapping.text.type" ) -//TO ASK ON REVIEW - difference between 20_missing_field and 21_missing_field_with_types. Empty {} vs test_index.mappings: {} + // overrides for indices.put_mapping/11_basic_with_types + task.replaceKeyInMatch("test_index.mappings.test_type.properties.text1.type", + "test_index.mappings._doc.properties.text1.type" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.properties.text1.analyzer", + "test_index.mappings._doc.properties.text1.analyzer" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.properties.text2.type", + "test_index.mappings._doc.properties.text2.type" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.properties.text2.analyzer", + "test_index.mappings._doc.properties.text2.analyzer" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.properties.subfield.properties.text3.type", + "test_index.mappings._doc.properties.subfield.properties.text3.type" + ) + task.replaceKeyInMatch("test_index.mappings.test_type.properties.text1.fields.text_raw.type", + "test_index.mappings._doc.properties.text1.fields.text_raw.type" + ) + // overrides for indices.put_mapping/all_path_options_with_types + task.replaceKeyInMatch("test_index1.mappings.test_type.properties.text.type", + "test_index1.mappings._doc.properties.text.type" + ) + task.replaceKeyInMatch("test_index1.mappings.test_type.properties.text.analyzer", + "test_index1.mappings._doc.properties.text.analyzer" + ) + task.replaceKeyInMatch("test_index2.mappings.test_type.properties.text.type", + "test_index2.mappings._doc.properties.text.type" + ) + task.replaceKeyInMatch("test_index2.mappings.test_type.properties.text.analyzer", + "test_index2.mappings._doc.properties.text.analyzer" + ) + task.replaceKeyInMatch("foo.mappings.test_type.properties.text.type", + "foo.mappings._doc.properties.text.type" + ) + task.replaceKeyInMatch("foo.mappings.test_type.properties.text.analyzer", + "foo.mappings._doc.properties.text.analyzer" + ) + // overrides for indices.get_mapping + task.replaceIsTrue("test_1.mappings.doc", "test_1.mappings._doc") + task.replaceIsTrue("test_2.mappings.doc", "test_2.mappings._doc") }) tasks.register('enforceYamlTestConvention').configure { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index 7a42524bdc43a..20c543b0b698d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -71,9 +71,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(indexEntry.key); boolean includeTypeName = params.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); - if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName) { + if (builder.getRestApiVersion() == RestApiVersion.V_7 && includeTypeName && indexEntry.value != null) { builder.startObject(MAPPINGS.getPreferredName()); - if (indexEntry.value.sourceAsMap().size() > 0) { + + if (indexEntry.value != MappingMetadata.EMPTY_MAPPINGS) { builder.field(MapperService.SINGLE_MAPPING_NAME, indexEntry.value.sourceAsMap()); } builder.endObject(); From fe4e7649d1631636be8b1cf9882dbac409af9601 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Mon, 19 Apr 2021 18:51:56 +0200 Subject: [PATCH 31/35] codestyle --- rest-api-spec/build.gradle | 4 +--- .../admin/indices/template/get/GetIndexTemplatesResponse.java | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 891a629708bb0..a826100bb9829 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -180,9 +180,7 @@ tasks.named("yamlRestCompatTest").configure { 'indices.get_mapping/20_missing_type/No type matching pattern returns 404', 'indices.get_mapping/20_missing_type/Non-existent type returns 404', 'indices.get_mapping/20_missing_type/Type missing when no types exist', -// 'indices.get_mapping/40_aliases/Getting mapping for aliases should return the real index as key', -// 'indices.get_mapping/61_empty_with_types/Check empty mapping when getting all mappings via /_mapping', -// 'indices.get_mapping/70_mix_typeless_typeful/GET mapping with typeless API on an index that has types', + // 'indices.open/10_basic/?wait_for_active_shards default is deprecated', 'indices.open/10_basic/?wait_for_active_shards=index-setting', diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index 277f444d24697..40ad35fd1eb5a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -21,7 +21,6 @@ import java.util.Objects; import static java.util.Collections.singletonMap; -import static org.elasticsearch.rest.BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY; import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject { From acdd2d0032757635d8b0ac1e2d42e4c1cc41f7c0 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 20 Apr 2021 13:59:42 +0200 Subject: [PATCH 32/35] remove comment --- .../rest/action/admin/indices/RestPutMappingAction.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java index 1fec5c6b5df38..2879ad16cc984 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java @@ -78,8 +78,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC throw new IllegalArgumentException("Types cannot be provided in put mapping requests, unless " + "the include_type_name parameter is set to true."); } - // TODO what about the type in the source? it should be skipped? - // (different than in RestCreateIndexAction which expects the _doc) + Map mappingSource = prepareV7Mappings(includeTypeName, sourceAsMap); putMappingRequest.source(mappingSource); } else { @@ -103,8 +102,8 @@ private Map prepareV7Mappings(boolean includeTypeName, Map typedMappings = (Map) mappings.get(typeName); - return typedMappings; + Map typelessMappings = (Map) mappings.get(typeName); + return typelessMappings; } return mappings; } From eba8e1e483486d7d87b1bb5e753d9646eb49511f Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 21 Apr 2021 14:26:58 +0200 Subject: [PATCH 33/35] enable index.create tests --- build.gradle | 2 +- .../compat/RestCompatTestTransformTask.java | 4 ++++ .../rest/transform/text/ReplaceIsFalse.java | 4 ++++ rest-api-spec/build.gradle | 17 +++++++++++------ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index fb009aafd9762..c8fbe32000efc 100644 --- a/build.gradle +++ b/build.gradle @@ -190,7 +190,7 @@ tasks.register("verifyVersions") { * after the backport of the backcompat code is complete. */ -boolean bwc_tests_enabled = false +boolean bwc_tests_enabled = true // place a PR link here when committing bwc changes: String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/71922" /* diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 216259f76b61f..994deb9c29440 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -148,6 +148,10 @@ public void replaceIsFalse(String oldValue, Object newValue) { transformations.add(new ReplaceIsFalse(oldValue, MAPPER.convertValue(newValue, TextNode.class))); } + public void replaceIsFalse(String oldValue, Object newValue, String testName) { + transformations.add(new ReplaceIsFalse(oldValue, MAPPER.convertValue(newValue, TextNode.class), testName)); + } + /** * Removes the key/value of a match assertion all project REST tests for the matching subkey. * For example "match":{"_type": "foo"} to "match":{} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/text/ReplaceIsFalse.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/text/ReplaceIsFalse.java index 0260c87d30e58..331c443a6e68f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/text/ReplaceIsFalse.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/transform/text/ReplaceIsFalse.java @@ -14,4 +14,8 @@ public class ReplaceIsFalse extends ReplaceTextual { public ReplaceIsFalse(String valueToBeReplaced, TextNode replacementNode) { super("is_false", valueToBeReplaced, replacementNode); } + + public ReplaceIsFalse(String valueToBeReplaced, TextNode replacementNode, String testName) { + super("is_false", valueToBeReplaced, replacementNode, testName); + } } diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 26e04bfb57564..e06f081d16832 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -160,10 +160,7 @@ tasks.named("yamlRestCompatTest").configure { 'get_source/86_source_missing_with_types/Missing document source with ignore', 'indices.clone/10_basic/Clone index via API', 'indices.create/10_basic/Create index without soft deletes', - // 5 below await retrofitting Removes typed URLs from mapping APIs #41676 - 'indices.create/11_basic_with_types/Create index with mappings', - 'indices.create/20_mix_typeless_typeful/Create a typed index while there is a typeless template', - 'indices.create/20_mix_typeless_typeful/Create a typeless index while there is a typed template', + // type information about the type is removed and not passed down. The logic to check for this is also removed. 'indices.create/20_mix_typeless_typeful/Implicitly create a typed index while there is a typeless template', 'indices.create/20_mix_typeless_typeful/Implicitly create a typeless index while there is a typed template', // @@ -311,8 +308,16 @@ tasks.named("transformV7RestTests").configure({ task -> task.replaceMatch("nodes.\$node_id.roles.9", "remote_cluster_client", "node_info role test") task.removeMatch("nodes.\$node_id.roles.10", "node_info role test") task.replaceIsTrue("test_index.mappings.type_1", "test_index.mappings._doc") - task.replaceIsFalse("test_index.mappings.type_1", "test_index.mappings._doc") - task.replaceIsFalse("test-1.mappings.my_type", "test-1.mappings._doc") + //override for indices.get and indices.create + //task.replaceIsFalse("test_index.mappings.type_1", "test_index.mappings._doc") + //overrides for indices.create/20_mix_typeless_typeful + task.replaceIsFalse("test-1.mappings._doc","false", "Create a typed index while there is a typeless template") + task.replaceIsFalse("test-1.mappings._doc","false", "Create a typeless index while there is a typed template") + + task.replaceIsTrue("test-1.mappings.my_type", "test-1.mappings._doc") + task.replaceIsTrue("test-1.mappings.my_type.properties.foo", "test-1.mappings._doc.properties.foo") + task.replaceIsTrue("test-1.mappings.my_type.properties.bar", "test-1.mappings._doc.properties.bar") + // overrides for indices.get_field_mapping task.replaceKeyInLength("test_index.mappings.test_type.text.mapping.text.type", "test_index.mappings._doc.text.mapping.text.type" From d6fc9950b18072dee6ca5f44373aac8f3fe082dd Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 21 Apr 2021 15:41:29 +0200 Subject: [PATCH 34/35] fix deprecation message --- .../rest/action/admin/indices/RestGetFieldMappingAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index 7fe5edbe5235f..baa2178325ee9 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -40,7 +40,7 @@ public class RestGetFieldMappingAction extends BaseRestHandler { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(logger.getName()); public static final String INCLUDE_TYPE_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get " + "field mapping requests is deprecated. The parameter will be removed in the next major version."; - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in put mapping request is deprecated. " + + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in get field mapping request is deprecated. " + "Use typeless api instead"; @Override public List routes() { From c12f5a2665bcc83528decbb57c67cc37b1e294a1 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 27 Apr 2021 10:49:10 +0200 Subject: [PATCH 35/35] missing method --- .../internal/rest/compat/RestCompatTestTransformTask.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java index 1c7e8509794d5..7c4ef4328d42f 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java @@ -148,7 +148,9 @@ public void replaceIsFalse(String oldValue, Object newValue) { transformations.add(new ReplaceIsFalse(oldValue, MAPPER.convertValue(newValue, TextNode.class))); } - + public void replaceIsFalse(String oldValue, Object newValue, String testName) { + transformations.add(new ReplaceIsFalse(oldValue, MAPPER.convertValue(newValue, TextNode.class), testName)); + } /** * Removes the key/value of a match assertion all project REST tests for the matching subkey.